diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0c732e3257..e76f8eafa8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,11 +31,16 @@ jobs: fetch-depth: 0 - uses: actions/setup-node@v4 with: - node-version: 20.x + node-version: 22.x cache: "npm" cache-dependency-path: | package-lock.json examples/**/package-lock.json + # go needed for fake_gcs_server used by the javascript tests + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version: "stable" - run: npm install - run: npm run format:fix - name: Check for dirty working directory @@ -54,7 +59,6 @@ jobs: - run: npm run build-package - run: npm publish --dry-run working-directory: dist/package - - uses: ./.github/actions/setup-firefox - name: Run JavaScript tests (including WebGL) run: npm test if: ${{ runner.os != 'macOS' }} @@ -99,7 +103,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-node@v4 with: - node-version: 20.x + node-version: 22.x - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -146,7 +150,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-node@v4 with: - node-version: 20.x + node-version: 22.x cache: "npm" - name: Set up Python uses: actions/setup-python@v5 @@ -221,7 +225,7 @@ jobs: - name: Use Node.js uses: actions/setup-node@v4 with: - node-version: 20.x + node-version: 22.x registry-url: "https://registry.npmjs.org" - uses: actions/download-artifact@v4 with: diff --git a/.github/workflows/build_preview.yml b/.github/workflows/build_preview.yml index 9dcac7d1e1..7a6c997612 100644 --- a/.github/workflows/build_preview.yml +++ b/.github/workflows/build_preview.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: node-version: - - "20.x" + - "22.x" runs-on: ubuntu-latest steps: diff --git a/.prettierignore b/.prettierignore index 42541345da..1a4619071f 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,9 +1,12 @@ /templates/ /python/ /third_party/jpgjs/jpg.js -/testdata/*.json +/testdata/**/*.json +zarr.json .parcel-cache dist /lib /docs/_build/ /.ruff_cache +package.json +package-lock.json diff --git a/build_tools/build-package.ts b/build_tools/build-package.ts index f95d924f98..7a2133f807 100644 --- a/build_tools/build-package.ts +++ b/build_tools/build-package.ts @@ -71,6 +71,7 @@ async function buildPackage(options: { outbase: srcDir, bundle: false, outdir: libDir, + target: "es2022", }); let compilerOptionsFromConfigFile: ts.CompilerOptions = {}; diff --git a/build_tools/update-conditions.ts b/build_tools/update-conditions.ts index 873829e161..25e0891f9d 100644 --- a/build_tools/update-conditions.ts +++ b/build_tools/update-conditions.ts @@ -17,70 +17,100 @@ const imports: Record = {}; imports["#src/third_party/jpgjs/jpg.js"] = "./src/third_party/jpgjs/jpg.js"; imports["#src/*.js"] = "./src/*.ts"; imports["#src/*"] = "./src/*"; +imports["#tests/fixtures/msw"] = { + node: "./tests/fixtures/msw_node.ts", + default: "./tests/fixtures/msw_browser.ts", +}; +imports["#tests/fixtures/gl"] = { + node: "./tests/fixtures/gl_node.ts", + default: "./tests/fixtures/gl_browser.ts", +}; +imports["#tests/*.js"] = "./tests/*.ts"; imports["#testdata/*"] = "./testdata/*"; -const datasourceDir = path.resolve(rootDir, "src", "datasource"); -const layerDir = path.resolve(rootDir, "src", "layer"); - -const datasources = ( - await fs.promises.readdir(datasourceDir, { withFileTypes: true }) -) - .filter((e) => e.isDirectory()) - .map((e) => e.name); - -const layers = (await fs.promises.readdir(layerDir, { withFileTypes: true })) - .filter((e) => e.isDirectory()) - .map((e) => e.name); - -const datasourceKeys = { - backend: "backend", - async_computation: "async_computation", - register_default: "frontend", - register_credentials_provider: "frontend", -} as const; +async function listSubdirs(dir: string): Promise { + return (await fs.promises.readdir(dir, { withFileTypes: true })) + .filter((e) => e.isDirectory()) + .map((e) => e.name); +} -const datasourceModules = Object.fromEntries( - Object.values(datasourceKeys).map((key) => [key, new Array()]), -); +async function writeModule(modulePath: string, imports: string[]) { + await fs.promises.writeFile( + modulePath, + "// DO NOT EDIT: Generated by config/update_conditions.ts\n" + + imports.map((name) => `import ${JSON.stringify(name)};\n`).join(""), + { encoding: "utf-8" }, + ); +} -for (const datasource of datasources) { - for (const [filePrefix, moduleKind] of Object.entries(datasourceKeys)) { - const sourcePrefix = `./src/datasource/${datasource}/${filePrefix}`; - if ( - await fs.promises - .stat(path.resolve(rootDir, `${sourcePrefix}.ts`)) - .catch(() => undefined) - ) { - const source = sourcePrefix + JS_EXT; - const conditions: Record = {}; - if (datasource === "python") { - conditions["neuroglancer/python"] = source; - conditions.default = NOOP; - } else { - if (filePrefix === "register_credentials_provider") { - conditions["neuroglancer/python"] = NOOP; +async function handleDrivers( + kind: string, + moduleMap: Record, +) { + const driverDir = path.resolve(rootDir, "src", kind); + const drivers = await listSubdirs(driverDir); + const modules: Record = {}; + for (const driver of drivers) { + for (const [filePrefix, moduleKinds] of Object.entries(moduleMap)) { + const sourcePrefix = `./src/${kind}/${driver}/${filePrefix}`; + if ( + await fs.promises + .stat(path.resolve(rootDir, `${sourcePrefix}.ts`)) + .catch(() => undefined) + ) { + const source = sourcePrefix + JS_EXT; + const conditions: Record = {}; + if (driver === "python") { + conditions["neuroglancer/python"] = source; + conditions.default = NOOP; + } else { + if (filePrefix === "register_credentials_provider") { + conditions["neuroglancer/python"] = NOOP; + } + conditions[`neuroglancer/${kind}/${driver}:enabled`] = source; + conditions[`neuroglancer/${kind}:none_by_default`] = NOOP; + conditions[`neuroglancer/${kind}/${driver}:disabled`] = NOOP; + conditions.default = source; + } + let moduleId = `#${kind}/${driver}`; + if (filePrefix !== "index") { + moduleId += `/${filePrefix}`; + } + imports[moduleId] = conditions; + for (const moduleKind of moduleKinds) { + if (modules[moduleKind] === undefined) { + modules[moduleKind] = []; + } + modules[moduleKind].push(moduleId); } - conditions[`neuroglancer/datasource/${datasource}:enabled`] = source; - conditions["neuroglancer/datasource:none_by_default"] = NOOP; - conditions[`neuroglancer/datasource/${datasource}:disabled`] = source; - conditions.default = source; } - const moduleId = `#datasource/${datasource}/${filePrefix}`; - imports[moduleId] = conditions; - datasourceModules[moduleKind].push(moduleId); } } + for (const [moduleKind, moduleIds] of Object.entries(modules)) { + await writeModule( + path.resolve(driverDir, `enabled_${moduleKind}_modules.ts`), + moduleIds, + ); + } } -for (const layer of layers) { - const source = `./src/layer/${layer}/index` + JS_EXT; - imports[`#layer/${layer}`] = { - [`neuroglancer/layer/${layer}:enabled`]: source, - "neuroglancer/layer:none_by_default": NOOP, - [`neuroglancer/layer/${layer}:enabled`]: source, - default: source, - }; -} +await handleDrivers("datasource", { + backend: ["backend"], + async_computation: ["async_computation"], + register_default: ["frontend"], + register_credentials_provider: ["frontend"], +}); + +await handleDrivers("kvstore", { + register: ["frontend", "backend"], + register_frontend: ["frontend"], + register_backend: ["backend"], + register_credentials_provider: ["frontend"], +}); + +await handleDrivers("layer", { + index: ["frontend"], +}); // main entrypoint. imports["#main"] = { @@ -94,32 +124,6 @@ imports["#python_integration_build"] = { default: NOOP, }; -async function writeModule(modulePath: string, imports: string[]) { - await fs.promises.writeFile( - modulePath, - "// DO NOT EDIT: Generated by config/update_conditions.ts\n" + - imports.map((name) => `import ${JSON.stringify(name)};\n`).join(""), - { encoding: "utf-8" }, - ); -} - -for (const [moduleKind, moduleIds] of Object.entries(datasourceModules)) { - await writeModule( - path.resolve( - rootDir, - "src", - "datasource", - `enabled_${moduleKind}_modules.ts`, - ), - moduleIds, - ); -} - -await writeModule( - path.resolve(rootDir, "src", "layer", "enabled_frontend_modules.ts"), - layers.map((name) => `#layer/${name}`), -); - packageJson.imports = imports; packageJson.exports = { diff --git a/build_tools/vitest/build_fake_gcs_server.ts b/build_tools/vitest/build_fake_gcs_server.ts new file mode 100644 index 0000000000..2009ba7365 --- /dev/null +++ b/build_tools/vitest/build_fake_gcs_server.ts @@ -0,0 +1,57 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { spawnSync } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; + +export async function getFakeGcsServerBin(): Promise { + const binDir = path.join( + import.meta.dirname, + "..", + "..", + "node_modules", + ".cache", + "gobin", + ); + const serverBinPath = + path.join(binDir, "fake-gcs-server") + + (process.platform === "win32" ? ".exe" : ""); + if ( + !(await fs.access(serverBinPath).then( + () => true, + () => false, + )) + ) { + console.log("Building fake-gcs-server"); + // Note: For unknown reasons, using `await promisify(spawn)` in place of + // `spawnSync` causes the vitest process to exit as soon as the child + // process completes. + spawnSync( + "go", + [ + "install", + "github.com/fsouza/fake-gcs-server@3b3d059cbaade55b480196a51dedb7aa82ec2b0a", + ], + { + env: { ...process.env, GOBIN: binDir }, + stdio: ["ignore", "inherit", "inherit"], + }, + ); + console.log("Done building fake-gcs-server"); + } + return serverBinPath; +} diff --git a/build_tools/vitest/fake_ngauth_server.ts b/build_tools/vitest/fake_ngauth_server.ts new file mode 100644 index 0000000000..ed0068b333 --- /dev/null +++ b/build_tools/vitest/fake_ngauth_server.ts @@ -0,0 +1,98 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Fake ngauth server used for tests. + +import type { AddressInfo } from "node:net"; +import cookie from "cookie"; +import express from "express"; + +const COOKIE_NAME = "ngauth_login"; +const COOKIE_VALUE = "fake_login"; +const TOKEN_VALUE = "fake_token"; + +export interface FakeNgauthServer extends AsyncDisposable { + url: string; +} + +export function startFakeNgauthServer(): Promise { + const app = express(); + app.use(express.json({ inflate: false, type: "*/*" })); + app.get("/login", (req, res) => { + const origin = (req.query.origin ?? "").toString(); + res.contentType("text/html"); + // Note: The real ngauth marks this cookie as http-only, but that prevents + // JavaScript from clearing it. + res.cookie(COOKIE_NAME, COOKIE_VALUE, { sameSite: "lax" }); + const jsonToken = JSON.stringify({ token: TOKEN_VALUE }); + const jsonOrigin = JSON.stringify(origin); + res.send(` + + + + + +`); + }); + app.post("/token", (req, res) => { + const cookies = cookie.parse(req.headers.cookie ?? ""); + const origin = req.headers.origin ?? ""; + res.set("x-frame-options", "deny"); + res.set("access-control-allow-origin", origin); + res.set("access-control-allow-credentials", "true"); + res.set("vary", "origin"); + + if (cookies[COOKIE_NAME] === COOKIE_VALUE) { + res.contentType("text/plain"); + res.send(TOKEN_VALUE); + } else { + res.status(401); + res.send(); + } + }); + app.post("/gcs_token", (req, res) => { + const origin = req.headers.origin ?? ""; + res.set("access-control-allow-origin", origin); + res.set("vary", "origin"); + const { body } = req; + if ( + typeof body !== "object" || + Array.isArray(body) || + body.token !== TOKEN_VALUE + ) { + res.status(400); + res.send(); + } else { + res.json({ token: "fake_gcs_token:" + body.bucket }); + } + }); + const server = app.listen(0, "localhost"); + // Don't block node from exiting while this server is running. + server.unref(); + return new Promise((resolve, reject) => { + server.on("error", reject); + server.on("listening", () => { + const port = (server.address() as AddressInfo).port; + resolve({ + url: `http://localhost:${port}`, + [Symbol.asyncDispose]: () => + new Promise((resolve) => server.close(() => resolve())), + }); + }); + }); +} diff --git a/build_tools/vitest/polyfill-browser-globals-in-node.ts b/build_tools/vitest/polyfill-browser-globals-in-node.ts new file mode 100644 index 0000000000..c4d2ce3452 --- /dev/null +++ b/build_tools/vitest/polyfill-browser-globals-in-node.ts @@ -0,0 +1,16 @@ +import { webcrypto } from "node:crypto"; +import type { JSDOM } from "jsdom"; + +declare let jsdom: JSDOM; + +Object.defineProperty(globalThis, "crypto", { + value: webcrypto, +}); + +for (const name of [ + /*"DOMParser", "XPathResult", "navigator"*/ +] as const) { + Object.defineProperty(globalThis, name, { + value: jsdom.window[name], + }); +} diff --git a/build_tools/vitest/setup-crypto.ts b/build_tools/vitest/setup-crypto.ts deleted file mode 100644 index 65e56e55a2..0000000000 --- a/build_tools/vitest/setup-crypto.ts +++ /dev/null @@ -1,7 +0,0 @@ -// Polyfill for missing `crypto` in jsdom. - -import { webcrypto } from "node:crypto"; - -Object.defineProperty(globalThis, "crypto", { - value: webcrypto, -}); diff --git a/build_tools/vitest/test_data_server.ts b/build_tools/vitest/test_data_server.ts new file mode 100644 index 0000000000..cde0e7ce18 --- /dev/null +++ b/build_tools/vitest/test_data_server.ts @@ -0,0 +1,51 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type * as http from "node:http"; +import type { AddressInfo } from "node:net"; +import { createServer } from "http-server"; + +export interface TestDataServer extends AsyncDisposable { + url: string; +} + +export async function startTestDataServer( + rootDirectory: string, +): Promise { + const server: http.Server = ( + createServer({ + root: rootDirectory, + cache: -1, + cors: true, + }) as any + ).server; + // Don't block node from exiting. + server.unref(); + const serverUrl = await new Promise((resolve, reject) => { + server.on("error", reject); + server.listen(0, "localhost", () => { + const port = (server.address() as AddressInfo).port; + resolve(`http://localhost:${port}/`); + }); + }); + console.log(`Serving ${rootDirectory} at ${serverUrl}`); + return { + url: serverUrl, + [Symbol.asyncDispose]: async () => { + await new Promise((resolve) => server.close(resolve)); + }, + }; +} diff --git a/eslint.config.js b/eslint.config.js index e877f40fe4..a7c11e0791 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -59,6 +59,7 @@ export default tseslint.config( "error", { argsIgnorePattern: "^_", + varsIgnorePattern: "^_", ignoreRestSiblings: true, }, ], diff --git a/examples/parcel/parcel-project-built/package-lock.json b/examples/parcel/parcel-project-built/package-lock.json index 52fdcccb14..5923e257ab 100644 --- a/examples/parcel/parcel-project-built/package-lock.json +++ b/examples/parcel/parcel-project-built/package-lock.json @@ -23,7 +23,8 @@ "license": "Apache-2.0", "dependencies": { "codemirror": "^5.61.1", - "core-js": "^3.39.0", + "core-js": "^3.40.0", + "crc-32": "^1.2.2", "gl-matrix": "3.1.0", "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", @@ -31,7 +32,7 @@ "numcodecs": "^0.3.2" }, "engines": { - "node": ">=20.11 <21 || >=21.2" + "node": ">=22" } }, "node_modules/@babel/code-frame": { diff --git a/examples/parcel/parcel-project-source/package-lock.json b/examples/parcel/parcel-project-source/package-lock.json index 8142ea13f6..d948e32fbb 100644 --- a/examples/parcel/parcel-project-source/package-lock.json +++ b/examples/parcel/parcel-project-source/package-lock.json @@ -22,7 +22,8 @@ "license": "Apache-2.0", "dependencies": { "codemirror": "^5.61.1", - "core-js": "^3.39.0", + "core-js": "^3.40.0", + "crc-32": "^1.2.2", "gl-matrix": "3.1.0", "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", @@ -30,38 +31,51 @@ "numcodecs": "^0.3.2" }, "devDependencies": { - "@eslint/js": "^9.16.0", - "@rspack/cli": "^1.1.6", - "@rspack/core": "^1.1.6", + "@eslint/js": "^9.18.0", + "@iodigital/vite-plugin-msw": "^2.0.0", + "@playwright/browser-chromium": "^1.49.1", + "@rspack/cli": "^1.1.8", + "@rspack/core": "^1.1.8", "@types/codemirror": "5.60.15", "@types/gl-matrix": "^2.4.5", + "@types/http-server": "^0.12.4", + "@types/jsdom": "^21.1.7", "@types/lodash-es": "^4.17.12", - "@types/node": "^22.10.2", + "@types/node": "^22.10.7", "@types/nunjucks": "^3.2.6", + "@types/s3rver": "^3.7.4", "@types/yargs": "^17.0.33", - "@vitest/browser": "^2.1.8", - "@vitest/ui": "^2.1.8", + "@vitest/browser": "^3.0.2", + "@vitest/ui": "^3.0.2", + "@vitest/web-worker": "^3.0.2", + "cookie": "^1.0.2", "css-loader": "^7.1.2", - "esbuild": "^0.24.0", - "eslint": "^9.16.0", + "esbuild": "^0.24.2", + "eslint": "^9.18.0", "eslint-import-resolver-typescript": "^3.7.0", "eslint-plugin-import": "^2.31.0", "eslint-rspack-plugin": "^4.2.1", - "glob": "^11.0.0", + "express": "^4.21.2", + "glob": "^11.0.1", + "http-server": "^14.1.1", + "jsdom": "^26.0.0", + "msw": "^2.7.0", "nunjucks": "^3.2.4", + "playwright": "^1.49.1", "prettier": "3.4.2", - "ts-checker-rspack-plugin": "^1.0.3", + "s3rver": "^3.7.1", + "ts-checker-rspack-plugin": "^1.1.1", "tsx": "^4.19.2", - "typescript": "^5.7.2", - "typescript-eslint": "^8.18.0", - "vitest": "^2.1.8", - "webdriverio": "^9.4.1", + "typescript": "^5.7.3", + "typescript-eslint": "^8.20.0", + "vitest": "^3.0.2", "webpack-bundle-analyzer": "^4.10.2", "webpack-merge": "^6.0.1", - "yargs": "^17.7.2" + "yargs": "^17.7.2", + "yauzl": "^3.2.0" }, "engines": { - "node": ">=20.11 <21 || >=21.2" + "node": ">=22" } }, "../../../node_modules/@aashutoshrathi/word-wrap": { @@ -733,6 +747,8 @@ "version": "2.0.1", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "debug": "4.3.4", "extract-zip": "2.0.1", @@ -966,6 +982,8 @@ "version": "5.6.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -977,6 +995,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "defer-to-connect": "^2.0.1" }, @@ -988,8 +1008,6 @@ "version": "1.1.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">= 6" } @@ -997,7 +1015,9 @@ "../../../node_modules/@tootallnate/quickjs-emscripten": { "version": "0.23.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/codemirror": { "version": "5.60.15", @@ -1042,7 +1062,9 @@ "../../../node_modules/@types/http-cache-semantics": { "version": "4.0.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/json-schema": { "version": "7.0.15", @@ -1088,12 +1110,16 @@ "../../../node_modules/@types/which": { "version": "2.0.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/ws": { "version": "8.5.10", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "*" } @@ -1116,6 +1142,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "@types/node": "*" } @@ -1275,6 +1302,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "8.28.0", "@wdio/types": "8.32.2", @@ -1292,6 +1321,8 @@ "version": "8.28.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "chalk": "^5.1.2", "loglevel": "^1.6.0", @@ -1306,6 +1337,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -1317,6 +1350,8 @@ "version": "5.3.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.17.0 || ^14.13 || >=16.0.0" }, @@ -1328,6 +1363,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ansi-regex": "^6.0.1" }, @@ -1341,12 +1378,16 @@ "../../../node_modules/@wdio/protocols": { "version": "8.32.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@wdio/repl": { "version": "8.24.12", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0" }, @@ -1358,6 +1399,8 @@ "version": "8.32.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0" }, @@ -1369,6 +1412,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@puppeteer/browsers": "^1.6.0", "@wdio/logger": "8.28.0", @@ -1566,9 +1611,7 @@ "../../../node_modules/abab": { "version": "2.0.6", "dev": true, - "license": "BSD-3-Clause", - "optional": true, - "peer": true + "license": "BSD-3-Clause" }, "../../../node_modules/abbrev": { "version": "2.0.0", @@ -1592,8 +1635,6 @@ "version": "6.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "acorn": "^7.1.1", "acorn-walk": "^7.1.1" @@ -1603,8 +1644,6 @@ "version": "7.4.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1634,8 +1673,6 @@ "version": "7.2.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">=0.4.0" } @@ -1644,8 +1681,6 @@ "version": "6.0.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "debug": "4" }, @@ -1701,6 +1736,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "archiver-utils": "^4.0.1", "async": "^3.2.4", @@ -1718,6 +1755,8 @@ "version": "4.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "glob": "^8.0.0", "graceful-fs": "^4.2.0", @@ -1734,6 +1773,8 @@ "version": "8.1.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1752,6 +1793,8 @@ "version": "5.1.6", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -1763,6 +1806,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1776,6 +1821,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1794,6 +1841,8 @@ "version": "5.3.0", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "dequal": "^2.0.3" } @@ -1933,6 +1982,8 @@ "version": "0.13.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tslib": "^2.0.1" }, @@ -1943,7 +1994,9 @@ "../../../node_modules/async": { "version": "3.2.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/async-limiter": { "version": "1.0.1", @@ -1978,7 +2031,9 @@ "../../../node_modules/b4a": { "version": "1.6.4", "dev": true, - "license": "ISC" + "license": "ISC", + "optional": true, + "peer": true }, "../../../node_modules/balanced-match": { "version": "1.0.2", @@ -1988,13 +2043,15 @@ "version": "2.2.0", "dev": true, "license": "Apache-2.0", - "optional": true + "optional": true, + "peer": true }, "../../../node_modules/bare-fs": { "version": "2.1.5", "dev": true, "license": "Apache-2.0", "optional": true, + "peer": true, "dependencies": { "bare-events": "^2.0.0", "bare-os": "^2.0.0", @@ -2006,13 +2063,15 @@ "version": "2.2.0", "dev": true, "license": "Apache-2.0", - "optional": true + "optional": true, + "peer": true }, "../../../node_modules/bare-path": { "version": "2.1.0", "dev": true, "license": "Apache-2.0", "optional": true, + "peer": true, "dependencies": { "bare-os": "^2.1.0" } @@ -2034,12 +2093,16 @@ "url": "https://feross.org/support" } ], - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/basic-ftp": { "version": "5.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.0.0" } @@ -2055,6 +2118,8 @@ "version": "1.6.52", "dev": true, "license": "Unlicense", + "optional": true, + "peer": true, "engines": { "node": ">=0.6" } @@ -2063,6 +2128,8 @@ "version": "0.3.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "buffers": "~0.1.1", "chainsaw": "~0.1.0" @@ -2071,7 +2138,9 @@ "../../../node_modules/bluebird": { "version": "3.4.7", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/brace-expansion": { "version": "2.0.1", @@ -2147,6 +2216,8 @@ "version": "1.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.10" } @@ -2154,6 +2225,8 @@ "../../../node_modules/buffers": { "version": "0.1.1", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.2.0" } @@ -2170,6 +2243,8 @@ "version": "7.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" } @@ -2178,6 +2253,8 @@ "version": "10.2.14", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/http-cache-semantics": "^4.0.2", "get-stream": "^6.0.1", @@ -2195,6 +2272,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -2269,6 +2348,8 @@ "version": "0.1.0", "dev": true, "license": "MIT/X11", + "optional": true, + "peer": true, "dependencies": { "traverse": ">=0.3.0 <0.4" } @@ -2313,6 +2394,8 @@ "version": "0.4.16", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "mitt": "3.0.0" }, @@ -2420,6 +2503,8 @@ "version": "9.5.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || >=14" } @@ -2428,6 +2513,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "crc-32": "^1.2.0", "crc32-stream": "^5.0.0", @@ -2442,6 +2529,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -2483,11 +2572,12 @@ "../../../node_modules/core-util-is": { "version": "1.0.3", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/crc-32": { "version": "1.2.2", - "dev": true, "license": "Apache-2.0", "bin": { "crc32": "bin/crc32.njs" @@ -2500,6 +2590,8 @@ "version": "5.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "crc-32": "^1.2.0", "readable-stream": "^3.4.0" @@ -2512,6 +2604,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -2525,6 +2619,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "node-fetch": "^2.6.12" } @@ -2533,6 +2629,8 @@ "version": "2.7.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "whatwg-url": "^5.0.0" }, @@ -2551,17 +2649,23 @@ "../../../node_modules/cross-fetch/node_modules/tr46": { "version": "0.0.3", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/cross-fetch/node_modules/webidl-conversions": { "version": "3.0.1", "dev": true, - "license": "BSD-2-Clause" + "license": "BSD-2-Clause", + "optional": true, + "peer": true }, "../../../node_modules/cross-fetch/node_modules/whatwg-url": { "version": "5.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" @@ -2615,11 +2719,15 @@ }, "../../../node_modules/css-shorthand-properties": { "version": "1.1.1", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "../../../node_modules/css-value": { "version": "0.0.1", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "../../../node_modules/cssesc": { "version": "3.0.0", @@ -2635,16 +2743,12 @@ "../../../node_modules/cssom": { "version": "0.5.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/cssstyle": { "version": "2.3.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "cssom": "~0.3.6" }, @@ -2655,9 +2759,7 @@ "../../../node_modules/cssstyle/node_modules/cssom": { "version": "0.3.8", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/dashdash": { "version": "1.14.1", @@ -2673,6 +2775,8 @@ "version": "4.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 12" } @@ -2681,8 +2785,6 @@ "version": "3.0.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "abab": "^2.0.6", "whatwg-mimetype": "^3.0.0", @@ -2696,8 +2798,6 @@ "version": "3.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "punycode": "^2.1.1" }, @@ -2709,8 +2809,6 @@ "version": "7.0.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=12" } @@ -2719,8 +2817,6 @@ "version": "3.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">=12" } @@ -2729,8 +2825,6 @@ "version": "11.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "tr46": "^3.0.0", "webidl-conversions": "^7.0.0" @@ -2764,6 +2858,8 @@ "version": "6.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -2774,14 +2870,14 @@ "../../../node_modules/decimal.js": { "version": "10.4.3", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/decompress-response": { "version": "6.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "mimic-response": "^3.1.0" }, @@ -2796,6 +2892,8 @@ "version": "3.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -2822,6 +2920,8 @@ "version": "5.1.0", "dev": true, "license": "BSD-3-Clause", + "optional": true, + "peer": true, "engines": { "node": ">=16.0.0" } @@ -2830,6 +2930,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" } @@ -2867,6 +2969,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ast-types": "^0.13.4", "escodegen": "^2.1.0", @@ -2887,6 +2991,8 @@ "version": "2.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=6" } @@ -2907,7 +3013,9 @@ "../../../node_modules/devtools-protocol": { "version": "0.0.1262051", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/diff-sequences": { "version": "29.6.3", @@ -2932,8 +3040,6 @@ "version": "2.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "webidl-conversions": "^5.0.0" }, @@ -2945,8 +3051,6 @@ "version": "5.0.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=8" } @@ -2960,6 +3064,8 @@ "version": "0.1.4", "dev": true, "license": "BSD-3-Clause", + "optional": true, + "peer": true, "dependencies": { "readable-stream": "^2.0.2" } @@ -2980,6 +3086,8 @@ "version": "3.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/which": "^2.0.1", "which": "^2.0.2" @@ -2996,6 +3104,8 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "^8.28.0", "decamelize": "^6.0.0", @@ -3012,6 +3122,8 @@ "version": "3.1.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=16" } @@ -3020,6 +3132,8 @@ "version": "4.0.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "isexe": "^3.1.1" }, @@ -3081,6 +3195,8 @@ "version": "1.4.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "once": "^1.4.0" } @@ -4018,6 +4134,8 @@ "version": "2.0.1", "dev": true, "license": "BSD-2-Clause", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.1.1", "get-stream": "^5.1.0", @@ -4037,6 +4155,8 @@ "version": "5.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "pump": "^3.0.0" }, @@ -4061,7 +4181,9 @@ "../../../node_modules/fast-fifo": { "version": "1.3.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/fast-glob": { "version": "3.3.2", @@ -4127,6 +4249,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" @@ -4235,8 +4359,6 @@ "version": "4.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -4250,6 +4372,8 @@ "version": "2.1.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 14.17" } @@ -4258,6 +4382,8 @@ "version": "4.0.10", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fetch-blob": "^3.1.2" }, @@ -4269,6 +4395,8 @@ "version": "11.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -4282,6 +4410,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -4309,6 +4439,8 @@ "version": "1.0.12", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "graceful-fs": "^4.1.2", "inherits": "~2.0.0", @@ -4323,6 +4455,8 @@ "version": "1.1.11", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -4332,6 +4466,8 @@ "version": "7.2.3", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -4351,6 +4487,8 @@ "version": "3.1.2", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -4362,6 +4500,8 @@ "version": "2.7.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "glob": "^7.1.3" }, @@ -4407,6 +4547,8 @@ "dev": true, "hasInstallScript": true, "license": "MPL-2.0", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "^8.28.0", "decamelize": "^6.0.0", @@ -4428,6 +4570,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -4439,6 +4583,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -4451,6 +4597,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -4463,6 +4611,8 @@ "version": "3.1.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=16" } @@ -4471,6 +4621,8 @@ "version": "4.0.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "isexe": "^3.1.1" }, @@ -4515,6 +4667,8 @@ "version": "7.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=16" }, @@ -4563,6 +4717,8 @@ "version": "6.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "basic-ftp": "^5.0.2", "data-uri-to-buffer": "^6.0.2", @@ -4577,6 +4733,8 @@ "version": "6.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 14" } @@ -4684,6 +4842,8 @@ "version": "12.6.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@sindresorhus/is": "^5.2.0", "@szmarczak/http-timer": "^5.0.1", @@ -4708,6 +4868,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -4723,7 +4885,9 @@ "../../../node_modules/grapheme-splitter": { "version": "1.0.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/graphemer": { "version": "1.4.0", @@ -4840,8 +5004,6 @@ "version": "2.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "whatwg-encoding": "^1.0.5" }, @@ -4857,14 +5019,14 @@ "../../../node_modules/http-cache-semantics": { "version": "4.1.1", "dev": true, - "license": "BSD-2-Clause" + "license": "BSD-2-Clause", + "optional": true, + "peer": true }, "../../../node_modules/http-proxy-agent": { "version": "4.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "@tootallnate/once": "1", "agent-base": "6", @@ -4891,6 +5053,8 @@ "version": "2.2.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "quick-lru": "^5.1.1", "resolve-alpn": "^1.2.0" @@ -4903,8 +5067,6 @@ "version": "5.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "agent-base": "6", "debug": "4" @@ -4959,7 +5121,9 @@ "url": "https://feross.org/support" } ], - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/ignore": { "version": "5.3.0", @@ -5231,6 +5395,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -5275,6 +5441,8 @@ "version": "9.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "jsbn": "1.1.0", "sprintf-js": "^1.1.3" @@ -5286,7 +5454,9 @@ "../../../node_modules/ip-address/node_modules/jsbn": { "version": "1.1.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/is-array-buffer": { "version": "3.0.2", @@ -5445,6 +5615,8 @@ "version": "4.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -5466,9 +5638,7 @@ "../../../node_modules/is-potential-custom-element-name": { "version": "1.0.1", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/is-regex": { "version": "1.1.4", @@ -5676,8 +5846,6 @@ "version": "17.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "abab": "^2.0.5", "acorn": "^8.4.1", @@ -5757,6 +5925,8 @@ "version": "6.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "universalify": "^2.0.0" }, @@ -5768,6 +5938,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -5807,6 +5979,8 @@ "version": "0.33.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -5818,6 +5992,8 @@ "version": "1.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "readable-stream": "^2.0.5" }, @@ -5912,7 +6088,9 @@ "../../../node_modules/listenercount": { "version": "1.0.1", "dev": true, - "license": "ISC" + "license": "ISC", + "optional": true, + "peer": true }, "../../../node_modules/loader-runner": { "version": "4.3.0", @@ -5943,6 +6121,8 @@ "version": "2.2.20", "dev": true, "license": "SEE LICENSE IN LICENSE", + "optional": true, + "peer": true, "dependencies": { "n12": "1.8.23", "type-fest": "2.13.0", @@ -5953,6 +6133,8 @@ "version": "2.13.0", "dev": true, "license": "(MIT OR CC0-1.0)", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" }, @@ -5985,7 +6167,9 @@ "../../../node_modules/lodash.clonedeep": { "version": "4.5.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/lodash.merge": { "version": "4.6.2", @@ -5999,12 +6183,16 @@ "../../../node_modules/lodash.zip": { "version": "4.2.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/loglevel": { "version": "1.9.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 0.6.0" }, @@ -6016,7 +6204,9 @@ "../../../node_modules/loglevel-plugin-prefix": { "version": "0.8.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/loupe": { "version": "2.3.7", @@ -6030,6 +6220,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -6101,6 +6293,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -6139,12 +6333,16 @@ "../../../node_modules/mitt": { "version": "3.0.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/mkdirp": { "version": "0.5.6", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "minimist": "^1.2.6" }, @@ -6179,7 +6377,9 @@ "../../../node_modules/n12": { "version": "1.8.23", "dev": true, - "license": "SEE LICENSE IN LICENSE" + "license": "SEE LICENSE IN LICENSE", + "optional": true, + "peer": true }, "../../../node_modules/nanoid": { "version": "3.3.7", @@ -6214,6 +6414,8 @@ "version": "2.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 0.4.0" } @@ -6239,6 +6441,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.5.0" } @@ -6247,6 +6451,8 @@ "version": "3.3.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", @@ -6284,6 +6490,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -6292,6 +6500,8 @@ "version": "8.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -6313,9 +6523,7 @@ "../../../node_modules/nwsapi": { "version": "2.2.7", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/oauth-sign": { "version": "0.9.0", @@ -6436,6 +6644,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" } @@ -6472,6 +6682,8 @@ "version": "7.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@tootallnate/quickjs-emscripten": "^0.23.0", "agent-base": "^7.0.2", @@ -6490,6 +6702,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -6501,6 +6715,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -6513,6 +6729,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -6525,6 +6743,8 @@ "version": "7.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "degenerator": "^5.0.0", "netmask": "^2.0.2" @@ -6547,9 +6767,7 @@ "../../../node_modules/parse5": { "version": "6.0.1", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/path-exists": { "version": "4.0.0", @@ -6807,12 +7025,16 @@ "../../../node_modules/process-nextick-args": { "version": "2.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/progress": { "version": "2.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.4.0" } @@ -6825,6 +7047,8 @@ "version": "6.4.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "^4.3.4", @@ -6843,6 +7067,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -6854,6 +7080,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -6866,6 +7094,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -6878,6 +7108,8 @@ "version": "7.18.3", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=12" } @@ -6885,7 +7117,9 @@ "../../../node_modules/proxy-from-env": { "version": "1.1.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/psl": { "version": "1.9.0", @@ -6895,6 +7129,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -6911,6 +7147,8 @@ "version": "20.9.0", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "@puppeteer/browsers": "1.4.6", "chromium-bidi": "0.4.16", @@ -6934,12 +7172,16 @@ "../../../node_modules/puppeteer-core/node_modules/devtools-protocol": { "version": "0.0.1147663", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/puppeteer-core/node_modules/ws": { "version": "8.13.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.0.0" }, @@ -6966,14 +7208,14 @@ "../../../node_modules/query-selector-shadow-dom": { "version": "1.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/querystringify": { "version": "2.2.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/queue-microtask": { "version": "1.2.3", @@ -6997,12 +7239,16 @@ "../../../node_modules/queue-tick": { "version": "1.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/quick-lru": { "version": "5.1.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -7029,6 +7275,8 @@ "version": "2.3.8", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -7042,17 +7290,23 @@ "../../../node_modules/readable-stream/node_modules/isarray": { "version": "1.0.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/readable-stream/node_modules/safe-buffer": { "version": "5.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/readdir-glob": { "version": "1.1.3", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "minimatch": "^5.1.0" } @@ -7061,6 +7315,8 @@ "version": "5.1.6", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -7186,9 +7442,7 @@ "../../../node_modules/requires-port": { "version": "1.0.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/resolve": { "version": "1.22.8", @@ -7209,7 +7463,9 @@ "../../../node_modules/resolve-alpn": { "version": "1.2.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/resolve-from": { "version": "4.0.0", @@ -7231,6 +7487,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "lowercase-keys": "^3.0.0" }, @@ -7245,6 +7503,8 @@ "version": "1.11.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fast-deep-equal": "^2.0.1" } @@ -7252,7 +7512,9 @@ "../../../node_modules/resq/node_modules/fast-deep-equal": { "version": "2.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/reusify": { "version": "1.0.4", @@ -7266,7 +7528,9 @@ "../../../node_modules/rgb2hex": { "version": "0.2.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/rimraf": { "version": "3.0.2", @@ -7381,7 +7645,9 @@ "../../../node_modules/safaridriver": { "version": "0.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/safe-array-concat": { "version": "1.1.0", @@ -7446,8 +7712,6 @@ "version": "5.0.1", "dev": true, "license": "ISC", - "optional": true, - "peer": true, "dependencies": { "xmlchars": "^2.2.0" }, @@ -7501,6 +7765,8 @@ "version": "11.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "type-fest": "^2.12.2" }, @@ -7552,7 +7818,9 @@ "../../../node_modules/setimmediate": { "version": "1.0.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/shallow-clone": { "version": "3.0.1", @@ -7635,6 +7903,8 @@ "version": "4.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 6.0.0", "npm": ">= 3.0.0" @@ -7644,6 +7914,8 @@ "version": "2.7.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ip-address": "^9.0.5", "smart-buffer": "^4.2.0" @@ -7657,6 +7929,8 @@ "version": "8.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "^4.3.4", @@ -7670,6 +7944,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -7709,6 +7985,8 @@ "version": "4.2.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">= 10.x" } @@ -7716,7 +7994,9 @@ "../../../node_modules/sprintf-js": { "version": "1.1.3", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/sshpk": { "version": "1.18.0", @@ -7762,6 +8042,8 @@ "version": "2.15.6", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fast-fifo": "^1.1.0", "queue-tick": "^1.0.1" @@ -7771,6 +8053,8 @@ "version": "1.1.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -7778,7 +8062,9 @@ "../../../node_modules/string_decoder/node_modules/safe-buffer": { "version": "5.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/string-width": { "version": "5.1.2", @@ -7982,6 +8268,8 @@ "version": "3.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "pump": "^3.0.0", "tar-stream": "^3.1.5" @@ -7995,6 +8283,8 @@ "version": "3.1.7", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "b4a": "^1.6.4", "fast-fifo": "^1.2.0", @@ -8070,7 +8360,9 @@ "../../../node_modules/through": { "version": "2.3.8", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/tinybench": { "version": "2.6.0", @@ -8116,8 +8408,6 @@ "version": "4.1.3", "dev": true, "license": "BSD-3-Clause", - "optional": true, - "peer": true, "dependencies": { "psl": "^1.1.33", "punycode": "^2.1.1", @@ -8132,8 +8422,6 @@ "version": "2.1.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "punycode": "^2.1.1" }, @@ -8144,7 +8432,9 @@ "../../../node_modules/traverse": { "version": "0.3.9", "dev": true, - "license": "MIT/X11" + "license": "MIT/X11", + "optional": true, + "peer": true }, "../../../node_modules/tsconfig-paths": { "version": "3.15.0", @@ -8171,7 +8461,9 @@ "../../../node_modules/tslib": { "version": "2.6.2", "dev": true, - "license": "0BSD" + "license": "0BSD", + "optional": true, + "peer": true }, "../../../node_modules/tsx": { "version": "4.7.1", @@ -8632,6 +8924,8 @@ "version": "2.19.0", "dev": true, "license": "(MIT OR CC0-1.0)", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" }, @@ -8735,6 +9029,8 @@ "version": "1.4.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "buffer": "^5.2.1", "through": "^2.3.8" @@ -8758,6 +9054,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -8772,8 +9070,6 @@ "version": "0.2.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">= 4.0.0" } @@ -8782,6 +9078,8 @@ "version": "0.10.14", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "big-integer": "^1.6.17", "binary": "~0.3.0", @@ -8837,8 +9135,6 @@ "version": "1.5.10", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" @@ -8847,6 +9143,8 @@ "../../../node_modules/userhome": { "version": "1.0.0", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 0.8.0" } @@ -9092,8 +9390,6 @@ "version": "2.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "xml-name-validator": "^3.0.0" }, @@ -9105,6 +9401,8 @@ "version": "1.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "chalk": "^4.1.2", "commander": "^9.3.0", @@ -9135,6 +9433,8 @@ "version": "3.3.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 8" } @@ -9143,6 +9443,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0", "@types/ws": "^8.5.3", @@ -9164,6 +9466,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0", "@wdio/config": "8.32.3", @@ -9206,8 +9510,6 @@ "version": "6.1.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=10.4" } @@ -9402,8 +9704,6 @@ "version": "9.1.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "tr46": "^2.1.0", "webidl-conversions": "^6.1.0" @@ -9597,9 +9897,7 @@ "../../../node_modules/xmlchars": { "version": "2.2.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/xmldom": { "version": "0.1.31", @@ -9687,6 +9985,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "archiver-utils": "^4.0.1", "compress-commons": "^5.0.1", @@ -9700,6 +10000,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", diff --git a/examples/rsbuild/rsbuild-project-built/package-lock.json b/examples/rsbuild/rsbuild-project-built/package-lock.json index b73b7b5a55..3c908105d8 100644 --- a/examples/rsbuild/rsbuild-project-built/package-lock.json +++ b/examples/rsbuild/rsbuild-project-built/package-lock.json @@ -21,7 +21,8 @@ "license": "Apache-2.0", "dependencies": { "codemirror": "^5.61.1", - "core-js": "^3.39.0", + "core-js": "^3.40.0", + "crc-32": "^1.2.2", "gl-matrix": "3.1.0", "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", @@ -29,7 +30,7 @@ "numcodecs": "^0.3.2" }, "engines": { - "node": ">=20.11 <21 || >=21.2" + "node": ">=22" } }, "node_modules/@module-federation/runtime": { diff --git a/examples/rsbuild/rsbuild-project-source/package-lock.json b/examples/rsbuild/rsbuild-project-source/package-lock.json index cd2de996d0..60f9b77745 100644 --- a/examples/rsbuild/rsbuild-project-source/package-lock.json +++ b/examples/rsbuild/rsbuild-project-source/package-lock.json @@ -20,7 +20,8 @@ "license": "Apache-2.0", "dependencies": { "codemirror": "^5.61.1", - "core-js": "^3.39.0", + "core-js": "^3.40.0", + "crc-32": "^1.2.2", "gl-matrix": "3.1.0", "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", @@ -28,38 +29,51 @@ "numcodecs": "^0.3.2" }, "devDependencies": { - "@eslint/js": "^9.16.0", - "@rspack/cli": "^1.1.6", - "@rspack/core": "^1.1.6", + "@eslint/js": "^9.18.0", + "@iodigital/vite-plugin-msw": "^2.0.0", + "@playwright/browser-chromium": "^1.49.1", + "@rspack/cli": "^1.1.8", + "@rspack/core": "^1.1.8", "@types/codemirror": "5.60.15", "@types/gl-matrix": "^2.4.5", + "@types/http-server": "^0.12.4", + "@types/jsdom": "^21.1.7", "@types/lodash-es": "^4.17.12", - "@types/node": "^22.10.2", + "@types/node": "^22.10.7", "@types/nunjucks": "^3.2.6", + "@types/s3rver": "^3.7.4", "@types/yargs": "^17.0.33", - "@vitest/browser": "^2.1.8", - "@vitest/ui": "^2.1.8", + "@vitest/browser": "^3.0.2", + "@vitest/ui": "^3.0.2", + "@vitest/web-worker": "^3.0.2", + "cookie": "^1.0.2", "css-loader": "^7.1.2", - "esbuild": "^0.24.0", - "eslint": "^9.16.0", + "esbuild": "^0.24.2", + "eslint": "^9.18.0", "eslint-import-resolver-typescript": "^3.7.0", "eslint-plugin-import": "^2.31.0", "eslint-rspack-plugin": "^4.2.1", - "glob": "^11.0.0", + "express": "^4.21.2", + "glob": "^11.0.1", + "http-server": "^14.1.1", + "jsdom": "^26.0.0", + "msw": "^2.7.0", "nunjucks": "^3.2.4", + "playwright": "^1.49.1", "prettier": "3.4.2", - "ts-checker-rspack-plugin": "^1.0.3", + "s3rver": "^3.7.1", + "ts-checker-rspack-plugin": "^1.1.1", "tsx": "^4.19.2", - "typescript": "^5.7.2", - "typescript-eslint": "^8.18.0", - "vitest": "^2.1.8", - "webdriverio": "^9.4.1", + "typescript": "^5.7.3", + "typescript-eslint": "^8.20.0", + "vitest": "^3.0.2", "webpack-bundle-analyzer": "^4.10.2", "webpack-merge": "^6.0.1", - "yargs": "^17.7.2" + "yargs": "^17.7.2", + "yauzl": "^3.2.0" }, "engines": { - "node": ">=20.11 <21 || >=21.2" + "node": ">=22" } }, "../../../node_modules/@aashutoshrathi/word-wrap": { @@ -731,6 +745,8 @@ "version": "2.0.1", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "debug": "4.3.4", "extract-zip": "2.0.1", @@ -964,6 +980,8 @@ "version": "5.6.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -975,6 +993,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "defer-to-connect": "^2.0.1" }, @@ -986,8 +1006,6 @@ "version": "1.1.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">= 6" } @@ -995,7 +1013,9 @@ "../../../node_modules/@tootallnate/quickjs-emscripten": { "version": "0.23.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/codemirror": { "version": "5.60.15", @@ -1040,7 +1060,9 @@ "../../../node_modules/@types/http-cache-semantics": { "version": "4.0.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/json-schema": { "version": "7.0.15", @@ -1086,12 +1108,16 @@ "../../../node_modules/@types/which": { "version": "2.0.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/ws": { "version": "8.5.10", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "*" } @@ -1114,6 +1140,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "@types/node": "*" } @@ -1273,6 +1300,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "8.28.0", "@wdio/types": "8.32.2", @@ -1290,6 +1319,8 @@ "version": "8.28.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "chalk": "^5.1.2", "loglevel": "^1.6.0", @@ -1304,6 +1335,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -1315,6 +1348,8 @@ "version": "5.3.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.17.0 || ^14.13 || >=16.0.0" }, @@ -1326,6 +1361,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ansi-regex": "^6.0.1" }, @@ -1339,12 +1376,16 @@ "../../../node_modules/@wdio/protocols": { "version": "8.32.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@wdio/repl": { "version": "8.24.12", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0" }, @@ -1356,6 +1397,8 @@ "version": "8.32.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0" }, @@ -1367,6 +1410,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@puppeteer/browsers": "^1.6.0", "@wdio/logger": "8.28.0", @@ -1564,9 +1609,7 @@ "../../../node_modules/abab": { "version": "2.0.6", "dev": true, - "license": "BSD-3-Clause", - "optional": true, - "peer": true + "license": "BSD-3-Clause" }, "../../../node_modules/abbrev": { "version": "2.0.0", @@ -1590,8 +1633,6 @@ "version": "6.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "acorn": "^7.1.1", "acorn-walk": "^7.1.1" @@ -1601,8 +1642,6 @@ "version": "7.4.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1632,8 +1671,6 @@ "version": "7.2.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">=0.4.0" } @@ -1642,8 +1679,6 @@ "version": "6.0.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "debug": "4" }, @@ -1699,6 +1734,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "archiver-utils": "^4.0.1", "async": "^3.2.4", @@ -1716,6 +1753,8 @@ "version": "4.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "glob": "^8.0.0", "graceful-fs": "^4.2.0", @@ -1732,6 +1771,8 @@ "version": "8.1.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1750,6 +1791,8 @@ "version": "5.1.6", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -1761,6 +1804,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1774,6 +1819,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1792,6 +1839,8 @@ "version": "5.3.0", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "dequal": "^2.0.3" } @@ -1931,6 +1980,8 @@ "version": "0.13.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tslib": "^2.0.1" }, @@ -1941,7 +1992,9 @@ "../../../node_modules/async": { "version": "3.2.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/async-limiter": { "version": "1.0.1", @@ -1976,7 +2029,9 @@ "../../../node_modules/b4a": { "version": "1.6.4", "dev": true, - "license": "ISC" + "license": "ISC", + "optional": true, + "peer": true }, "../../../node_modules/balanced-match": { "version": "1.0.2", @@ -1986,13 +2041,15 @@ "version": "2.2.0", "dev": true, "license": "Apache-2.0", - "optional": true + "optional": true, + "peer": true }, "../../../node_modules/bare-fs": { "version": "2.1.5", "dev": true, "license": "Apache-2.0", "optional": true, + "peer": true, "dependencies": { "bare-events": "^2.0.0", "bare-os": "^2.0.0", @@ -2004,13 +2061,15 @@ "version": "2.2.0", "dev": true, "license": "Apache-2.0", - "optional": true + "optional": true, + "peer": true }, "../../../node_modules/bare-path": { "version": "2.1.0", "dev": true, "license": "Apache-2.0", "optional": true, + "peer": true, "dependencies": { "bare-os": "^2.1.0" } @@ -2032,12 +2091,16 @@ "url": "https://feross.org/support" } ], - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/basic-ftp": { "version": "5.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.0.0" } @@ -2053,6 +2116,8 @@ "version": "1.6.52", "dev": true, "license": "Unlicense", + "optional": true, + "peer": true, "engines": { "node": ">=0.6" } @@ -2061,6 +2126,8 @@ "version": "0.3.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "buffers": "~0.1.1", "chainsaw": "~0.1.0" @@ -2069,7 +2136,9 @@ "../../../node_modules/bluebird": { "version": "3.4.7", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/brace-expansion": { "version": "2.0.1", @@ -2145,6 +2214,8 @@ "version": "1.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.10" } @@ -2152,6 +2223,8 @@ "../../../node_modules/buffers": { "version": "0.1.1", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.2.0" } @@ -2168,6 +2241,8 @@ "version": "7.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" } @@ -2176,6 +2251,8 @@ "version": "10.2.14", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/http-cache-semantics": "^4.0.2", "get-stream": "^6.0.1", @@ -2193,6 +2270,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -2267,6 +2346,8 @@ "version": "0.1.0", "dev": true, "license": "MIT/X11", + "optional": true, + "peer": true, "dependencies": { "traverse": ">=0.3.0 <0.4" } @@ -2311,6 +2392,8 @@ "version": "0.4.16", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "mitt": "3.0.0" }, @@ -2418,6 +2501,8 @@ "version": "9.5.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || >=14" } @@ -2426,6 +2511,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "crc-32": "^1.2.0", "crc32-stream": "^5.0.0", @@ -2440,6 +2527,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -2481,11 +2570,12 @@ "../../../node_modules/core-util-is": { "version": "1.0.3", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/crc-32": { "version": "1.2.2", - "dev": true, "license": "Apache-2.0", "bin": { "crc32": "bin/crc32.njs" @@ -2498,6 +2588,8 @@ "version": "5.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "crc-32": "^1.2.0", "readable-stream": "^3.4.0" @@ -2510,6 +2602,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -2523,6 +2617,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "node-fetch": "^2.6.12" } @@ -2531,6 +2627,8 @@ "version": "2.7.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "whatwg-url": "^5.0.0" }, @@ -2549,17 +2647,23 @@ "../../../node_modules/cross-fetch/node_modules/tr46": { "version": "0.0.3", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/cross-fetch/node_modules/webidl-conversions": { "version": "3.0.1", "dev": true, - "license": "BSD-2-Clause" + "license": "BSD-2-Clause", + "optional": true, + "peer": true }, "../../../node_modules/cross-fetch/node_modules/whatwg-url": { "version": "5.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" @@ -2613,11 +2717,15 @@ }, "../../../node_modules/css-shorthand-properties": { "version": "1.1.1", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "../../../node_modules/css-value": { "version": "0.0.1", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "../../../node_modules/cssesc": { "version": "3.0.0", @@ -2633,16 +2741,12 @@ "../../../node_modules/cssom": { "version": "0.5.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/cssstyle": { "version": "2.3.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "cssom": "~0.3.6" }, @@ -2653,9 +2757,7 @@ "../../../node_modules/cssstyle/node_modules/cssom": { "version": "0.3.8", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/dashdash": { "version": "1.14.1", @@ -2671,6 +2773,8 @@ "version": "4.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 12" } @@ -2679,8 +2783,6 @@ "version": "3.0.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "abab": "^2.0.6", "whatwg-mimetype": "^3.0.0", @@ -2694,8 +2796,6 @@ "version": "3.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "punycode": "^2.1.1" }, @@ -2707,8 +2807,6 @@ "version": "7.0.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=12" } @@ -2717,8 +2815,6 @@ "version": "3.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">=12" } @@ -2727,8 +2823,6 @@ "version": "11.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "tr46": "^3.0.0", "webidl-conversions": "^7.0.0" @@ -2762,6 +2856,8 @@ "version": "6.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -2772,14 +2868,14 @@ "../../../node_modules/decimal.js": { "version": "10.4.3", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/decompress-response": { "version": "6.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "mimic-response": "^3.1.0" }, @@ -2794,6 +2890,8 @@ "version": "3.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -2820,6 +2918,8 @@ "version": "5.1.0", "dev": true, "license": "BSD-3-Clause", + "optional": true, + "peer": true, "engines": { "node": ">=16.0.0" } @@ -2828,6 +2928,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" } @@ -2865,6 +2967,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ast-types": "^0.13.4", "escodegen": "^2.1.0", @@ -2885,6 +2989,8 @@ "version": "2.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=6" } @@ -2905,7 +3011,9 @@ "../../../node_modules/devtools-protocol": { "version": "0.0.1262051", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/diff-sequences": { "version": "29.6.3", @@ -2930,8 +3038,6 @@ "version": "2.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "webidl-conversions": "^5.0.0" }, @@ -2943,8 +3049,6 @@ "version": "5.0.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=8" } @@ -2958,6 +3062,8 @@ "version": "0.1.4", "dev": true, "license": "BSD-3-Clause", + "optional": true, + "peer": true, "dependencies": { "readable-stream": "^2.0.2" } @@ -2978,6 +3084,8 @@ "version": "3.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/which": "^2.0.1", "which": "^2.0.2" @@ -2994,6 +3102,8 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "^8.28.0", "decamelize": "^6.0.0", @@ -3010,6 +3120,8 @@ "version": "3.1.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=16" } @@ -3018,6 +3130,8 @@ "version": "4.0.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "isexe": "^3.1.1" }, @@ -3079,6 +3193,8 @@ "version": "1.4.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "once": "^1.4.0" } @@ -4016,6 +4132,8 @@ "version": "2.0.1", "dev": true, "license": "BSD-2-Clause", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.1.1", "get-stream": "^5.1.0", @@ -4035,6 +4153,8 @@ "version": "5.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "pump": "^3.0.0" }, @@ -4059,7 +4179,9 @@ "../../../node_modules/fast-fifo": { "version": "1.3.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/fast-glob": { "version": "3.3.2", @@ -4125,6 +4247,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" @@ -4233,8 +4357,6 @@ "version": "4.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -4248,6 +4370,8 @@ "version": "2.1.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 14.17" } @@ -4256,6 +4380,8 @@ "version": "4.0.10", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fetch-blob": "^3.1.2" }, @@ -4267,6 +4393,8 @@ "version": "11.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -4280,6 +4408,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -4307,6 +4437,8 @@ "version": "1.0.12", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "graceful-fs": "^4.1.2", "inherits": "~2.0.0", @@ -4321,6 +4453,8 @@ "version": "1.1.11", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -4330,6 +4464,8 @@ "version": "7.2.3", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -4349,6 +4485,8 @@ "version": "3.1.2", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -4360,6 +4498,8 @@ "version": "2.7.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "glob": "^7.1.3" }, @@ -4405,6 +4545,8 @@ "dev": true, "hasInstallScript": true, "license": "MPL-2.0", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "^8.28.0", "decamelize": "^6.0.0", @@ -4426,6 +4568,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -4437,6 +4581,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -4449,6 +4595,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -4461,6 +4609,8 @@ "version": "3.1.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=16" } @@ -4469,6 +4619,8 @@ "version": "4.0.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "isexe": "^3.1.1" }, @@ -4513,6 +4665,8 @@ "version": "7.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=16" }, @@ -4561,6 +4715,8 @@ "version": "6.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "basic-ftp": "^5.0.2", "data-uri-to-buffer": "^6.0.2", @@ -4575,6 +4731,8 @@ "version": "6.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 14" } @@ -4682,6 +4840,8 @@ "version": "12.6.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@sindresorhus/is": "^5.2.0", "@szmarczak/http-timer": "^5.0.1", @@ -4706,6 +4866,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -4721,7 +4883,9 @@ "../../../node_modules/grapheme-splitter": { "version": "1.0.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/graphemer": { "version": "1.4.0", @@ -4838,8 +5002,6 @@ "version": "2.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "whatwg-encoding": "^1.0.5" }, @@ -4855,14 +5017,14 @@ "../../../node_modules/http-cache-semantics": { "version": "4.1.1", "dev": true, - "license": "BSD-2-Clause" + "license": "BSD-2-Clause", + "optional": true, + "peer": true }, "../../../node_modules/http-proxy-agent": { "version": "4.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "@tootallnate/once": "1", "agent-base": "6", @@ -4889,6 +5051,8 @@ "version": "2.2.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "quick-lru": "^5.1.1", "resolve-alpn": "^1.2.0" @@ -4901,8 +5065,6 @@ "version": "5.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "agent-base": "6", "debug": "4" @@ -4957,7 +5119,9 @@ "url": "https://feross.org/support" } ], - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/ignore": { "version": "5.3.0", @@ -5229,6 +5393,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -5273,6 +5439,8 @@ "version": "9.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "jsbn": "1.1.0", "sprintf-js": "^1.1.3" @@ -5284,7 +5452,9 @@ "../../../node_modules/ip-address/node_modules/jsbn": { "version": "1.1.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/is-array-buffer": { "version": "3.0.2", @@ -5443,6 +5613,8 @@ "version": "4.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -5464,9 +5636,7 @@ "../../../node_modules/is-potential-custom-element-name": { "version": "1.0.1", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/is-regex": { "version": "1.1.4", @@ -5674,8 +5844,6 @@ "version": "17.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "abab": "^2.0.5", "acorn": "^8.4.1", @@ -5755,6 +5923,8 @@ "version": "6.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "universalify": "^2.0.0" }, @@ -5766,6 +5936,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -5805,6 +5977,8 @@ "version": "0.33.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -5816,6 +5990,8 @@ "version": "1.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "readable-stream": "^2.0.5" }, @@ -5910,7 +6086,9 @@ "../../../node_modules/listenercount": { "version": "1.0.1", "dev": true, - "license": "ISC" + "license": "ISC", + "optional": true, + "peer": true }, "../../../node_modules/loader-runner": { "version": "4.3.0", @@ -5941,6 +6119,8 @@ "version": "2.2.20", "dev": true, "license": "SEE LICENSE IN LICENSE", + "optional": true, + "peer": true, "dependencies": { "n12": "1.8.23", "type-fest": "2.13.0", @@ -5951,6 +6131,8 @@ "version": "2.13.0", "dev": true, "license": "(MIT OR CC0-1.0)", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" }, @@ -5983,7 +6165,9 @@ "../../../node_modules/lodash.clonedeep": { "version": "4.5.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/lodash.merge": { "version": "4.6.2", @@ -5997,12 +6181,16 @@ "../../../node_modules/lodash.zip": { "version": "4.2.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/loglevel": { "version": "1.9.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 0.6.0" }, @@ -6014,7 +6202,9 @@ "../../../node_modules/loglevel-plugin-prefix": { "version": "0.8.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/loupe": { "version": "2.3.7", @@ -6028,6 +6218,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -6099,6 +6291,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -6137,12 +6331,16 @@ "../../../node_modules/mitt": { "version": "3.0.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/mkdirp": { "version": "0.5.6", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "minimist": "^1.2.6" }, @@ -6177,7 +6375,9 @@ "../../../node_modules/n12": { "version": "1.8.23", "dev": true, - "license": "SEE LICENSE IN LICENSE" + "license": "SEE LICENSE IN LICENSE", + "optional": true, + "peer": true }, "../../../node_modules/nanoid": { "version": "3.3.7", @@ -6212,6 +6412,8 @@ "version": "2.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 0.4.0" } @@ -6237,6 +6439,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.5.0" } @@ -6245,6 +6449,8 @@ "version": "3.3.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", @@ -6282,6 +6488,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -6290,6 +6498,8 @@ "version": "8.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -6311,9 +6521,7 @@ "../../../node_modules/nwsapi": { "version": "2.2.7", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/oauth-sign": { "version": "0.9.0", @@ -6434,6 +6642,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" } @@ -6470,6 +6680,8 @@ "version": "7.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@tootallnate/quickjs-emscripten": "^0.23.0", "agent-base": "^7.0.2", @@ -6488,6 +6700,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -6499,6 +6713,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -6511,6 +6727,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -6523,6 +6741,8 @@ "version": "7.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "degenerator": "^5.0.0", "netmask": "^2.0.2" @@ -6545,9 +6765,7 @@ "../../../node_modules/parse5": { "version": "6.0.1", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/path-exists": { "version": "4.0.0", @@ -6805,12 +7023,16 @@ "../../../node_modules/process-nextick-args": { "version": "2.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/progress": { "version": "2.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.4.0" } @@ -6823,6 +7045,8 @@ "version": "6.4.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "^4.3.4", @@ -6841,6 +7065,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -6852,6 +7078,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -6864,6 +7092,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -6876,6 +7106,8 @@ "version": "7.18.3", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=12" } @@ -6883,7 +7115,9 @@ "../../../node_modules/proxy-from-env": { "version": "1.1.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/psl": { "version": "1.9.0", @@ -6893,6 +7127,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -6909,6 +7145,8 @@ "version": "20.9.0", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "@puppeteer/browsers": "1.4.6", "chromium-bidi": "0.4.16", @@ -6932,12 +7170,16 @@ "../../../node_modules/puppeteer-core/node_modules/devtools-protocol": { "version": "0.0.1147663", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/puppeteer-core/node_modules/ws": { "version": "8.13.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.0.0" }, @@ -6964,14 +7206,14 @@ "../../../node_modules/query-selector-shadow-dom": { "version": "1.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/querystringify": { "version": "2.2.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/queue-microtask": { "version": "1.2.3", @@ -6995,12 +7237,16 @@ "../../../node_modules/queue-tick": { "version": "1.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/quick-lru": { "version": "5.1.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -7027,6 +7273,8 @@ "version": "2.3.8", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -7040,17 +7288,23 @@ "../../../node_modules/readable-stream/node_modules/isarray": { "version": "1.0.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/readable-stream/node_modules/safe-buffer": { "version": "5.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/readdir-glob": { "version": "1.1.3", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "minimatch": "^5.1.0" } @@ -7059,6 +7313,8 @@ "version": "5.1.6", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -7184,9 +7440,7 @@ "../../../node_modules/requires-port": { "version": "1.0.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/resolve": { "version": "1.22.8", @@ -7207,7 +7461,9 @@ "../../../node_modules/resolve-alpn": { "version": "1.2.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/resolve-from": { "version": "4.0.0", @@ -7229,6 +7485,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "lowercase-keys": "^3.0.0" }, @@ -7243,6 +7501,8 @@ "version": "1.11.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fast-deep-equal": "^2.0.1" } @@ -7250,7 +7510,9 @@ "../../../node_modules/resq/node_modules/fast-deep-equal": { "version": "2.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/reusify": { "version": "1.0.4", @@ -7264,7 +7526,9 @@ "../../../node_modules/rgb2hex": { "version": "0.2.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/rimraf": { "version": "3.0.2", @@ -7379,7 +7643,9 @@ "../../../node_modules/safaridriver": { "version": "0.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/safe-array-concat": { "version": "1.1.0", @@ -7444,8 +7710,6 @@ "version": "5.0.1", "dev": true, "license": "ISC", - "optional": true, - "peer": true, "dependencies": { "xmlchars": "^2.2.0" }, @@ -7499,6 +7763,8 @@ "version": "11.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "type-fest": "^2.12.2" }, @@ -7550,7 +7816,9 @@ "../../../node_modules/setimmediate": { "version": "1.0.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/shallow-clone": { "version": "3.0.1", @@ -7633,6 +7901,8 @@ "version": "4.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 6.0.0", "npm": ">= 3.0.0" @@ -7642,6 +7912,8 @@ "version": "2.7.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ip-address": "^9.0.5", "smart-buffer": "^4.2.0" @@ -7655,6 +7927,8 @@ "version": "8.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "^4.3.4", @@ -7668,6 +7942,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -7707,6 +7983,8 @@ "version": "4.2.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">= 10.x" } @@ -7714,7 +7992,9 @@ "../../../node_modules/sprintf-js": { "version": "1.1.3", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/sshpk": { "version": "1.18.0", @@ -7760,6 +8040,8 @@ "version": "2.15.6", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fast-fifo": "^1.1.0", "queue-tick": "^1.0.1" @@ -7769,6 +8051,8 @@ "version": "1.1.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -7776,7 +8060,9 @@ "../../../node_modules/string_decoder/node_modules/safe-buffer": { "version": "5.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/string-width": { "version": "5.1.2", @@ -7980,6 +8266,8 @@ "version": "3.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "pump": "^3.0.0", "tar-stream": "^3.1.5" @@ -7993,6 +8281,8 @@ "version": "3.1.7", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "b4a": "^1.6.4", "fast-fifo": "^1.2.0", @@ -8068,7 +8358,9 @@ "../../../node_modules/through": { "version": "2.3.8", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/tinybench": { "version": "2.6.0", @@ -8114,8 +8406,6 @@ "version": "4.1.3", "dev": true, "license": "BSD-3-Clause", - "optional": true, - "peer": true, "dependencies": { "psl": "^1.1.33", "punycode": "^2.1.1", @@ -8130,8 +8420,6 @@ "version": "2.1.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "punycode": "^2.1.1" }, @@ -8142,7 +8430,9 @@ "../../../node_modules/traverse": { "version": "0.3.9", "dev": true, - "license": "MIT/X11" + "license": "MIT/X11", + "optional": true, + "peer": true }, "../../../node_modules/tsconfig-paths": { "version": "3.15.0", @@ -8169,7 +8459,9 @@ "../../../node_modules/tslib": { "version": "2.6.2", "dev": true, - "license": "0BSD" + "license": "0BSD", + "optional": true, + "peer": true }, "../../../node_modules/tsx": { "version": "4.7.1", @@ -8630,6 +8922,8 @@ "version": "2.19.0", "dev": true, "license": "(MIT OR CC0-1.0)", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" }, @@ -8733,6 +9027,8 @@ "version": "1.4.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "buffer": "^5.2.1", "through": "^2.3.8" @@ -8756,6 +9052,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -8770,8 +9068,6 @@ "version": "0.2.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">= 4.0.0" } @@ -8780,6 +9076,8 @@ "version": "0.10.14", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "big-integer": "^1.6.17", "binary": "~0.3.0", @@ -8835,8 +9133,6 @@ "version": "1.5.10", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" @@ -8845,6 +9141,8 @@ "../../../node_modules/userhome": { "version": "1.0.0", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 0.8.0" } @@ -9090,8 +9388,6 @@ "version": "2.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "xml-name-validator": "^3.0.0" }, @@ -9103,6 +9399,8 @@ "version": "1.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "chalk": "^4.1.2", "commander": "^9.3.0", @@ -9133,6 +9431,8 @@ "version": "3.3.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 8" } @@ -9141,6 +9441,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0", "@types/ws": "^8.5.3", @@ -9162,6 +9464,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0", "@wdio/config": "8.32.3", @@ -9204,8 +9508,6 @@ "version": "6.1.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=10.4" } @@ -9400,8 +9702,6 @@ "version": "9.1.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "tr46": "^2.1.0", "webidl-conversions": "^6.1.0" @@ -9595,9 +9895,7 @@ "../../../node_modules/xmlchars": { "version": "2.2.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/xmldom": { "version": "0.1.31", @@ -9685,6 +9983,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "archiver-utils": "^4.0.1", "compress-commons": "^5.0.1", @@ -9698,6 +9998,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", diff --git a/examples/rspack/rspack-project-built/package-lock.json b/examples/rspack/rspack-project-built/package-lock.json index b55a287ff8..b434075015 100644 --- a/examples/rspack/rspack-project-built/package-lock.json +++ b/examples/rspack/rspack-project-built/package-lock.json @@ -22,7 +22,8 @@ "license": "Apache-2.0", "dependencies": { "codemirror": "^5.61.1", - "core-js": "^3.39.0", + "core-js": "^3.40.0", + "crc-32": "^1.2.2", "gl-matrix": "3.1.0", "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", @@ -30,7 +31,7 @@ "numcodecs": "^0.3.2" }, "engines": { - "node": ">=20.11 <21 || >=21.2" + "node": ">=22" } }, "node_modules/@discoveryjs/json-ext": { diff --git a/examples/rspack/rspack-project-source/package-lock.json b/examples/rspack/rspack-project-source/package-lock.json index 064914d4c2..014e357e9d 100644 --- a/examples/rspack/rspack-project-source/package-lock.json +++ b/examples/rspack/rspack-project-source/package-lock.json @@ -21,7 +21,8 @@ "license": "Apache-2.0", "dependencies": { "codemirror": "^5.61.1", - "core-js": "^3.39.0", + "core-js": "^3.40.0", + "crc-32": "^1.2.2", "gl-matrix": "3.1.0", "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", @@ -29,38 +30,51 @@ "numcodecs": "^0.3.2" }, "devDependencies": { - "@eslint/js": "^9.16.0", - "@rspack/cli": "^1.1.6", - "@rspack/core": "^1.1.6", + "@eslint/js": "^9.18.0", + "@iodigital/vite-plugin-msw": "^2.0.0", + "@playwright/browser-chromium": "^1.49.1", + "@rspack/cli": "^1.1.8", + "@rspack/core": "^1.1.8", "@types/codemirror": "5.60.15", "@types/gl-matrix": "^2.4.5", + "@types/http-server": "^0.12.4", + "@types/jsdom": "^21.1.7", "@types/lodash-es": "^4.17.12", - "@types/node": "^22.10.2", + "@types/node": "^22.10.7", "@types/nunjucks": "^3.2.6", + "@types/s3rver": "^3.7.4", "@types/yargs": "^17.0.33", - "@vitest/browser": "^2.1.8", - "@vitest/ui": "^2.1.8", + "@vitest/browser": "^3.0.2", + "@vitest/ui": "^3.0.2", + "@vitest/web-worker": "^3.0.2", + "cookie": "^1.0.2", "css-loader": "^7.1.2", - "esbuild": "^0.24.0", - "eslint": "^9.16.0", + "esbuild": "^0.24.2", + "eslint": "^9.18.0", "eslint-import-resolver-typescript": "^3.7.0", "eslint-plugin-import": "^2.31.0", "eslint-rspack-plugin": "^4.2.1", - "glob": "^11.0.0", + "express": "^4.21.2", + "glob": "^11.0.1", + "http-server": "^14.1.1", + "jsdom": "^26.0.0", + "msw": "^2.7.0", "nunjucks": "^3.2.4", + "playwright": "^1.49.1", "prettier": "3.4.2", - "ts-checker-rspack-plugin": "^1.0.3", + "s3rver": "^3.7.1", + "ts-checker-rspack-plugin": "^1.1.1", "tsx": "^4.19.2", - "typescript": "^5.7.2", - "typescript-eslint": "^8.18.0", - "vitest": "^2.1.8", - "webdriverio": "^9.4.1", + "typescript": "^5.7.3", + "typescript-eslint": "^8.20.0", + "vitest": "^3.0.2", "webpack-bundle-analyzer": "^4.10.2", "webpack-merge": "^6.0.1", - "yargs": "^17.7.2" + "yargs": "^17.7.2", + "yauzl": "^3.2.0" }, "engines": { - "node": ">=20.11 <21 || >=21.2" + "node": ">=22" } }, "../../../node_modules/@aashutoshrathi/word-wrap": { @@ -732,6 +746,8 @@ "version": "2.0.1", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "debug": "4.3.4", "extract-zip": "2.0.1", @@ -965,6 +981,8 @@ "version": "5.6.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -976,6 +994,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "defer-to-connect": "^2.0.1" }, @@ -987,8 +1007,6 @@ "version": "1.1.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">= 6" } @@ -996,7 +1014,9 @@ "../../../node_modules/@tootallnate/quickjs-emscripten": { "version": "0.23.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/codemirror": { "version": "5.60.15", @@ -1041,7 +1061,9 @@ "../../../node_modules/@types/http-cache-semantics": { "version": "4.0.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/json-schema": { "version": "7.0.15", @@ -1087,12 +1109,16 @@ "../../../node_modules/@types/which": { "version": "2.0.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/ws": { "version": "8.5.10", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "*" } @@ -1115,6 +1141,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "@types/node": "*" } @@ -1274,6 +1301,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "8.28.0", "@wdio/types": "8.32.2", @@ -1291,6 +1320,8 @@ "version": "8.28.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "chalk": "^5.1.2", "loglevel": "^1.6.0", @@ -1305,6 +1336,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -1316,6 +1349,8 @@ "version": "5.3.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.17.0 || ^14.13 || >=16.0.0" }, @@ -1327,6 +1362,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ansi-regex": "^6.0.1" }, @@ -1340,12 +1377,16 @@ "../../../node_modules/@wdio/protocols": { "version": "8.32.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@wdio/repl": { "version": "8.24.12", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0" }, @@ -1357,6 +1398,8 @@ "version": "8.32.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0" }, @@ -1368,6 +1411,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@puppeteer/browsers": "^1.6.0", "@wdio/logger": "8.28.0", @@ -1565,9 +1610,7 @@ "../../../node_modules/abab": { "version": "2.0.6", "dev": true, - "license": "BSD-3-Clause", - "optional": true, - "peer": true + "license": "BSD-3-Clause" }, "../../../node_modules/abbrev": { "version": "2.0.0", @@ -1591,8 +1634,6 @@ "version": "6.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "acorn": "^7.1.1", "acorn-walk": "^7.1.1" @@ -1602,8 +1643,6 @@ "version": "7.4.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1633,8 +1672,6 @@ "version": "7.2.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">=0.4.0" } @@ -1643,8 +1680,6 @@ "version": "6.0.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "debug": "4" }, @@ -1700,6 +1735,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "archiver-utils": "^4.0.1", "async": "^3.2.4", @@ -1717,6 +1754,8 @@ "version": "4.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "glob": "^8.0.0", "graceful-fs": "^4.2.0", @@ -1733,6 +1772,8 @@ "version": "8.1.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1751,6 +1792,8 @@ "version": "5.1.6", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -1762,6 +1805,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1775,6 +1820,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1793,6 +1840,8 @@ "version": "5.3.0", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "dequal": "^2.0.3" } @@ -1932,6 +1981,8 @@ "version": "0.13.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tslib": "^2.0.1" }, @@ -1942,7 +1993,9 @@ "../../../node_modules/async": { "version": "3.2.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/async-limiter": { "version": "1.0.1", @@ -1977,7 +2030,9 @@ "../../../node_modules/b4a": { "version": "1.6.4", "dev": true, - "license": "ISC" + "license": "ISC", + "optional": true, + "peer": true }, "../../../node_modules/balanced-match": { "version": "1.0.2", @@ -1987,13 +2042,15 @@ "version": "2.2.0", "dev": true, "license": "Apache-2.0", - "optional": true + "optional": true, + "peer": true }, "../../../node_modules/bare-fs": { "version": "2.1.5", "dev": true, "license": "Apache-2.0", "optional": true, + "peer": true, "dependencies": { "bare-events": "^2.0.0", "bare-os": "^2.0.0", @@ -2005,13 +2062,15 @@ "version": "2.2.0", "dev": true, "license": "Apache-2.0", - "optional": true + "optional": true, + "peer": true }, "../../../node_modules/bare-path": { "version": "2.1.0", "dev": true, "license": "Apache-2.0", "optional": true, + "peer": true, "dependencies": { "bare-os": "^2.1.0" } @@ -2033,12 +2092,16 @@ "url": "https://feross.org/support" } ], - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/basic-ftp": { "version": "5.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.0.0" } @@ -2054,6 +2117,8 @@ "version": "1.6.52", "dev": true, "license": "Unlicense", + "optional": true, + "peer": true, "engines": { "node": ">=0.6" } @@ -2062,6 +2127,8 @@ "version": "0.3.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "buffers": "~0.1.1", "chainsaw": "~0.1.0" @@ -2070,7 +2137,9 @@ "../../../node_modules/bluebird": { "version": "3.4.7", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/brace-expansion": { "version": "2.0.1", @@ -2146,6 +2215,8 @@ "version": "1.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.10" } @@ -2153,6 +2224,8 @@ "../../../node_modules/buffers": { "version": "0.1.1", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.2.0" } @@ -2169,6 +2242,8 @@ "version": "7.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" } @@ -2177,6 +2252,8 @@ "version": "10.2.14", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/http-cache-semantics": "^4.0.2", "get-stream": "^6.0.1", @@ -2194,6 +2271,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -2268,6 +2347,8 @@ "version": "0.1.0", "dev": true, "license": "MIT/X11", + "optional": true, + "peer": true, "dependencies": { "traverse": ">=0.3.0 <0.4" } @@ -2312,6 +2393,8 @@ "version": "0.4.16", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "mitt": "3.0.0" }, @@ -2419,6 +2502,8 @@ "version": "9.5.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || >=14" } @@ -2427,6 +2512,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "crc-32": "^1.2.0", "crc32-stream": "^5.0.0", @@ -2441,6 +2528,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -2482,11 +2571,12 @@ "../../../node_modules/core-util-is": { "version": "1.0.3", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/crc-32": { "version": "1.2.2", - "dev": true, "license": "Apache-2.0", "bin": { "crc32": "bin/crc32.njs" @@ -2499,6 +2589,8 @@ "version": "5.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "crc-32": "^1.2.0", "readable-stream": "^3.4.0" @@ -2511,6 +2603,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -2524,6 +2618,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "node-fetch": "^2.6.12" } @@ -2532,6 +2628,8 @@ "version": "2.7.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "whatwg-url": "^5.0.0" }, @@ -2550,17 +2648,23 @@ "../../../node_modules/cross-fetch/node_modules/tr46": { "version": "0.0.3", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/cross-fetch/node_modules/webidl-conversions": { "version": "3.0.1", "dev": true, - "license": "BSD-2-Clause" + "license": "BSD-2-Clause", + "optional": true, + "peer": true }, "../../../node_modules/cross-fetch/node_modules/whatwg-url": { "version": "5.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" @@ -2614,11 +2718,15 @@ }, "../../../node_modules/css-shorthand-properties": { "version": "1.1.1", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "../../../node_modules/css-value": { "version": "0.0.1", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "../../../node_modules/cssesc": { "version": "3.0.0", @@ -2634,16 +2742,12 @@ "../../../node_modules/cssom": { "version": "0.5.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/cssstyle": { "version": "2.3.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "cssom": "~0.3.6" }, @@ -2654,9 +2758,7 @@ "../../../node_modules/cssstyle/node_modules/cssom": { "version": "0.3.8", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/dashdash": { "version": "1.14.1", @@ -2672,6 +2774,8 @@ "version": "4.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 12" } @@ -2680,8 +2784,6 @@ "version": "3.0.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "abab": "^2.0.6", "whatwg-mimetype": "^3.0.0", @@ -2695,8 +2797,6 @@ "version": "3.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "punycode": "^2.1.1" }, @@ -2708,8 +2808,6 @@ "version": "7.0.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=12" } @@ -2718,8 +2816,6 @@ "version": "3.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">=12" } @@ -2728,8 +2824,6 @@ "version": "11.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "tr46": "^3.0.0", "webidl-conversions": "^7.0.0" @@ -2763,6 +2857,8 @@ "version": "6.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -2773,14 +2869,14 @@ "../../../node_modules/decimal.js": { "version": "10.4.3", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/decompress-response": { "version": "6.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "mimic-response": "^3.1.0" }, @@ -2795,6 +2891,8 @@ "version": "3.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -2821,6 +2919,8 @@ "version": "5.1.0", "dev": true, "license": "BSD-3-Clause", + "optional": true, + "peer": true, "engines": { "node": ">=16.0.0" } @@ -2829,6 +2929,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" } @@ -2866,6 +2968,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ast-types": "^0.13.4", "escodegen": "^2.1.0", @@ -2886,6 +2990,8 @@ "version": "2.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=6" } @@ -2906,7 +3012,9 @@ "../../../node_modules/devtools-protocol": { "version": "0.0.1262051", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/diff-sequences": { "version": "29.6.3", @@ -2931,8 +3039,6 @@ "version": "2.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "webidl-conversions": "^5.0.0" }, @@ -2944,8 +3050,6 @@ "version": "5.0.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=8" } @@ -2959,6 +3063,8 @@ "version": "0.1.4", "dev": true, "license": "BSD-3-Clause", + "optional": true, + "peer": true, "dependencies": { "readable-stream": "^2.0.2" } @@ -2979,6 +3085,8 @@ "version": "3.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/which": "^2.0.1", "which": "^2.0.2" @@ -2995,6 +3103,8 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "^8.28.0", "decamelize": "^6.0.0", @@ -3011,6 +3121,8 @@ "version": "3.1.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=16" } @@ -3019,6 +3131,8 @@ "version": "4.0.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "isexe": "^3.1.1" }, @@ -3080,6 +3194,8 @@ "version": "1.4.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "once": "^1.4.0" } @@ -4017,6 +4133,8 @@ "version": "2.0.1", "dev": true, "license": "BSD-2-Clause", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.1.1", "get-stream": "^5.1.0", @@ -4036,6 +4154,8 @@ "version": "5.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "pump": "^3.0.0" }, @@ -4060,7 +4180,9 @@ "../../../node_modules/fast-fifo": { "version": "1.3.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/fast-glob": { "version": "3.3.2", @@ -4126,6 +4248,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" @@ -4234,8 +4358,6 @@ "version": "4.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -4249,6 +4371,8 @@ "version": "2.1.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 14.17" } @@ -4257,6 +4381,8 @@ "version": "4.0.10", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fetch-blob": "^3.1.2" }, @@ -4268,6 +4394,8 @@ "version": "11.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -4281,6 +4409,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -4308,6 +4438,8 @@ "version": "1.0.12", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "graceful-fs": "^4.1.2", "inherits": "~2.0.0", @@ -4322,6 +4454,8 @@ "version": "1.1.11", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -4331,6 +4465,8 @@ "version": "7.2.3", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -4350,6 +4486,8 @@ "version": "3.1.2", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -4361,6 +4499,8 @@ "version": "2.7.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "glob": "^7.1.3" }, @@ -4406,6 +4546,8 @@ "dev": true, "hasInstallScript": true, "license": "MPL-2.0", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "^8.28.0", "decamelize": "^6.0.0", @@ -4427,6 +4569,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -4438,6 +4582,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -4450,6 +4596,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -4462,6 +4610,8 @@ "version": "3.1.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=16" } @@ -4470,6 +4620,8 @@ "version": "4.0.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "isexe": "^3.1.1" }, @@ -4514,6 +4666,8 @@ "version": "7.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=16" }, @@ -4562,6 +4716,8 @@ "version": "6.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "basic-ftp": "^5.0.2", "data-uri-to-buffer": "^6.0.2", @@ -4576,6 +4732,8 @@ "version": "6.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 14" } @@ -4683,6 +4841,8 @@ "version": "12.6.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@sindresorhus/is": "^5.2.0", "@szmarczak/http-timer": "^5.0.1", @@ -4707,6 +4867,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -4722,7 +4884,9 @@ "../../../node_modules/grapheme-splitter": { "version": "1.0.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/graphemer": { "version": "1.4.0", @@ -4839,8 +5003,6 @@ "version": "2.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "whatwg-encoding": "^1.0.5" }, @@ -4856,14 +5018,14 @@ "../../../node_modules/http-cache-semantics": { "version": "4.1.1", "dev": true, - "license": "BSD-2-Clause" + "license": "BSD-2-Clause", + "optional": true, + "peer": true }, "../../../node_modules/http-proxy-agent": { "version": "4.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "@tootallnate/once": "1", "agent-base": "6", @@ -4890,6 +5052,8 @@ "version": "2.2.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "quick-lru": "^5.1.1", "resolve-alpn": "^1.2.0" @@ -4902,8 +5066,6 @@ "version": "5.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "agent-base": "6", "debug": "4" @@ -4958,7 +5120,9 @@ "url": "https://feross.org/support" } ], - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/ignore": { "version": "5.3.0", @@ -5230,6 +5394,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -5274,6 +5440,8 @@ "version": "9.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "jsbn": "1.1.0", "sprintf-js": "^1.1.3" @@ -5285,7 +5453,9 @@ "../../../node_modules/ip-address/node_modules/jsbn": { "version": "1.1.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/is-array-buffer": { "version": "3.0.2", @@ -5444,6 +5614,8 @@ "version": "4.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -5465,9 +5637,7 @@ "../../../node_modules/is-potential-custom-element-name": { "version": "1.0.1", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/is-regex": { "version": "1.1.4", @@ -5675,8 +5845,6 @@ "version": "17.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "abab": "^2.0.5", "acorn": "^8.4.1", @@ -5756,6 +5924,8 @@ "version": "6.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "universalify": "^2.0.0" }, @@ -5767,6 +5937,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -5806,6 +5978,8 @@ "version": "0.33.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -5817,6 +5991,8 @@ "version": "1.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "readable-stream": "^2.0.5" }, @@ -5911,7 +6087,9 @@ "../../../node_modules/listenercount": { "version": "1.0.1", "dev": true, - "license": "ISC" + "license": "ISC", + "optional": true, + "peer": true }, "../../../node_modules/loader-runner": { "version": "4.3.0", @@ -5942,6 +6120,8 @@ "version": "2.2.20", "dev": true, "license": "SEE LICENSE IN LICENSE", + "optional": true, + "peer": true, "dependencies": { "n12": "1.8.23", "type-fest": "2.13.0", @@ -5952,6 +6132,8 @@ "version": "2.13.0", "dev": true, "license": "(MIT OR CC0-1.0)", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" }, @@ -5984,7 +6166,9 @@ "../../../node_modules/lodash.clonedeep": { "version": "4.5.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/lodash.merge": { "version": "4.6.2", @@ -5998,12 +6182,16 @@ "../../../node_modules/lodash.zip": { "version": "4.2.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/loglevel": { "version": "1.9.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 0.6.0" }, @@ -6015,7 +6203,9 @@ "../../../node_modules/loglevel-plugin-prefix": { "version": "0.8.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/loupe": { "version": "2.3.7", @@ -6029,6 +6219,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -6100,6 +6292,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -6138,12 +6332,16 @@ "../../../node_modules/mitt": { "version": "3.0.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/mkdirp": { "version": "0.5.6", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "minimist": "^1.2.6" }, @@ -6178,7 +6376,9 @@ "../../../node_modules/n12": { "version": "1.8.23", "dev": true, - "license": "SEE LICENSE IN LICENSE" + "license": "SEE LICENSE IN LICENSE", + "optional": true, + "peer": true }, "../../../node_modules/nanoid": { "version": "3.3.7", @@ -6213,6 +6413,8 @@ "version": "2.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 0.4.0" } @@ -6238,6 +6440,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.5.0" } @@ -6246,6 +6450,8 @@ "version": "3.3.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", @@ -6283,6 +6489,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -6291,6 +6499,8 @@ "version": "8.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -6312,9 +6522,7 @@ "../../../node_modules/nwsapi": { "version": "2.2.7", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/oauth-sign": { "version": "0.9.0", @@ -6435,6 +6643,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" } @@ -6471,6 +6681,8 @@ "version": "7.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@tootallnate/quickjs-emscripten": "^0.23.0", "agent-base": "^7.0.2", @@ -6489,6 +6701,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -6500,6 +6714,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -6512,6 +6728,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -6524,6 +6742,8 @@ "version": "7.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "degenerator": "^5.0.0", "netmask": "^2.0.2" @@ -6546,9 +6766,7 @@ "../../../node_modules/parse5": { "version": "6.0.1", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/path-exists": { "version": "4.0.0", @@ -6806,12 +7024,16 @@ "../../../node_modules/process-nextick-args": { "version": "2.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/progress": { "version": "2.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.4.0" } @@ -6824,6 +7046,8 @@ "version": "6.4.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "^4.3.4", @@ -6842,6 +7066,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -6853,6 +7079,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -6865,6 +7093,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -6877,6 +7107,8 @@ "version": "7.18.3", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=12" } @@ -6884,7 +7116,9 @@ "../../../node_modules/proxy-from-env": { "version": "1.1.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/psl": { "version": "1.9.0", @@ -6894,6 +7128,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -6910,6 +7146,8 @@ "version": "20.9.0", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "@puppeteer/browsers": "1.4.6", "chromium-bidi": "0.4.16", @@ -6933,12 +7171,16 @@ "../../../node_modules/puppeteer-core/node_modules/devtools-protocol": { "version": "0.0.1147663", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/puppeteer-core/node_modules/ws": { "version": "8.13.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.0.0" }, @@ -6965,14 +7207,14 @@ "../../../node_modules/query-selector-shadow-dom": { "version": "1.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/querystringify": { "version": "2.2.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/queue-microtask": { "version": "1.2.3", @@ -6996,12 +7238,16 @@ "../../../node_modules/queue-tick": { "version": "1.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/quick-lru": { "version": "5.1.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -7028,6 +7274,8 @@ "version": "2.3.8", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -7041,17 +7289,23 @@ "../../../node_modules/readable-stream/node_modules/isarray": { "version": "1.0.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/readable-stream/node_modules/safe-buffer": { "version": "5.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/readdir-glob": { "version": "1.1.3", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "minimatch": "^5.1.0" } @@ -7060,6 +7314,8 @@ "version": "5.1.6", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -7185,9 +7441,7 @@ "../../../node_modules/requires-port": { "version": "1.0.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/resolve": { "version": "1.22.8", @@ -7208,7 +7462,9 @@ "../../../node_modules/resolve-alpn": { "version": "1.2.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/resolve-from": { "version": "4.0.0", @@ -7230,6 +7486,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "lowercase-keys": "^3.0.0" }, @@ -7244,6 +7502,8 @@ "version": "1.11.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fast-deep-equal": "^2.0.1" } @@ -7251,7 +7511,9 @@ "../../../node_modules/resq/node_modules/fast-deep-equal": { "version": "2.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/reusify": { "version": "1.0.4", @@ -7265,7 +7527,9 @@ "../../../node_modules/rgb2hex": { "version": "0.2.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/rimraf": { "version": "3.0.2", @@ -7380,7 +7644,9 @@ "../../../node_modules/safaridriver": { "version": "0.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/safe-array-concat": { "version": "1.1.0", @@ -7445,8 +7711,6 @@ "version": "5.0.1", "dev": true, "license": "ISC", - "optional": true, - "peer": true, "dependencies": { "xmlchars": "^2.2.0" }, @@ -7500,6 +7764,8 @@ "version": "11.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "type-fest": "^2.12.2" }, @@ -7551,7 +7817,9 @@ "../../../node_modules/setimmediate": { "version": "1.0.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/shallow-clone": { "version": "3.0.1", @@ -7634,6 +7902,8 @@ "version": "4.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 6.0.0", "npm": ">= 3.0.0" @@ -7643,6 +7913,8 @@ "version": "2.7.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ip-address": "^9.0.5", "smart-buffer": "^4.2.0" @@ -7656,6 +7928,8 @@ "version": "8.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "^4.3.4", @@ -7669,6 +7943,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -7708,6 +7984,8 @@ "version": "4.2.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">= 10.x" } @@ -7715,7 +7993,9 @@ "../../../node_modules/sprintf-js": { "version": "1.1.3", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/sshpk": { "version": "1.18.0", @@ -7761,6 +8041,8 @@ "version": "2.15.6", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fast-fifo": "^1.1.0", "queue-tick": "^1.0.1" @@ -7770,6 +8052,8 @@ "version": "1.1.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -7777,7 +8061,9 @@ "../../../node_modules/string_decoder/node_modules/safe-buffer": { "version": "5.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/string-width": { "version": "5.1.2", @@ -7981,6 +8267,8 @@ "version": "3.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "pump": "^3.0.0", "tar-stream": "^3.1.5" @@ -7994,6 +8282,8 @@ "version": "3.1.7", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "b4a": "^1.6.4", "fast-fifo": "^1.2.0", @@ -8069,7 +8359,9 @@ "../../../node_modules/through": { "version": "2.3.8", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/tinybench": { "version": "2.6.0", @@ -8115,8 +8407,6 @@ "version": "4.1.3", "dev": true, "license": "BSD-3-Clause", - "optional": true, - "peer": true, "dependencies": { "psl": "^1.1.33", "punycode": "^2.1.1", @@ -8131,8 +8421,6 @@ "version": "2.1.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "punycode": "^2.1.1" }, @@ -8143,7 +8431,9 @@ "../../../node_modules/traverse": { "version": "0.3.9", "dev": true, - "license": "MIT/X11" + "license": "MIT/X11", + "optional": true, + "peer": true }, "../../../node_modules/tsconfig-paths": { "version": "3.15.0", @@ -8170,7 +8460,9 @@ "../../../node_modules/tslib": { "version": "2.6.2", "dev": true, - "license": "0BSD" + "license": "0BSD", + "optional": true, + "peer": true }, "../../../node_modules/tsx": { "version": "4.7.1", @@ -8631,6 +8923,8 @@ "version": "2.19.0", "dev": true, "license": "(MIT OR CC0-1.0)", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" }, @@ -8734,6 +9028,8 @@ "version": "1.4.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "buffer": "^5.2.1", "through": "^2.3.8" @@ -8757,6 +9053,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -8771,8 +9069,6 @@ "version": "0.2.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">= 4.0.0" } @@ -8781,6 +9077,8 @@ "version": "0.10.14", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "big-integer": "^1.6.17", "binary": "~0.3.0", @@ -8836,8 +9134,6 @@ "version": "1.5.10", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" @@ -8846,6 +9142,8 @@ "../../../node_modules/userhome": { "version": "1.0.0", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 0.8.0" } @@ -9091,8 +9389,6 @@ "version": "2.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "xml-name-validator": "^3.0.0" }, @@ -9104,6 +9400,8 @@ "version": "1.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "chalk": "^4.1.2", "commander": "^9.3.0", @@ -9134,6 +9432,8 @@ "version": "3.3.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 8" } @@ -9142,6 +9442,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0", "@types/ws": "^8.5.3", @@ -9163,6 +9465,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0", "@wdio/config": "8.32.3", @@ -9205,8 +9509,6 @@ "version": "6.1.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=10.4" } @@ -9401,8 +9703,6 @@ "version": "9.1.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "tr46": "^2.1.0", "webidl-conversions": "^6.1.0" @@ -9596,9 +9896,7 @@ "../../../node_modules/xmlchars": { "version": "2.2.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/xmldom": { "version": "0.1.31", @@ -9686,6 +9984,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "archiver-utils": "^4.0.1", "compress-commons": "^5.0.1", @@ -9699,6 +9999,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", diff --git a/examples/vite/vite-project-built/package-lock.json b/examples/vite/vite-project-built/package-lock.json index 4ce8cdcdc4..eb70c5c6d2 100644 --- a/examples/vite/vite-project-built/package-lock.json +++ b/examples/vite/vite-project-built/package-lock.json @@ -21,7 +21,8 @@ "license": "Apache-2.0", "dependencies": { "codemirror": "^5.61.1", - "core-js": "^3.39.0", + "core-js": "^3.40.0", + "crc-32": "^1.2.2", "gl-matrix": "3.1.0", "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", @@ -29,7 +30,7 @@ "numcodecs": "^0.3.2" }, "engines": { - "node": ">=20.11 <21 || >=21.2" + "node": ">=22" } }, "node_modules/@esbuild/aix-ppc64": { diff --git a/examples/vite/vite-project-source/package-lock.json b/examples/vite/vite-project-source/package-lock.json index b9561fd2cc..e163cabd93 100644 --- a/examples/vite/vite-project-source/package-lock.json +++ b/examples/vite/vite-project-source/package-lock.json @@ -20,7 +20,8 @@ "license": "Apache-2.0", "dependencies": { "codemirror": "^5.61.1", - "core-js": "^3.39.0", + "core-js": "^3.40.0", + "crc-32": "^1.2.2", "gl-matrix": "3.1.0", "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", @@ -28,38 +29,51 @@ "numcodecs": "^0.3.2" }, "devDependencies": { - "@eslint/js": "^9.16.0", - "@rspack/cli": "^1.1.6", - "@rspack/core": "^1.1.6", + "@eslint/js": "^9.18.0", + "@iodigital/vite-plugin-msw": "^2.0.0", + "@playwright/browser-chromium": "^1.49.1", + "@rspack/cli": "^1.1.8", + "@rspack/core": "^1.1.8", "@types/codemirror": "5.60.15", "@types/gl-matrix": "^2.4.5", + "@types/http-server": "^0.12.4", + "@types/jsdom": "^21.1.7", "@types/lodash-es": "^4.17.12", - "@types/node": "^22.10.2", + "@types/node": "^22.10.7", "@types/nunjucks": "^3.2.6", + "@types/s3rver": "^3.7.4", "@types/yargs": "^17.0.33", - "@vitest/browser": "^2.1.8", - "@vitest/ui": "^2.1.8", + "@vitest/browser": "^3.0.2", + "@vitest/ui": "^3.0.2", + "@vitest/web-worker": "^3.0.2", + "cookie": "^1.0.2", "css-loader": "^7.1.2", - "esbuild": "^0.24.0", - "eslint": "^9.16.0", + "esbuild": "^0.24.2", + "eslint": "^9.18.0", "eslint-import-resolver-typescript": "^3.7.0", "eslint-plugin-import": "^2.31.0", "eslint-rspack-plugin": "^4.2.1", - "glob": "^11.0.0", + "express": "^4.21.2", + "glob": "^11.0.1", + "http-server": "^14.1.1", + "jsdom": "^26.0.0", + "msw": "^2.7.0", "nunjucks": "^3.2.4", + "playwright": "^1.49.1", "prettier": "3.4.2", - "ts-checker-rspack-plugin": "^1.0.3", + "s3rver": "^3.7.1", + "ts-checker-rspack-plugin": "^1.1.1", "tsx": "^4.19.2", - "typescript": "^5.7.2", - "typescript-eslint": "^8.18.0", - "vitest": "^2.1.8", - "webdriverio": "^9.4.1", + "typescript": "^5.7.3", + "typescript-eslint": "^8.20.0", + "vitest": "^3.0.2", "webpack-bundle-analyzer": "^4.10.2", "webpack-merge": "^6.0.1", - "yargs": "^17.7.2" + "yargs": "^17.7.2", + "yauzl": "^3.2.0" }, "engines": { - "node": ">=20.11 <21 || >=21.2" + "node": ">=22" } }, "../../../node_modules/@aashutoshrathi/word-wrap": { @@ -379,6 +393,8 @@ "version": "2.0.1", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "debug": "4.3.4", "extract-zip": "2.0.1", @@ -428,6 +444,8 @@ "version": "5.6.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -439,6 +457,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "defer-to-connect": "^2.0.1" }, @@ -450,8 +470,6 @@ "version": "1.1.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">= 6" } @@ -459,7 +477,9 @@ "../../../node_modules/@tootallnate/quickjs-emscripten": { "version": "0.23.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/codemirror": { "version": "5.60.15", @@ -504,7 +524,9 @@ "../../../node_modules/@types/http-cache-semantics": { "version": "4.0.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/json-schema": { "version": "7.0.15", @@ -550,12 +572,16 @@ "../../../node_modules/@types/which": { "version": "2.0.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/ws": { "version": "8.5.10", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "*" } @@ -578,6 +604,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "@types/node": "*" } @@ -737,6 +764,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "8.28.0", "@wdio/types": "8.32.2", @@ -754,6 +783,8 @@ "version": "8.28.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "chalk": "^5.1.2", "loglevel": "^1.6.0", @@ -768,6 +799,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -779,6 +812,8 @@ "version": "5.3.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.17.0 || ^14.13 || >=16.0.0" }, @@ -790,6 +825,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ansi-regex": "^6.0.1" }, @@ -803,12 +840,16 @@ "../../../node_modules/@wdio/protocols": { "version": "8.32.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@wdio/repl": { "version": "8.24.12", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0" }, @@ -820,6 +861,8 @@ "version": "8.32.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0" }, @@ -831,6 +874,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@puppeteer/browsers": "^1.6.0", "@wdio/logger": "8.28.0", @@ -1028,9 +1073,7 @@ "../../../node_modules/abab": { "version": "2.0.6", "dev": true, - "license": "BSD-3-Clause", - "optional": true, - "peer": true + "license": "BSD-3-Clause" }, "../../../node_modules/abbrev": { "version": "2.0.0", @@ -1054,8 +1097,6 @@ "version": "6.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "acorn": "^7.1.1", "acorn-walk": "^7.1.1" @@ -1065,8 +1106,6 @@ "version": "7.4.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1096,8 +1135,6 @@ "version": "7.2.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">=0.4.0" } @@ -1106,8 +1143,6 @@ "version": "6.0.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "debug": "4" }, @@ -1163,6 +1198,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "archiver-utils": "^4.0.1", "async": "^3.2.4", @@ -1180,6 +1217,8 @@ "version": "4.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "glob": "^8.0.0", "graceful-fs": "^4.2.0", @@ -1196,6 +1235,8 @@ "version": "8.1.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1214,6 +1255,8 @@ "version": "5.1.6", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -1225,6 +1268,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1238,6 +1283,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1256,6 +1303,8 @@ "version": "5.3.0", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "dequal": "^2.0.3" } @@ -1395,6 +1444,8 @@ "version": "0.13.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tslib": "^2.0.1" }, @@ -1405,7 +1456,9 @@ "../../../node_modules/async": { "version": "3.2.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/async-limiter": { "version": "1.0.1", @@ -1440,7 +1493,9 @@ "../../../node_modules/b4a": { "version": "1.6.4", "dev": true, - "license": "ISC" + "license": "ISC", + "optional": true, + "peer": true }, "../../../node_modules/balanced-match": { "version": "1.0.2", @@ -1450,13 +1505,15 @@ "version": "2.2.0", "dev": true, "license": "Apache-2.0", - "optional": true + "optional": true, + "peer": true }, "../../../node_modules/bare-fs": { "version": "2.1.5", "dev": true, "license": "Apache-2.0", "optional": true, + "peer": true, "dependencies": { "bare-events": "^2.0.0", "bare-os": "^2.0.0", @@ -1468,13 +1525,15 @@ "version": "2.2.0", "dev": true, "license": "Apache-2.0", - "optional": true + "optional": true, + "peer": true }, "../../../node_modules/bare-path": { "version": "2.1.0", "dev": true, "license": "Apache-2.0", "optional": true, + "peer": true, "dependencies": { "bare-os": "^2.1.0" } @@ -1496,12 +1555,16 @@ "url": "https://feross.org/support" } ], - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/basic-ftp": { "version": "5.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.0.0" } @@ -1517,6 +1580,8 @@ "version": "1.6.52", "dev": true, "license": "Unlicense", + "optional": true, + "peer": true, "engines": { "node": ">=0.6" } @@ -1525,6 +1590,8 @@ "version": "0.3.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "buffers": "~0.1.1", "chainsaw": "~0.1.0" @@ -1533,7 +1600,9 @@ "../../../node_modules/bluebird": { "version": "3.4.7", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/brace-expansion": { "version": "2.0.1", @@ -1609,6 +1678,8 @@ "version": "1.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.10" } @@ -1616,6 +1687,8 @@ "../../../node_modules/buffers": { "version": "0.1.1", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.2.0" } @@ -1632,6 +1705,8 @@ "version": "7.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" } @@ -1640,6 +1715,8 @@ "version": "10.2.14", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/http-cache-semantics": "^4.0.2", "get-stream": "^6.0.1", @@ -1657,6 +1734,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -1731,6 +1810,8 @@ "version": "0.1.0", "dev": true, "license": "MIT/X11", + "optional": true, + "peer": true, "dependencies": { "traverse": ">=0.3.0 <0.4" } @@ -1775,6 +1856,8 @@ "version": "0.4.16", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "mitt": "3.0.0" }, @@ -1882,6 +1965,8 @@ "version": "9.5.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || >=14" } @@ -1890,6 +1975,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "crc-32": "^1.2.0", "crc32-stream": "^5.0.0", @@ -1904,6 +1991,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1945,11 +2034,12 @@ "../../../node_modules/core-util-is": { "version": "1.0.3", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/crc-32": { "version": "1.2.2", - "dev": true, "license": "Apache-2.0", "bin": { "crc32": "bin/crc32.njs" @@ -1962,6 +2052,8 @@ "version": "5.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "crc-32": "^1.2.0", "readable-stream": "^3.4.0" @@ -1974,6 +2066,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1987,6 +2081,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "node-fetch": "^2.6.12" } @@ -1995,6 +2091,8 @@ "version": "2.7.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "whatwg-url": "^5.0.0" }, @@ -2013,17 +2111,23 @@ "../../../node_modules/cross-fetch/node_modules/tr46": { "version": "0.0.3", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/cross-fetch/node_modules/webidl-conversions": { "version": "3.0.1", "dev": true, - "license": "BSD-2-Clause" + "license": "BSD-2-Clause", + "optional": true, + "peer": true }, "../../../node_modules/cross-fetch/node_modules/whatwg-url": { "version": "5.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" @@ -2077,11 +2181,15 @@ }, "../../../node_modules/css-shorthand-properties": { "version": "1.1.1", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "../../../node_modules/css-value": { "version": "0.0.1", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "../../../node_modules/cssesc": { "version": "3.0.0", @@ -2097,16 +2205,12 @@ "../../../node_modules/cssom": { "version": "0.5.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/cssstyle": { "version": "2.3.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "cssom": "~0.3.6" }, @@ -2117,9 +2221,7 @@ "../../../node_modules/cssstyle/node_modules/cssom": { "version": "0.3.8", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/dashdash": { "version": "1.14.1", @@ -2135,6 +2237,8 @@ "version": "4.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 12" } @@ -2143,8 +2247,6 @@ "version": "3.0.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "abab": "^2.0.6", "whatwg-mimetype": "^3.0.0", @@ -2158,8 +2260,6 @@ "version": "3.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "punycode": "^2.1.1" }, @@ -2171,8 +2271,6 @@ "version": "7.0.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=12" } @@ -2181,8 +2279,6 @@ "version": "3.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">=12" } @@ -2191,8 +2287,6 @@ "version": "11.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "tr46": "^3.0.0", "webidl-conversions": "^7.0.0" @@ -2226,6 +2320,8 @@ "version": "6.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -2236,14 +2332,14 @@ "../../../node_modules/decimal.js": { "version": "10.4.3", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/decompress-response": { "version": "6.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "mimic-response": "^3.1.0" }, @@ -2258,6 +2354,8 @@ "version": "3.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -2284,6 +2382,8 @@ "version": "5.1.0", "dev": true, "license": "BSD-3-Clause", + "optional": true, + "peer": true, "engines": { "node": ">=16.0.0" } @@ -2292,6 +2392,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" } @@ -2329,6 +2431,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ast-types": "^0.13.4", "escodegen": "^2.1.0", @@ -2349,6 +2453,8 @@ "version": "2.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=6" } @@ -2369,7 +2475,9 @@ "../../../node_modules/devtools-protocol": { "version": "0.0.1262051", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/diff-sequences": { "version": "29.6.3", @@ -2394,8 +2502,6 @@ "version": "2.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "webidl-conversions": "^5.0.0" }, @@ -2407,8 +2513,6 @@ "version": "5.0.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=8" } @@ -2422,6 +2526,8 @@ "version": "0.1.4", "dev": true, "license": "BSD-3-Clause", + "optional": true, + "peer": true, "dependencies": { "readable-stream": "^2.0.2" } @@ -2442,6 +2548,8 @@ "version": "3.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/which": "^2.0.1", "which": "^2.0.2" @@ -2458,6 +2566,8 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "^8.28.0", "decamelize": "^6.0.0", @@ -2474,6 +2584,8 @@ "version": "3.1.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=16" } @@ -2482,6 +2594,8 @@ "version": "4.0.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "isexe": "^3.1.1" }, @@ -2543,6 +2657,8 @@ "version": "1.4.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "once": "^1.4.0" } @@ -3128,6 +3244,8 @@ "version": "2.0.1", "dev": true, "license": "BSD-2-Clause", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.1.1", "get-stream": "^5.1.0", @@ -3147,6 +3265,8 @@ "version": "5.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "pump": "^3.0.0" }, @@ -3171,7 +3291,9 @@ "../../../node_modules/fast-fifo": { "version": "1.3.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/fast-glob": { "version": "3.3.2", @@ -3237,6 +3359,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" @@ -3345,8 +3469,6 @@ "version": "4.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -3360,6 +3482,8 @@ "version": "2.1.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 14.17" } @@ -3368,6 +3492,8 @@ "version": "4.0.10", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fetch-blob": "^3.1.2" }, @@ -3379,6 +3505,8 @@ "version": "11.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -3392,6 +3520,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -3405,6 +3535,8 @@ "version": "1.0.12", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "graceful-fs": "^4.1.2", "inherits": "~2.0.0", @@ -3419,6 +3551,8 @@ "version": "1.1.11", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -3428,6 +3562,8 @@ "version": "7.2.3", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -3447,6 +3583,8 @@ "version": "3.1.2", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -3458,6 +3596,8 @@ "version": "2.7.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "glob": "^7.1.3" }, @@ -3503,6 +3643,8 @@ "dev": true, "hasInstallScript": true, "license": "MPL-2.0", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "^8.28.0", "decamelize": "^6.0.0", @@ -3524,6 +3666,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -3535,6 +3679,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -3547,6 +3693,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -3559,6 +3707,8 @@ "version": "3.1.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=16" } @@ -3567,6 +3717,8 @@ "version": "4.0.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "isexe": "^3.1.1" }, @@ -3611,6 +3763,8 @@ "version": "7.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=16" }, @@ -3659,6 +3813,8 @@ "version": "6.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "basic-ftp": "^5.0.2", "data-uri-to-buffer": "^6.0.2", @@ -3673,6 +3829,8 @@ "version": "6.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 14" } @@ -3780,6 +3938,8 @@ "version": "12.6.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@sindresorhus/is": "^5.2.0", "@szmarczak/http-timer": "^5.0.1", @@ -3804,6 +3964,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -3819,7 +3981,9 @@ "../../../node_modules/grapheme-splitter": { "version": "1.0.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/graphemer": { "version": "1.4.0", @@ -3936,8 +4100,6 @@ "version": "2.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "whatwg-encoding": "^1.0.5" }, @@ -3953,14 +4115,14 @@ "../../../node_modules/http-cache-semantics": { "version": "4.1.1", "dev": true, - "license": "BSD-2-Clause" + "license": "BSD-2-Clause", + "optional": true, + "peer": true }, "../../../node_modules/http-proxy-agent": { "version": "4.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "@tootallnate/once": "1", "agent-base": "6", @@ -3987,6 +4149,8 @@ "version": "2.2.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "quick-lru": "^5.1.1", "resolve-alpn": "^1.2.0" @@ -3999,8 +4163,6 @@ "version": "5.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "agent-base": "6", "debug": "4" @@ -4055,7 +4217,9 @@ "url": "https://feross.org/support" } ], - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/ignore": { "version": "5.3.0", @@ -4327,6 +4491,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -4371,6 +4537,8 @@ "version": "9.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "jsbn": "1.1.0", "sprintf-js": "^1.1.3" @@ -4382,7 +4550,9 @@ "../../../node_modules/ip-address/node_modules/jsbn": { "version": "1.1.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/is-array-buffer": { "version": "3.0.2", @@ -4541,6 +4711,8 @@ "version": "4.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -4562,9 +4734,7 @@ "../../../node_modules/is-potential-custom-element-name": { "version": "1.0.1", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/is-regex": { "version": "1.1.4", @@ -4772,8 +4942,6 @@ "version": "17.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "abab": "^2.0.5", "acorn": "^8.4.1", @@ -4853,6 +5021,8 @@ "version": "6.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "universalify": "^2.0.0" }, @@ -4864,6 +5034,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -4903,6 +5075,8 @@ "version": "0.33.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -4914,6 +5088,8 @@ "version": "1.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "readable-stream": "^2.0.5" }, @@ -5008,7 +5184,9 @@ "../../../node_modules/listenercount": { "version": "1.0.1", "dev": true, - "license": "ISC" + "license": "ISC", + "optional": true, + "peer": true }, "../../../node_modules/loader-runner": { "version": "4.3.0", @@ -5039,6 +5217,8 @@ "version": "2.2.20", "dev": true, "license": "SEE LICENSE IN LICENSE", + "optional": true, + "peer": true, "dependencies": { "n12": "1.8.23", "type-fest": "2.13.0", @@ -5049,6 +5229,8 @@ "version": "2.13.0", "dev": true, "license": "(MIT OR CC0-1.0)", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" }, @@ -5081,7 +5263,9 @@ "../../../node_modules/lodash.clonedeep": { "version": "4.5.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/lodash.merge": { "version": "4.6.2", @@ -5095,12 +5279,16 @@ "../../../node_modules/lodash.zip": { "version": "4.2.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/loglevel": { "version": "1.9.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 0.6.0" }, @@ -5112,7 +5300,9 @@ "../../../node_modules/loglevel-plugin-prefix": { "version": "0.8.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/loupe": { "version": "2.3.7", @@ -5126,6 +5316,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -5197,6 +5389,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -5235,12 +5429,16 @@ "../../../node_modules/mitt": { "version": "3.0.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/mkdirp": { "version": "0.5.6", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "minimist": "^1.2.6" }, @@ -5275,7 +5473,9 @@ "../../../node_modules/n12": { "version": "1.8.23", "dev": true, - "license": "SEE LICENSE IN LICENSE" + "license": "SEE LICENSE IN LICENSE", + "optional": true, + "peer": true }, "../../../node_modules/nanoid": { "version": "3.3.7", @@ -5310,6 +5510,8 @@ "version": "2.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 0.4.0" } @@ -5335,6 +5537,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.5.0" } @@ -5343,6 +5547,8 @@ "version": "3.3.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", @@ -5380,6 +5586,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -5388,6 +5596,8 @@ "version": "8.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -5409,9 +5619,7 @@ "../../../node_modules/nwsapi": { "version": "2.2.7", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/oauth-sign": { "version": "0.9.0", @@ -5532,6 +5740,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" } @@ -5568,6 +5778,8 @@ "version": "7.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@tootallnate/quickjs-emscripten": "^0.23.0", "agent-base": "^7.0.2", @@ -5586,6 +5798,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -5597,6 +5811,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -5609,6 +5825,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -5621,6 +5839,8 @@ "version": "7.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "degenerator": "^5.0.0", "netmask": "^2.0.2" @@ -5643,9 +5863,7 @@ "../../../node_modules/parse5": { "version": "6.0.1", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/path-exists": { "version": "4.0.0", @@ -5901,12 +6119,16 @@ "../../../node_modules/process-nextick-args": { "version": "2.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/progress": { "version": "2.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.4.0" } @@ -5919,6 +6141,8 @@ "version": "6.4.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "^4.3.4", @@ -5937,6 +6161,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -5948,6 +6174,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -5960,6 +6188,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -5972,6 +6202,8 @@ "version": "7.18.3", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=12" } @@ -5979,7 +6211,9 @@ "../../../node_modules/proxy-from-env": { "version": "1.1.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/psl": { "version": "1.9.0", @@ -5989,6 +6223,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -6005,6 +6241,8 @@ "version": "20.9.0", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "@puppeteer/browsers": "1.4.6", "chromium-bidi": "0.4.16", @@ -6028,12 +6266,16 @@ "../../../node_modules/puppeteer-core/node_modules/devtools-protocol": { "version": "0.0.1147663", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/puppeteer-core/node_modules/ws": { "version": "8.13.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.0.0" }, @@ -6060,14 +6302,14 @@ "../../../node_modules/query-selector-shadow-dom": { "version": "1.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/querystringify": { "version": "2.2.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/queue-microtask": { "version": "1.2.3", @@ -6091,12 +6333,16 @@ "../../../node_modules/queue-tick": { "version": "1.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/quick-lru": { "version": "5.1.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -6123,6 +6369,8 @@ "version": "2.3.8", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -6136,17 +6384,23 @@ "../../../node_modules/readable-stream/node_modules/isarray": { "version": "1.0.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/readable-stream/node_modules/safe-buffer": { "version": "5.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/readdir-glob": { "version": "1.1.3", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "minimatch": "^5.1.0" } @@ -6155,6 +6409,8 @@ "version": "5.1.6", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -6280,9 +6536,7 @@ "../../../node_modules/requires-port": { "version": "1.0.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/resolve": { "version": "1.22.8", @@ -6303,7 +6557,9 @@ "../../../node_modules/resolve-alpn": { "version": "1.2.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/resolve-from": { "version": "4.0.0", @@ -6325,6 +6581,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "lowercase-keys": "^3.0.0" }, @@ -6339,6 +6597,8 @@ "version": "1.11.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fast-deep-equal": "^2.0.1" } @@ -6346,7 +6606,9 @@ "../../../node_modules/resq/node_modules/fast-deep-equal": { "version": "2.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/reusify": { "version": "1.0.4", @@ -6360,7 +6622,9 @@ "../../../node_modules/rgb2hex": { "version": "0.2.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/rimraf": { "version": "3.0.2", @@ -6471,7 +6735,9 @@ "../../../node_modules/safaridriver": { "version": "0.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/safe-array-concat": { "version": "1.1.0", @@ -6536,8 +6802,6 @@ "version": "5.0.1", "dev": true, "license": "ISC", - "optional": true, - "peer": true, "dependencies": { "xmlchars": "^2.2.0" }, @@ -6591,6 +6855,8 @@ "version": "11.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "type-fest": "^2.12.2" }, @@ -6642,7 +6908,9 @@ "../../../node_modules/setimmediate": { "version": "1.0.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/shallow-clone": { "version": "3.0.1", @@ -6725,6 +6993,8 @@ "version": "4.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 6.0.0", "npm": ">= 3.0.0" @@ -6734,6 +7004,8 @@ "version": "2.7.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ip-address": "^9.0.5", "smart-buffer": "^4.2.0" @@ -6747,6 +7019,8 @@ "version": "8.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "^4.3.4", @@ -6760,6 +7034,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -6798,6 +7074,8 @@ "version": "4.2.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">= 10.x" } @@ -6805,7 +7083,9 @@ "../../../node_modules/sprintf-js": { "version": "1.1.3", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/sshpk": { "version": "1.18.0", @@ -6851,6 +7131,8 @@ "version": "2.15.6", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fast-fifo": "^1.1.0", "queue-tick": "^1.0.1" @@ -6860,6 +7142,8 @@ "version": "1.1.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -6867,7 +7151,9 @@ "../../../node_modules/string_decoder/node_modules/safe-buffer": { "version": "5.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/string-width": { "version": "5.1.2", @@ -7071,6 +7357,8 @@ "version": "3.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "pump": "^3.0.0", "tar-stream": "^3.1.5" @@ -7084,6 +7372,8 @@ "version": "3.1.7", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "b4a": "^1.6.4", "fast-fifo": "^1.2.0", @@ -7159,7 +7449,9 @@ "../../../node_modules/through": { "version": "2.3.8", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/tinybench": { "version": "2.6.0", @@ -7205,8 +7497,6 @@ "version": "4.1.3", "dev": true, "license": "BSD-3-Clause", - "optional": true, - "peer": true, "dependencies": { "psl": "^1.1.33", "punycode": "^2.1.1", @@ -7221,8 +7511,6 @@ "version": "2.1.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "punycode": "^2.1.1" }, @@ -7233,7 +7521,9 @@ "../../../node_modules/traverse": { "version": "0.3.9", "dev": true, - "license": "MIT/X11" + "license": "MIT/X11", + "optional": true, + "peer": true }, "../../../node_modules/tsconfig-paths": { "version": "3.15.0", @@ -7260,7 +7550,9 @@ "../../../node_modules/tslib": { "version": "2.6.2", "dev": true, - "license": "0BSD" + "license": "0BSD", + "optional": true, + "peer": true }, "../../../node_modules/tsx": { "version": "4.7.1", @@ -7369,6 +7661,8 @@ "version": "2.19.0", "dev": true, "license": "(MIT OR CC0-1.0)", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" }, @@ -7472,6 +7766,8 @@ "version": "1.4.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "buffer": "^5.2.1", "through": "^2.3.8" @@ -7495,6 +7791,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -7509,8 +7807,6 @@ "version": "0.2.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">= 4.0.0" } @@ -7519,6 +7815,8 @@ "version": "0.10.14", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "big-integer": "^1.6.17", "binary": "~0.3.0", @@ -7574,8 +7872,6 @@ "version": "1.5.10", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" @@ -7584,6 +7880,8 @@ "../../../node_modules/userhome": { "version": "1.0.0", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 0.8.0" } @@ -7826,8 +8124,6 @@ "version": "2.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "xml-name-validator": "^3.0.0" }, @@ -7839,6 +8135,8 @@ "version": "1.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "chalk": "^4.1.2", "commander": "^9.3.0", @@ -7869,6 +8167,8 @@ "version": "3.3.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 8" } @@ -7877,6 +8177,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0", "@types/ws": "^8.5.3", @@ -7898,6 +8200,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0", "@wdio/config": "8.32.3", @@ -7940,8 +8244,6 @@ "version": "6.1.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=10.4" } @@ -8136,8 +8438,6 @@ "version": "9.1.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "tr46": "^2.1.0", "webidl-conversions": "^6.1.0" @@ -8331,9 +8631,7 @@ "../../../node_modules/xmlchars": { "version": "2.2.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/xmldom": { "version": "0.1.31", @@ -8421,6 +8719,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "archiver-utils": "^4.0.1", "compress-commons": "^5.0.1", @@ -8434,6 +8734,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", diff --git a/examples/webpack/webpack-project-built/package-lock.json b/examples/webpack/webpack-project-built/package-lock.json index fa88b0bb71..1f2d085634 100644 --- a/examples/webpack/webpack-project-built/package-lock.json +++ b/examples/webpack/webpack-project-built/package-lock.json @@ -26,7 +26,8 @@ "license": "Apache-2.0", "dependencies": { "codemirror": "^5.61.1", - "core-js": "^3.39.0", + "core-js": "^3.40.0", + "crc-32": "^1.2.2", "gl-matrix": "3.1.0", "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", @@ -34,7 +35,7 @@ "numcodecs": "^0.3.2" }, "engines": { - "node": ">=20.11 <21 || >=21.2" + "node": ">=22" } }, "node_modules/@discoveryjs/json-ext": { diff --git a/examples/webpack/webpack-project-source/package-lock.json b/examples/webpack/webpack-project-source/package-lock.json index 39f771cc94..1a2e108e58 100644 --- a/examples/webpack/webpack-project-source/package-lock.json +++ b/examples/webpack/webpack-project-source/package-lock.json @@ -26,7 +26,8 @@ "license": "Apache-2.0", "dependencies": { "codemirror": "^5.61.1", - "core-js": "^3.39.0", + "core-js": "^3.40.0", + "crc-32": "^1.2.2", "gl-matrix": "3.1.0", "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", @@ -34,38 +35,51 @@ "numcodecs": "^0.3.2" }, "devDependencies": { - "@eslint/js": "^9.16.0", - "@rspack/cli": "^1.1.6", - "@rspack/core": "^1.1.6", + "@eslint/js": "^9.18.0", + "@iodigital/vite-plugin-msw": "^2.0.0", + "@playwright/browser-chromium": "^1.49.1", + "@rspack/cli": "^1.1.8", + "@rspack/core": "^1.1.8", "@types/codemirror": "5.60.15", "@types/gl-matrix": "^2.4.5", + "@types/http-server": "^0.12.4", + "@types/jsdom": "^21.1.7", "@types/lodash-es": "^4.17.12", - "@types/node": "^22.10.2", + "@types/node": "^22.10.7", "@types/nunjucks": "^3.2.6", + "@types/s3rver": "^3.7.4", "@types/yargs": "^17.0.33", - "@vitest/browser": "^2.1.8", - "@vitest/ui": "^2.1.8", + "@vitest/browser": "^3.0.2", + "@vitest/ui": "^3.0.2", + "@vitest/web-worker": "^3.0.2", + "cookie": "^1.0.2", "css-loader": "^7.1.2", - "esbuild": "^0.24.0", - "eslint": "^9.16.0", + "esbuild": "^0.24.2", + "eslint": "^9.18.0", "eslint-import-resolver-typescript": "^3.7.0", "eslint-plugin-import": "^2.31.0", "eslint-rspack-plugin": "^4.2.1", - "glob": "^11.0.0", + "express": "^4.21.2", + "glob": "^11.0.1", + "http-server": "^14.1.1", + "jsdom": "^26.0.0", + "msw": "^2.7.0", "nunjucks": "^3.2.4", + "playwright": "^1.49.1", "prettier": "3.4.2", - "ts-checker-rspack-plugin": "^1.0.3", + "s3rver": "^3.7.1", + "ts-checker-rspack-plugin": "^1.1.1", "tsx": "^4.19.2", - "typescript": "^5.7.2", - "typescript-eslint": "^8.18.0", - "vitest": "^2.1.8", - "webdriverio": "^9.4.1", + "typescript": "^5.7.3", + "typescript-eslint": "^8.20.0", + "vitest": "^3.0.2", "webpack-bundle-analyzer": "^4.10.2", "webpack-merge": "^6.0.1", - "yargs": "^17.7.2" + "yargs": "^17.7.2", + "yauzl": "^3.2.0" }, "engines": { - "node": ">=20.11 <21 || >=21.2" + "node": ">=22" } }, "../../../node_modules/@aashutoshrathi/word-wrap": { @@ -737,6 +751,8 @@ "version": "2.0.1", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "debug": "4.3.4", "extract-zip": "2.0.1", @@ -970,6 +986,8 @@ "version": "5.6.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -981,6 +999,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "defer-to-connect": "^2.0.1" }, @@ -992,8 +1012,6 @@ "version": "1.1.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">= 6" } @@ -1001,7 +1019,9 @@ "../../../node_modules/@tootallnate/quickjs-emscripten": { "version": "0.23.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/codemirror": { "version": "5.60.15", @@ -1046,7 +1066,9 @@ "../../../node_modules/@types/http-cache-semantics": { "version": "4.0.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/json-schema": { "version": "7.0.15", @@ -1092,12 +1114,16 @@ "../../../node_modules/@types/which": { "version": "2.0.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@types/ws": { "version": "8.5.10", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "*" } @@ -1120,6 +1146,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "@types/node": "*" } @@ -1279,6 +1306,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "8.28.0", "@wdio/types": "8.32.2", @@ -1296,6 +1325,8 @@ "version": "8.28.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "chalk": "^5.1.2", "loglevel": "^1.6.0", @@ -1310,6 +1341,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -1321,6 +1354,8 @@ "version": "5.3.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.17.0 || ^14.13 || >=16.0.0" }, @@ -1332,6 +1367,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ansi-regex": "^6.0.1" }, @@ -1345,12 +1382,16 @@ "../../../node_modules/@wdio/protocols": { "version": "8.32.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/@wdio/repl": { "version": "8.24.12", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0" }, @@ -1362,6 +1403,8 @@ "version": "8.32.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0" }, @@ -1373,6 +1416,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@puppeteer/browsers": "^1.6.0", "@wdio/logger": "8.28.0", @@ -1570,9 +1615,7 @@ "../../../node_modules/abab": { "version": "2.0.6", "dev": true, - "license": "BSD-3-Clause", - "optional": true, - "peer": true + "license": "BSD-3-Clause" }, "../../../node_modules/abbrev": { "version": "2.0.0", @@ -1596,8 +1639,6 @@ "version": "6.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "acorn": "^7.1.1", "acorn-walk": "^7.1.1" @@ -1607,8 +1648,6 @@ "version": "7.4.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1638,8 +1677,6 @@ "version": "7.2.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">=0.4.0" } @@ -1648,8 +1685,6 @@ "version": "6.0.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "debug": "4" }, @@ -1705,6 +1740,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "archiver-utils": "^4.0.1", "async": "^3.2.4", @@ -1722,6 +1759,8 @@ "version": "4.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "glob": "^8.0.0", "graceful-fs": "^4.2.0", @@ -1738,6 +1777,8 @@ "version": "8.1.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1756,6 +1797,8 @@ "version": "5.1.6", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -1767,6 +1810,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1780,6 +1825,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1798,6 +1845,8 @@ "version": "5.3.0", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "dequal": "^2.0.3" } @@ -1937,6 +1986,8 @@ "version": "0.13.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tslib": "^2.0.1" }, @@ -1947,7 +1998,9 @@ "../../../node_modules/async": { "version": "3.2.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/async-limiter": { "version": "1.0.1", @@ -1982,7 +2035,9 @@ "../../../node_modules/b4a": { "version": "1.6.4", "dev": true, - "license": "ISC" + "license": "ISC", + "optional": true, + "peer": true }, "../../../node_modules/balanced-match": { "version": "1.0.2", @@ -1992,13 +2047,15 @@ "version": "2.2.0", "dev": true, "license": "Apache-2.0", - "optional": true + "optional": true, + "peer": true }, "../../../node_modules/bare-fs": { "version": "2.1.5", "dev": true, "license": "Apache-2.0", "optional": true, + "peer": true, "dependencies": { "bare-events": "^2.0.0", "bare-os": "^2.0.0", @@ -2010,13 +2067,15 @@ "version": "2.2.0", "dev": true, "license": "Apache-2.0", - "optional": true + "optional": true, + "peer": true }, "../../../node_modules/bare-path": { "version": "2.1.0", "dev": true, "license": "Apache-2.0", "optional": true, + "peer": true, "dependencies": { "bare-os": "^2.1.0" } @@ -2038,12 +2097,16 @@ "url": "https://feross.org/support" } ], - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/basic-ftp": { "version": "5.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.0.0" } @@ -2059,6 +2122,8 @@ "version": "1.6.52", "dev": true, "license": "Unlicense", + "optional": true, + "peer": true, "engines": { "node": ">=0.6" } @@ -2067,6 +2132,8 @@ "version": "0.3.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "buffers": "~0.1.1", "chainsaw": "~0.1.0" @@ -2075,7 +2142,9 @@ "../../../node_modules/bluebird": { "version": "3.4.7", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/brace-expansion": { "version": "2.0.1", @@ -2151,6 +2220,8 @@ "version": "1.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.10" } @@ -2158,6 +2229,8 @@ "../../../node_modules/buffers": { "version": "0.1.1", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.2.0" } @@ -2174,6 +2247,8 @@ "version": "7.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" } @@ -2182,6 +2257,8 @@ "version": "10.2.14", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/http-cache-semantics": "^4.0.2", "get-stream": "^6.0.1", @@ -2199,6 +2276,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -2273,6 +2352,8 @@ "version": "0.1.0", "dev": true, "license": "MIT/X11", + "optional": true, + "peer": true, "dependencies": { "traverse": ">=0.3.0 <0.4" } @@ -2317,6 +2398,8 @@ "version": "0.4.16", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "mitt": "3.0.0" }, @@ -2424,6 +2507,8 @@ "version": "9.5.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || >=14" } @@ -2432,6 +2517,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "crc-32": "^1.2.0", "crc32-stream": "^5.0.0", @@ -2446,6 +2533,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -2487,11 +2576,12 @@ "../../../node_modules/core-util-is": { "version": "1.0.3", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/crc-32": { "version": "1.2.2", - "dev": true, "license": "Apache-2.0", "bin": { "crc32": "bin/crc32.njs" @@ -2504,6 +2594,8 @@ "version": "5.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "crc-32": "^1.2.0", "readable-stream": "^3.4.0" @@ -2516,6 +2608,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -2529,6 +2623,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "node-fetch": "^2.6.12" } @@ -2537,6 +2633,8 @@ "version": "2.7.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "whatwg-url": "^5.0.0" }, @@ -2555,17 +2653,23 @@ "../../../node_modules/cross-fetch/node_modules/tr46": { "version": "0.0.3", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/cross-fetch/node_modules/webidl-conversions": { "version": "3.0.1", "dev": true, - "license": "BSD-2-Clause" + "license": "BSD-2-Clause", + "optional": true, + "peer": true }, "../../../node_modules/cross-fetch/node_modules/whatwg-url": { "version": "5.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" @@ -2619,11 +2723,15 @@ }, "../../../node_modules/css-shorthand-properties": { "version": "1.1.1", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "../../../node_modules/css-value": { "version": "0.0.1", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "../../../node_modules/cssesc": { "version": "3.0.0", @@ -2639,16 +2747,12 @@ "../../../node_modules/cssom": { "version": "0.5.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/cssstyle": { "version": "2.3.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "cssom": "~0.3.6" }, @@ -2659,9 +2763,7 @@ "../../../node_modules/cssstyle/node_modules/cssom": { "version": "0.3.8", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/dashdash": { "version": "1.14.1", @@ -2677,6 +2779,8 @@ "version": "4.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 12" } @@ -2685,8 +2789,6 @@ "version": "3.0.2", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "abab": "^2.0.6", "whatwg-mimetype": "^3.0.0", @@ -2700,8 +2802,6 @@ "version": "3.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "punycode": "^2.1.1" }, @@ -2713,8 +2813,6 @@ "version": "7.0.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=12" } @@ -2723,8 +2821,6 @@ "version": "3.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">=12" } @@ -2733,8 +2829,6 @@ "version": "11.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "tr46": "^3.0.0", "webidl-conversions": "^7.0.0" @@ -2768,6 +2862,8 @@ "version": "6.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -2778,14 +2874,14 @@ "../../../node_modules/decimal.js": { "version": "10.4.3", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/decompress-response": { "version": "6.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "mimic-response": "^3.1.0" }, @@ -2800,6 +2896,8 @@ "version": "3.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -2826,6 +2924,8 @@ "version": "5.1.0", "dev": true, "license": "BSD-3-Clause", + "optional": true, + "peer": true, "engines": { "node": ">=16.0.0" } @@ -2834,6 +2934,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" } @@ -2871,6 +2973,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ast-types": "^0.13.4", "escodegen": "^2.1.0", @@ -2891,6 +2995,8 @@ "version": "2.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=6" } @@ -2911,7 +3017,9 @@ "../../../node_modules/devtools-protocol": { "version": "0.0.1262051", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/diff-sequences": { "version": "29.6.3", @@ -2936,8 +3044,6 @@ "version": "2.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "webidl-conversions": "^5.0.0" }, @@ -2949,8 +3055,6 @@ "version": "5.0.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=8" } @@ -2964,6 +3068,8 @@ "version": "0.1.4", "dev": true, "license": "BSD-3-Clause", + "optional": true, + "peer": true, "dependencies": { "readable-stream": "^2.0.2" } @@ -2984,6 +3090,8 @@ "version": "3.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/which": "^2.0.1", "which": "^2.0.2" @@ -3000,6 +3108,8 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "^8.28.0", "decamelize": "^6.0.0", @@ -3016,6 +3126,8 @@ "version": "3.1.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=16" } @@ -3024,6 +3136,8 @@ "version": "4.0.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "isexe": "^3.1.1" }, @@ -3085,6 +3199,8 @@ "version": "1.4.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "once": "^1.4.0" } @@ -4022,6 +4138,8 @@ "version": "2.0.1", "dev": true, "license": "BSD-2-Clause", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.1.1", "get-stream": "^5.1.0", @@ -4041,6 +4159,8 @@ "version": "5.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "pump": "^3.0.0" }, @@ -4065,7 +4185,9 @@ "../../../node_modules/fast-fifo": { "version": "1.3.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/fast-glob": { "version": "3.3.2", @@ -4131,6 +4253,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" @@ -4239,8 +4363,6 @@ "version": "4.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -4254,6 +4376,8 @@ "version": "2.1.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 14.17" } @@ -4262,6 +4386,8 @@ "version": "4.0.10", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fetch-blob": "^3.1.2" }, @@ -4273,6 +4399,8 @@ "version": "11.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -4286,6 +4414,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -4313,6 +4443,8 @@ "version": "1.0.12", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "graceful-fs": "^4.1.2", "inherits": "~2.0.0", @@ -4327,6 +4459,8 @@ "version": "1.1.11", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -4336,6 +4470,8 @@ "version": "7.2.3", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -4355,6 +4491,8 @@ "version": "3.1.2", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -4366,6 +4504,8 @@ "version": "2.7.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "glob": "^7.1.3" }, @@ -4411,6 +4551,8 @@ "dev": true, "hasInstallScript": true, "license": "MPL-2.0", + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "^8.28.0", "decamelize": "^6.0.0", @@ -4432,6 +4574,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -4443,6 +4587,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -4455,6 +4601,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -4467,6 +4615,8 @@ "version": "3.1.1", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=16" } @@ -4475,6 +4625,8 @@ "version": "4.0.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "isexe": "^3.1.1" }, @@ -4519,6 +4671,8 @@ "version": "7.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=16" }, @@ -4567,6 +4721,8 @@ "version": "6.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "basic-ftp": "^5.0.2", "data-uri-to-buffer": "^6.0.2", @@ -4581,6 +4737,8 @@ "version": "6.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 14" } @@ -4688,6 +4846,8 @@ "version": "12.6.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@sindresorhus/is": "^5.2.0", "@szmarczak/http-timer": "^5.0.1", @@ -4712,6 +4872,8 @@ "version": "6.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -4727,7 +4889,9 @@ "../../../node_modules/grapheme-splitter": { "version": "1.0.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/graphemer": { "version": "1.4.0", @@ -4844,8 +5008,6 @@ "version": "2.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "whatwg-encoding": "^1.0.5" }, @@ -4861,14 +5023,14 @@ "../../../node_modules/http-cache-semantics": { "version": "4.1.1", "dev": true, - "license": "BSD-2-Clause" + "license": "BSD-2-Clause", + "optional": true, + "peer": true }, "../../../node_modules/http-proxy-agent": { "version": "4.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "@tootallnate/once": "1", "agent-base": "6", @@ -4895,6 +5057,8 @@ "version": "2.2.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "quick-lru": "^5.1.1", "resolve-alpn": "^1.2.0" @@ -4907,8 +5071,6 @@ "version": "5.0.1", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "agent-base": "6", "debug": "4" @@ -4963,7 +5125,9 @@ "url": "https://feross.org/support" } ], - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/ignore": { "version": "5.3.0", @@ -5235,6 +5399,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -5279,6 +5445,8 @@ "version": "9.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "jsbn": "1.1.0", "sprintf-js": "^1.1.3" @@ -5290,7 +5458,9 @@ "../../../node_modules/ip-address/node_modules/jsbn": { "version": "1.1.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/is-array-buffer": { "version": "3.0.2", @@ -5449,6 +5619,8 @@ "version": "4.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -5470,9 +5642,7 @@ "../../../node_modules/is-potential-custom-element-name": { "version": "1.0.1", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/is-regex": { "version": "1.1.4", @@ -5680,8 +5850,6 @@ "version": "17.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "abab": "^2.0.5", "acorn": "^8.4.1", @@ -5761,6 +5929,8 @@ "version": "6.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "universalify": "^2.0.0" }, @@ -5772,6 +5942,8 @@ "version": "2.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -5811,6 +5983,8 @@ "version": "0.33.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -5822,6 +5996,8 @@ "version": "1.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "readable-stream": "^2.0.5" }, @@ -5916,7 +6092,9 @@ "../../../node_modules/listenercount": { "version": "1.0.1", "dev": true, - "license": "ISC" + "license": "ISC", + "optional": true, + "peer": true }, "../../../node_modules/loader-runner": { "version": "4.3.0", @@ -5947,6 +6125,8 @@ "version": "2.2.20", "dev": true, "license": "SEE LICENSE IN LICENSE", + "optional": true, + "peer": true, "dependencies": { "n12": "1.8.23", "type-fest": "2.13.0", @@ -5957,6 +6137,8 @@ "version": "2.13.0", "dev": true, "license": "(MIT OR CC0-1.0)", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" }, @@ -5989,7 +6171,9 @@ "../../../node_modules/lodash.clonedeep": { "version": "4.5.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/lodash.merge": { "version": "4.6.2", @@ -6003,12 +6187,16 @@ "../../../node_modules/lodash.zip": { "version": "4.2.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/loglevel": { "version": "1.9.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 0.6.0" }, @@ -6020,7 +6208,9 @@ "../../../node_modules/loglevel-plugin-prefix": { "version": "0.8.4", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/loupe": { "version": "2.3.7", @@ -6034,6 +6224,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -6105,6 +6297,8 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -6143,12 +6337,16 @@ "../../../node_modules/mitt": { "version": "3.0.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/mkdirp": { "version": "0.5.6", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "minimist": "^1.2.6" }, @@ -6183,7 +6381,9 @@ "../../../node_modules/n12": { "version": "1.8.23", "dev": true, - "license": "SEE LICENSE IN LICENSE" + "license": "SEE LICENSE IN LICENSE", + "optional": true, + "peer": true }, "../../../node_modules/nanoid": { "version": "3.3.7", @@ -6218,6 +6418,8 @@ "version": "2.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 0.4.0" } @@ -6243,6 +6445,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.5.0" } @@ -6251,6 +6455,8 @@ "version": "3.3.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", @@ -6288,6 +6494,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -6296,6 +6504,8 @@ "version": "8.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.16" }, @@ -6317,9 +6527,7 @@ "../../../node_modules/nwsapi": { "version": "2.2.7", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/oauth-sign": { "version": "0.9.0", @@ -6440,6 +6648,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" } @@ -6476,6 +6686,8 @@ "version": "7.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@tootallnate/quickjs-emscripten": "^0.23.0", "agent-base": "^7.0.2", @@ -6494,6 +6706,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -6505,6 +6719,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -6517,6 +6733,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -6529,6 +6747,8 @@ "version": "7.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "degenerator": "^5.0.0", "netmask": "^2.0.2" @@ -6551,9 +6771,7 @@ "../../../node_modules/parse5": { "version": "6.0.1", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/path-exists": { "version": "4.0.0", @@ -6811,12 +7029,16 @@ "../../../node_modules/process-nextick-args": { "version": "2.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/progress": { "version": "2.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.4.0" } @@ -6829,6 +7051,8 @@ "version": "6.4.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "^4.3.4", @@ -6847,6 +7071,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -6858,6 +7084,8 @@ "version": "7.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -6870,6 +7098,8 @@ "version": "7.0.4", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -6882,6 +7112,8 @@ "version": "7.18.3", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">=12" } @@ -6889,7 +7121,9 @@ "../../../node_modules/proxy-from-env": { "version": "1.1.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/psl": { "version": "1.9.0", @@ -6899,6 +7133,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -6915,6 +7151,8 @@ "version": "20.9.0", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "@puppeteer/browsers": "1.4.6", "chromium-bidi": "0.4.16", @@ -6938,12 +7176,16 @@ "../../../node_modules/puppeteer-core/node_modules/devtools-protocol": { "version": "0.0.1147663", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/puppeteer-core/node_modules/ws": { "version": "8.13.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10.0.0" }, @@ -6970,14 +7212,14 @@ "../../../node_modules/query-selector-shadow-dom": { "version": "1.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/querystringify": { "version": "2.2.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/queue-microtask": { "version": "1.2.3", @@ -7001,12 +7243,16 @@ "../../../node_modules/queue-tick": { "version": "1.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/quick-lru": { "version": "5.1.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -7033,6 +7279,8 @@ "version": "2.3.8", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -7046,17 +7294,23 @@ "../../../node_modules/readable-stream/node_modules/isarray": { "version": "1.0.0", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/readable-stream/node_modules/safe-buffer": { "version": "5.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/readdir-glob": { "version": "1.1.3", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "dependencies": { "minimatch": "^5.1.0" } @@ -7065,6 +7319,8 @@ "version": "5.1.6", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -7190,9 +7446,7 @@ "../../../node_modules/requires-port": { "version": "1.0.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/resolve": { "version": "1.22.8", @@ -7213,7 +7467,9 @@ "../../../node_modules/resolve-alpn": { "version": "1.2.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/resolve-from": { "version": "4.0.0", @@ -7235,6 +7491,8 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "lowercase-keys": "^3.0.0" }, @@ -7249,6 +7507,8 @@ "version": "1.11.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fast-deep-equal": "^2.0.1" } @@ -7256,7 +7516,9 @@ "../../../node_modules/resq/node_modules/fast-deep-equal": { "version": "2.0.1", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/reusify": { "version": "1.0.4", @@ -7270,7 +7532,9 @@ "../../../node_modules/rgb2hex": { "version": "0.2.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/rimraf": { "version": "3.0.2", @@ -7385,7 +7649,9 @@ "../../../node_modules/safaridriver": { "version": "0.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/safe-array-concat": { "version": "1.1.0", @@ -7450,8 +7716,6 @@ "version": "5.0.1", "dev": true, "license": "ISC", - "optional": true, - "peer": true, "dependencies": { "xmlchars": "^2.2.0" }, @@ -7505,6 +7769,8 @@ "version": "11.0.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "type-fest": "^2.12.2" }, @@ -7556,7 +7822,9 @@ "../../../node_modules/setimmediate": { "version": "1.0.5", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/shallow-clone": { "version": "3.0.1", @@ -7639,6 +7907,8 @@ "version": "4.2.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 6.0.0", "npm": ">= 3.0.0" @@ -7648,6 +7918,8 @@ "version": "2.7.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "ip-address": "^9.0.5", "smart-buffer": "^4.2.0" @@ -7661,6 +7933,8 @@ "version": "8.0.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "^4.3.4", @@ -7674,6 +7948,8 @@ "version": "7.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -7713,6 +7989,8 @@ "version": "4.2.0", "dev": true, "license": "ISC", + "optional": true, + "peer": true, "engines": { "node": ">= 10.x" } @@ -7720,7 +7998,9 @@ "../../../node_modules/sprintf-js": { "version": "1.1.3", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true, + "peer": true }, "../../../node_modules/sshpk": { "version": "1.18.0", @@ -7766,6 +8046,8 @@ "version": "2.15.6", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "fast-fifo": "^1.1.0", "queue-tick": "^1.0.1" @@ -7775,6 +8057,8 @@ "version": "1.1.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -7782,7 +8066,9 @@ "../../../node_modules/string_decoder/node_modules/safe-buffer": { "version": "5.1.2", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/string-width": { "version": "5.1.2", @@ -7986,6 +8272,8 @@ "version": "3.0.5", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "pump": "^3.0.0", "tar-stream": "^3.1.5" @@ -7999,6 +8287,8 @@ "version": "3.1.7", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "b4a": "^1.6.4", "fast-fifo": "^1.2.0", @@ -8074,7 +8364,9 @@ "../../../node_modules/through": { "version": "2.3.8", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "../../../node_modules/tinybench": { "version": "2.6.0", @@ -8120,8 +8412,6 @@ "version": "4.1.3", "dev": true, "license": "BSD-3-Clause", - "optional": true, - "peer": true, "dependencies": { "psl": "^1.1.33", "punycode": "^2.1.1", @@ -8136,8 +8426,6 @@ "version": "2.1.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "punycode": "^2.1.1" }, @@ -8148,7 +8436,9 @@ "../../../node_modules/traverse": { "version": "0.3.9", "dev": true, - "license": "MIT/X11" + "license": "MIT/X11", + "optional": true, + "peer": true }, "../../../node_modules/tsconfig-paths": { "version": "3.15.0", @@ -8175,7 +8465,9 @@ "../../../node_modules/tslib": { "version": "2.6.2", "dev": true, - "license": "0BSD" + "license": "0BSD", + "optional": true, + "peer": true }, "../../../node_modules/tsx": { "version": "4.7.1", @@ -8636,6 +8928,8 @@ "version": "2.19.0", "dev": true, "license": "(MIT OR CC0-1.0)", + "optional": true, + "peer": true, "engines": { "node": ">=12.20" }, @@ -8739,6 +9033,8 @@ "version": "1.4.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "buffer": "^5.2.1", "through": "^2.3.8" @@ -8762,6 +9058,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -8776,8 +9074,6 @@ "version": "0.2.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "engines": { "node": ">= 4.0.0" } @@ -8786,6 +9082,8 @@ "version": "0.10.14", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "big-integer": "^1.6.17", "binary": "~0.3.0", @@ -8841,8 +9139,6 @@ "version": "1.5.10", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" @@ -8851,6 +9147,8 @@ "../../../node_modules/userhome": { "version": "1.0.0", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 0.8.0" } @@ -9096,8 +9394,6 @@ "version": "2.0.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "xml-name-validator": "^3.0.0" }, @@ -9109,6 +9405,8 @@ "version": "1.1.0", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "chalk": "^4.1.2", "commander": "^9.3.0", @@ -9139,6 +9437,8 @@ "version": "3.3.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 8" } @@ -9147,6 +9447,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0", "@types/ws": "^8.5.3", @@ -9168,6 +9470,8 @@ "version": "8.32.3", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0", "@wdio/config": "8.32.3", @@ -9210,8 +9514,6 @@ "version": "6.1.0", "dev": true, "license": "BSD-2-Clause", - "optional": true, - "peer": true, "engines": { "node": ">=10.4" } @@ -9406,8 +9708,6 @@ "version": "9.1.0", "dev": true, "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "tr46": "^2.1.0", "webidl-conversions": "^6.1.0" @@ -9601,9 +9901,7 @@ "../../../node_modules/xmlchars": { "version": "2.2.0", "dev": true, - "license": "MIT", - "optional": true, - "peer": true + "license": "MIT" }, "../../../node_modules/xmldom": { "version": "0.1.31", @@ -9691,6 +9989,8 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "archiver-utils": "^4.0.1", "compress-commons": "^5.0.1", @@ -9704,6 +10004,8 @@ "version": "3.6.2", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", diff --git a/package-lock.json b/package-lock.json index 1ee22589e7..95cce58459 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,8 @@ "license": "Apache-2.0", "dependencies": { "codemirror": "^5.61.1", - "core-js": "^3.39.0", + "core-js": "^3.40.0", + "crc-32": "^1.2.2", "gl-matrix": "3.1.0", "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", @@ -18,38 +19,51 @@ "numcodecs": "^0.3.2" }, "devDependencies": { - "@eslint/js": "^9.16.0", - "@rspack/cli": "^1.1.6", - "@rspack/core": "^1.1.6", + "@eslint/js": "^9.18.0", + "@iodigital/vite-plugin-msw": "^2.0.0", + "@playwright/browser-chromium": "^1.49.1", + "@rspack/cli": "^1.1.8", + "@rspack/core": "^1.1.8", "@types/codemirror": "5.60.15", "@types/gl-matrix": "^2.4.5", + "@types/http-server": "^0.12.4", + "@types/jsdom": "^21.1.7", "@types/lodash-es": "^4.17.12", - "@types/node": "^22.10.2", + "@types/node": "^22.10.7", "@types/nunjucks": "^3.2.6", + "@types/s3rver": "^3.7.4", "@types/yargs": "^17.0.33", - "@vitest/browser": "^2.1.8", - "@vitest/ui": "^2.1.8", + "@vitest/browser": "^3.0.2", + "@vitest/ui": "^3.0.2", + "@vitest/web-worker": "^3.0.2", + "cookie": "^1.0.2", "css-loader": "^7.1.2", - "esbuild": "^0.24.0", - "eslint": "^9.16.0", + "esbuild": "^0.24.2", + "eslint": "^9.18.0", "eslint-import-resolver-typescript": "^3.7.0", "eslint-plugin-import": "^2.31.0", "eslint-rspack-plugin": "^4.2.1", - "glob": "^11.0.0", + "express": "^4.21.2", + "glob": "^11.0.1", + "http-server": "^14.1.1", + "jsdom": "^26.0.0", + "msw": "^2.7.0", "nunjucks": "^3.2.4", + "playwright": "^1.49.1", "prettier": "3.4.2", - "ts-checker-rspack-plugin": "^1.0.3", + "s3rver": "^3.7.1", + "ts-checker-rspack-plugin": "^1.1.1", "tsx": "^4.19.2", - "typescript": "^5.7.2", - "typescript-eslint": "^8.18.0", - "vitest": "^2.1.8", - "webdriverio": "^9.4.1", + "typescript": "^5.7.3", + "typescript-eslint": "^8.20.0", + "vitest": "^3.0.2", "webpack-bundle-analyzer": "^4.10.2", "webpack-merge": "^6.0.1", - "yargs": "^17.7.2" + "yargs": "^17.7.2", + "yauzl": "^3.2.0" }, "engines": { - "node": ">=20.11 <21 || >=21.2" + "node": ">=22" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -61,6 +75,27 @@ "node": ">=0.10.0" } }, + "node_modules/@asamuzakjp/css-color": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-2.8.3.tgz", + "integrity": "sha512-GIc76d9UI1hCvOATjZPyHFmE5qhRccp3/zGfMPapK3jBi+yocEzp6BBB0UnfRYP9NP4FANqUZYb0hnfs3TM3hw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@csstools/css-calc": "^2.1.1", + "@csstools/css-color-parser": "^3.0.7", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "lru-cache": "^10.4.3" + } + }, + "node_modules/@asamuzakjp/css-color/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, "node_modules/@babel/code-frame": { "version": "7.26.2", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", @@ -85,10 +120,11 @@ } }, "node_modules/@babel/runtime": { - "version": "7.24.8", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.24.8.tgz", - "integrity": "sha512-5F7SDGs1T72ZczbRwbGO9lQi0NLjQxzl6i4lJxLxfW9U5UluCSyEJeniWvnhl3/euNiqQVbo8zruhsDfid0esA==", + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.0.tgz", + "integrity": "sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==", "dev": true, + "license": "MIT", "dependencies": { "regenerator-runtime": "^0.14.0" }, @@ -105,6 +141,16 @@ "cookie": "^0.7.2" } }, + "node_modules/@bundled-es-modules/cookie/node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/@bundled-es-modules/statuses": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@bundled-es-modules/statuses/-/statuses-1.0.1.tgz", @@ -124,6 +170,141 @@ "tough-cookie": "^4.1.4" } }, + "node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "dev": true, + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@csstools/color-helpers": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.0.1.tgz", + "integrity": "sha512-MKtmkA0BX87PKaO1NFRTFH+UnkgnmySQOvNxJubsadusqPEC2aJ9MOQiMceZJJ6oitUl/i0L6u0M1IrmAOmgBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@csstools/css-calc": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.1.tgz", + "integrity": "sha512-rL7kaUnTkL9K+Cvo2pnCieqNpTKgQzy5f+N+5Iuko9HAoasP+xgprVh7KN/MaJVvVL1l0EzQq2MoqBHKSrDrag==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.0.7.tgz", + "integrity": "sha512-nkMp2mTICw32uE5NN+EsJ4f5N+IGFeCFu4bGpiKgb2Pq/7J/MpyLBeQ5ry4KKtRFZaYs6sTmcMYrSRIyj5DFKA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/color-helpers": "^5.0.1", + "@csstools/css-calc": "^2.1.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.4.tgz", + "integrity": "sha512-Up7rBoV77rv29d3uKHUIVubz1BTcgyUK72IvCQAbfbMv584xHcGKCKbWh7i8hPrRJ7qU4Y8IO3IY9m+iTB7P3A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^3.0.3" + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.3.tgz", + "integrity": "sha512-UJnjoFsmxfKUdNYdWgOB0mWUypuLvAfQPH1+pyvRJs6euowbFkFC6P13w1l8mJyi3vxYMxc9kld5jZEGRQs6bw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@dabh/diagnostics": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", + "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", + "dev": true, + "dependencies": { + "colorspace": "1.1.x", + "enabled": "2.0.x", + "kuler": "^2.0.0" + } + }, "node_modules/@discoveryjs/json-ext": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz", @@ -134,13 +315,14 @@ } }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.24.0.tgz", - "integrity": "sha512-WtKdFM7ls47zkKHFVzMz8opM7LkcsIp9amDUBIAWirg70RM71WRSjdILPsY5Uv1D42ZpUfaPILDlfactHgsRkw==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.24.2.tgz", + "integrity": "sha512-thpVCb/rhxE/BnMLQ7GReQLLN8q9qbHmI55F4489/ByVg2aQaQ6kbcLb6FHkocZzQhxc4gx0sCk0tJkKBFzDhA==", "cpu": [ "ppc64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "aix" @@ -150,13 +332,14 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.24.0.tgz", - "integrity": "sha512-arAtTPo76fJ/ICkXWetLCc9EwEHKaeya4vMrReVlEIUCAUncH7M4bhMQ+M9Vf+FFOZJdTNMXNBrWwW+OXWpSew==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.24.2.tgz", + "integrity": "sha512-tmwl4hJkCfNHwFB3nBa8z1Uy3ypZpxqxfTQOcHX+xRByyYgunVbZ9MzUUfb0RxaHIMnbHagwAxuTL+tnNM+1/Q==", "cpu": [ "arm" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "android" @@ -166,13 +349,14 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.24.0.tgz", - "integrity": "sha512-Vsm497xFM7tTIPYK9bNTYJyF/lsP590Qc1WxJdlB6ljCbdZKU9SY8i7+Iin4kyhV/KV5J2rOKsBQbB77Ab7L/w==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.24.2.tgz", + "integrity": "sha512-cNLgeqCqV8WxfcTIOeL4OAtSmL8JjcN6m09XIgro1Wi7cF4t/THaWEa7eL5CMoMBdjoHOTh/vwTO/o2TRXIyzg==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "android" @@ -182,13 +366,14 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.24.0.tgz", - "integrity": "sha512-t8GrvnFkiIY7pa7mMgJd7p8p8qqYIz1NYiAoKc75Zyv73L3DZW++oYMSHPRarcotTKuSs6m3hTOa5CKHaS02TQ==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.24.2.tgz", + "integrity": "sha512-B6Q0YQDqMx9D7rvIcsXfmJfvUYLoP722bgfBlO5cGvNVb5V/+Y7nhBE3mHV9OpxBf4eAS2S68KZztiPaWq4XYw==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "android" @@ -198,13 +383,14 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.24.0.tgz", - "integrity": "sha512-CKyDpRbK1hXwv79soeTJNHb5EiG6ct3efd/FTPdzOWdbZZfGhpbcqIpiD0+vwmpu0wTIL97ZRPZu8vUt46nBSw==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.24.2.tgz", + "integrity": "sha512-kj3AnYWc+CekmZnS5IPu9D+HWtUI49hbnyqk0FLEJDbzCIQt7hg7ucF1SQAilhtYpIujfaHr6O0UHlzzSPdOeA==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "darwin" @@ -214,13 +400,14 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.24.0.tgz", - "integrity": "sha512-rgtz6flkVkh58od4PwTRqxbKH9cOjaXCMZgWD905JOzjFKW+7EiUObfd/Kav+A6Gyud6WZk9w+xu6QLytdi2OA==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.24.2.tgz", + "integrity": "sha512-WeSrmwwHaPkNR5H3yYfowhZcbriGqooyu3zI/3GGpF8AyUdsrrP0X6KumITGA9WOyiJavnGZUwPGvxvwfWPHIA==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "darwin" @@ -230,13 +417,14 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.24.0.tgz", - "integrity": "sha512-6Mtdq5nHggwfDNLAHkPlyLBpE5L6hwsuXZX8XNmHno9JuL2+bg2BX5tRkwjyfn6sKbxZTq68suOjgWqCicvPXA==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.24.2.tgz", + "integrity": "sha512-UN8HXjtJ0k/Mj6a9+5u6+2eZ2ERD7Edt1Q9IZiB5UZAIdPnVKDoG7mdTVGhHJIeEml60JteamR3qhsr1r8gXvg==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "freebsd" @@ -246,13 +434,14 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.24.0.tgz", - "integrity": "sha512-D3H+xh3/zphoX8ck4S2RxKR6gHlHDXXzOf6f/9dbFt/NRBDIE33+cVa49Kil4WUjxMGW0ZIYBYtaGCa2+OsQwQ==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.24.2.tgz", + "integrity": "sha512-TvW7wE/89PYW+IevEJXZ5sF6gJRDY/14hyIGFXdIucxCsbRmLUcjseQu1SyTko+2idmCw94TgyaEZi9HUSOe3Q==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "freebsd" @@ -262,13 +451,14 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.24.0.tgz", - "integrity": "sha512-gJKIi2IjRo5G6Glxb8d3DzYXlxdEj2NlkixPsqePSZMhLudqPhtZ4BUrpIuTjJYXxvF9njql+vRjB2oaC9XpBw==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.24.2.tgz", + "integrity": "sha512-n0WRM/gWIdU29J57hJyUdIsk0WarGd6To0s+Y+LwvlC55wt+GT/OgkwoXCXvIue1i1sSNWblHEig00GBWiJgfA==", "cpu": [ "arm" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -278,13 +468,14 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.24.0.tgz", - "integrity": "sha512-TDijPXTOeE3eaMkRYpcy3LarIg13dS9wWHRdwYRnzlwlA370rNdZqbcp0WTyyV/k2zSxfko52+C7jU5F9Tfj1g==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.24.2.tgz", + "integrity": "sha512-7HnAD6074BW43YvvUmE/35Id9/NB7BeX5EoNkK9obndmZBUk8xmJJeU7DwmUeN7tkysslb2eSl6CTrYz6oEMQg==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -294,13 +485,14 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.24.0.tgz", - "integrity": "sha512-K40ip1LAcA0byL05TbCQ4yJ4swvnbzHscRmUilrmP9Am7//0UjPreh4lpYzvThT2Quw66MhjG//20mrufm40mA==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.24.2.tgz", + "integrity": "sha512-sfv0tGPQhcZOgTKO3oBE9xpHuUqguHvSo4jl+wjnKwFpapx+vUDcawbwPNuBIAYdRAvIDBfZVvXprIj3HA+Ugw==", "cpu": [ "ia32" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -310,13 +502,14 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.24.0.tgz", - "integrity": "sha512-0mswrYP/9ai+CU0BzBfPMZ8RVm3RGAN/lmOMgW4aFUSOQBjA31UP8Mr6DDhWSuMwj7jaWOT0p0WoZ6jeHhrD7g==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.24.2.tgz", + "integrity": "sha512-CN9AZr8kEndGooS35ntToZLTQLHEjtVB5n7dl8ZcTZMonJ7CCfStrYhrzF97eAecqVbVJ7APOEe18RPI4KLhwQ==", "cpu": [ "loong64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -326,13 +519,14 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.24.0.tgz", - "integrity": "sha512-hIKvXm0/3w/5+RDtCJeXqMZGkI2s4oMUGj3/jM0QzhgIASWrGO5/RlzAzm5nNh/awHE0A19h/CvHQe6FaBNrRA==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.24.2.tgz", + "integrity": "sha512-iMkk7qr/wl3exJATwkISxI7kTcmHKE+BlymIAbHO8xanq/TjHaaVThFF6ipWzPHryoFsesNQJPE/3wFJw4+huw==", "cpu": [ "mips64el" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -342,13 +536,14 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.24.0.tgz", - "integrity": "sha512-HcZh5BNq0aC52UoocJxaKORfFODWXZxtBaaZNuN3PUX3MoDsChsZqopzi5UupRhPHSEHotoiptqikjN/B77mYQ==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.24.2.tgz", + "integrity": "sha512-shsVrgCZ57Vr2L8mm39kO5PPIb+843FStGt7sGGoqiiWYconSxwTiuswC1VJZLCjNiMLAMh34jg4VSEQb+iEbw==", "cpu": [ "ppc64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -358,13 +553,14 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.24.0.tgz", - "integrity": "sha512-bEh7dMn/h3QxeR2KTy1DUszQjUrIHPZKyO6aN1X4BCnhfYhuQqedHaa5MxSQA/06j3GpiIlFGSsy1c7Gf9padw==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.24.2.tgz", + "integrity": "sha512-4eSFWnU9Hhd68fW16GD0TINewo1L6dRrB+oLNNbYyMUAeOD2yCK5KXGK1GH4qD/kT+bTEXjsyTCiJGHPZ3eM9Q==", "cpu": [ "riscv64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -374,13 +570,14 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.24.0.tgz", - "integrity": "sha512-ZcQ6+qRkw1UcZGPyrCiHHkmBaj9SiCD8Oqd556HldP+QlpUIe2Wgn3ehQGVoPOvZvtHm8HPx+bH20c9pvbkX3g==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.24.2.tgz", + "integrity": "sha512-S0Bh0A53b0YHL2XEXC20bHLuGMOhFDO6GN4b3YjRLK//Ep3ql3erpNcPlEFed93hsQAjAQDNsvcK+hV90FubSw==", "cpu": [ "s390x" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -390,13 +587,14 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.24.0.tgz", - "integrity": "sha512-vbutsFqQ+foy3wSSbmjBXXIJ6PL3scghJoM8zCL142cGaZKAdCZHyf+Bpu/MmX9zT9Q0zFBVKb36Ma5Fzfa8xA==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.24.2.tgz", + "integrity": "sha512-8Qi4nQcCTbLnK9WoMjdC9NiTG6/E38RNICU6sUNqK0QFxCYgoARqVqxdFmWkdonVsvGqWhmm7MO0jyTqLqwj0Q==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" @@ -405,14 +603,32 @@ "node": ">=18" } }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.24.2.tgz", + "integrity": "sha512-wuLK/VztRRpMt9zyHSazyCVdCXlpHkKm34WUyinD2lzK07FAHTq0KQvZZlXikNWkDGoT6x3TD51jKQ7gMVpopw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.24.0.tgz", - "integrity": "sha512-hjQ0R/ulkO8fCYFsG0FZoH+pWgTTDreqpqY7UnQntnaKv95uP5iW3+dChxnx7C3trQQU40S+OgWhUVwCjVFLvg==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.24.2.tgz", + "integrity": "sha512-VefFaQUc4FMmJuAxmIHgUmfNiLXY438XrL4GDNV1Y1H/RW3qow68xTwjZKfj/+Plp9NANmzbH5R40Meudu8mmw==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "netbsd" @@ -422,13 +638,14 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.24.0.tgz", - "integrity": "sha512-MD9uzzkPQbYehwcN583yx3Tu5M8EIoTD+tUgKF982WYL9Pf5rKy9ltgD0eUgs8pvKnmizxjXZyLt0z6DC3rRXg==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.24.2.tgz", + "integrity": "sha512-YQbi46SBct6iKnszhSvdluqDmxCJA+Pu280Av9WICNwQmMxV7nLRHZfjQzwbPs3jeWnuAhE9Jy0NrnJ12Oz+0A==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "openbsd" @@ -438,13 +655,14 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.24.0.tgz", - "integrity": "sha512-4ir0aY1NGUhIC1hdoCzr1+5b43mw99uNwVzhIq1OY3QcEwPDO3B7WNXBzaKY5Nsf1+N11i1eOfFcq+D/gOS15Q==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.24.2.tgz", + "integrity": "sha512-+iDS6zpNM6EnJyWv0bMGLWSWeXGN/HTaF/LXHXHwejGsVi+ooqDfMCCTerNFxEkM3wYVcExkeGXNqshc9iMaOA==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "openbsd" @@ -454,13 +672,14 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.24.0.tgz", - "integrity": "sha512-jVzdzsbM5xrotH+W5f1s+JtUy1UWgjU0Cf4wMvffTB8m6wP5/kx0KiaLHlbJO+dMgtxKV8RQ/JvtlFcdZ1zCPA==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.24.2.tgz", + "integrity": "sha512-hTdsW27jcktEvpwNHJU4ZwWFGkz2zRJUz8pvddmXPtXDzVKTTINmlmga3ZzwcuMpUvLw7JkLy9QLKyGpD2Yxig==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "sunos" @@ -470,13 +689,14 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.24.0.tgz", - "integrity": "sha512-iKc8GAslzRpBytO2/aN3d2yb2z8XTVfNV0PjGlCxKo5SgWmNXx82I/Q3aG1tFfS+A2igVCY97TJ8tnYwpUWLCA==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.24.2.tgz", + "integrity": "sha512-LihEQ2BBKVFLOC9ZItT9iFprsE9tqjDjnbulhHoFxYQtQfai7qfluVODIYxt1PgdoyQkz23+01rzwNwYfutxUQ==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" @@ -486,13 +706,14 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.24.0.tgz", - "integrity": "sha512-vQW36KZolfIudCcTnaTpmLQ24Ha1RjygBo39/aLkM2kmjkWmZGEJ5Gn9l5/7tzXA42QGIoWbICfg6KLLkIw6yw==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.24.2.tgz", + "integrity": "sha512-q+iGUwfs8tncmFC9pcnD5IvRHAzmbwQ3GPS5/ceCyHdjXubwQWI12MKWSNSMYLJMq23/IUCvJMS76PDqXe1fxA==", "cpu": [ "ia32" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" @@ -502,13 +723,14 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.24.0.tgz", - "integrity": "sha512-7IAFPrjSQIJrGsK6flwg7NFmwBoSTyF3rl7If0hNUFQU4ilTsEPL6GuMuU9BfIWVVGuRnuIidkSMC+c0Otu8IA==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.24.2.tgz", + "integrity": "sha512-7VTgWzgMGvup6aSqDPLiW5zHaxYJGTO4OokMjIlrCtf+VpEL+cXKtCvg723iguPYI5oaUNdS+/V7OU2gvXVWEg==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" @@ -578,10 +800,11 @@ } }, "node_modules/@eslint/core": { - "version": "0.9.1", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.9.1.tgz", - "integrity": "sha512-GuUdqkyyzQI5RMIWkHhvTWLCyLo1jNK3vzkSyaExH5kHPDHcuL2VOpHjmMY+y3+NC69qAKToBqldTBgYeLSr9Q==", + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.10.0.tgz", + "integrity": "sha512-gFHJ+xBOo4G3WRlR1e/3G8A6/KZAH6zcE/hkLRCZTi/B9avAG365QhFA8uOGzTMqgTghpn7/fSnscW++dpMSAw==", "dev": true, + "license": "Apache-2.0", "dependencies": { "@types/json-schema": "^7.0.15" }, @@ -635,10 +858,11 @@ } }, "node_modules/@eslint/js": { - "version": "9.16.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.16.0.tgz", - "integrity": "sha512-tw2HxzQkrbeuvyj1tG2Yqq+0H9wGoI2IMk4EOsQeX+vmd75FtJAzf+gTA69WF+baUKRYQ3x2kbLE08js5OsTVg==", + "version": "9.18.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.18.0.tgz", + "integrity": "sha512-fK6L7rxcq6/z+AaQMtiFTkvbHkBLNlwyRxHpKawP0x3u9+NC6MQTnFW+AdpwC6gfHTW0051cokQgtTN2FqlxQA==", "dev": true, + "license": "MIT", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } @@ -653,11 +877,13 @@ } }, "node_modules/@eslint/plugin-kit": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.2.4.tgz", - "integrity": "sha512-zSkKow6H5Kdm0ZUQUB2kV5JIXqoG0+uH5YADhaEHswm664N9Db8dXSi0nMJpacpMf+MyyglF1vnZohpEg5yUtg==", + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.2.5.tgz", + "integrity": "sha512-lB05FkqEdUg2AA0xEbUz0SnkXT1LcCTa438W4IWTUh4hdOnVbQyOJ81OrDXsJk/LSiJHubgGEFoR5EHq1NsH1A==", "dev": true, + "license": "Apache-2.0", "dependencies": { + "@eslint/core": "^0.10.0", "levn": "^0.4.1" }, "engines": { @@ -816,6 +1042,74 @@ "@types/node": ">=18" } }, + "node_modules/@iodigital/vite-plugin-msw": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@iodigital/vite-plugin-msw/-/vite-plugin-msw-2.0.0.tgz", + "integrity": "sha512-b4041DewCVSGKQHu5BocPzMSW94/8h59AV6n6umEXHoeqf4nX77ve+gPb/+lmMuPXOtPL1d8465CKyx0ek6Fuw==", + "dev": true, + "dependencies": { + "@mswjs/interceptors": "^0.25.7", + "body-parser": "^1.20.2", + "fs-extra": "^11.1.1", + "headers-polyfill": "^4.0.2", + "strict-event-emitter": "^0.5.1" + }, + "peerDependencies": { + "msw": "^2.0.0" + } + }, + "node_modules/@iodigital/vite-plugin-msw/node_modules/@mswjs/interceptors": { + "version": "0.25.16", + "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.25.16.tgz", + "integrity": "sha512-8QC8JyKztvoGAdPgyZy49c9vSHHAZjHagwl4RY9E8carULk8ym3iTaiawrT1YoLF/qb449h48f71XDPgkUSOUg==", + "dev": true, + "dependencies": { + "@open-draft/deferred-promise": "^2.2.0", + "@open-draft/logger": "^0.3.0", + "@open-draft/until": "^2.0.0", + "is-node-process": "^1.2.0", + "outvariant": "^1.2.1", + "strict-event-emitter": "^0.5.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@iodigital/vite-plugin-msw/node_modules/fs-extra": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/@iodigital/vite-plugin-msw/node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/@iodigital/vite-plugin-msw/node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "dev": true, + "engines": { + "node": ">= 10.0.0" + } + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -1008,6 +1302,57 @@ "tslib": "2" } }, + "node_modules/@koa/router": { + "version": "9.4.0", + "resolved": "https://registry.npmjs.org/@koa/router/-/router-9.4.0.tgz", + "integrity": "sha512-dOOXgzqaDoHu5qqMEPLKEgLz5CeIA7q8+1W62mCvFVCOqeC71UoTGJ4u1xUSOpIl2J1x2pqrNULkFteUeZW3/A==", + "deprecated": "**IMPORTANT 10x+ PERFORMANCE UPGRADE**: Please upgrade to v12.0.1+ as we have fixed an issue with debuglog causing 10x slower router benchmark performance, see https://github.com/koajs/router/pull/173", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "http-errors": "^1.7.3", + "koa-compose": "^4.1.0", + "methods": "^1.1.2", + "path-to-regexp": "^6.1.0" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/@koa/router/node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@koa/router/node_modules/http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "dev": true, + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@koa/router/node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/@leichtgewicht/ip-codec": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz", @@ -1146,6 +1491,20 @@ "node": ">=14" } }, + "node_modules/@playwright/browser-chromium": { + "version": "1.49.1", + "resolved": "https://registry.npmjs.org/@playwright/browser-chromium/-/browser-chromium-1.49.1.tgz", + "integrity": "sha512-LLeyllKSucbojsJBOpdJshwW27ZXZs3oypqffkVWLUvxX2azHJMOevsOcWpjCfoYbpevkaEozM2xHeSUGF00lg==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.49.1" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@polka/url": { "version": "1.0.0-next.24", "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.24.tgz", @@ -1167,6 +1526,8 @@ "url": "https://github.com/webgptorg/promptbook/blob/main/README.md#%EF%B8%8F-contributing" } ], + "optional": true, + "peer": true, "dependencies": { "spacetrim": "0.11.59" } @@ -1176,6 +1537,8 @@ "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.6.1.tgz", "integrity": "sha512-aBSREisdsGH890S2rQqK82qmQYU3uFpSH8wcZWHgHzl3LfzsxAKbLNiAG9mO8v1Y0UICBeClICxPJvyr0rcuxg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "debug": "^4.4.0", "extract-zip": "^2.0.1", @@ -1194,391 +1557,421 @@ } }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.28.1.tgz", - "integrity": "sha512-2aZp8AES04KI2dy3Ss6/MDjXbwBzj+i0GqKtWXgw2/Ma6E4jJvujryO6gJAghIRVz7Vwr9Gtl/8na3nDUKpraQ==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.30.1.tgz", + "integrity": "sha512-pSWY+EVt3rJ9fQ3IqlrEUtXh3cGqGtPDH1FQlNZehO2yYxCHEX1SPsz1M//NXwYfbTlcKr9WObLnJX9FsS9K1Q==", "cpu": [ "arm" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "android" ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.28.1.tgz", - "integrity": "sha512-EbkK285O+1YMrg57xVA+Dp0tDBRB93/BZKph9XhMjezf6F4TpYjaUSuPt5J0fZXlSag0LmZAsTmdGGqPp4pQFA==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.30.1.tgz", + "integrity": "sha512-/NA2qXxE3D/BRjOJM8wQblmArQq1YoBVJjrjoTSBS09jgUisq7bqxNHJ8kjCHeV21W/9WDGwJEWSN0KQ2mtD/w==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "android" ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.28.1.tgz", - "integrity": "sha512-prduvrMKU6NzMq6nxzQw445zXgaDBbMQvmKSJaxpaZ5R1QDM8w+eGxo6Y/jhT/cLoCvnZI42oEqf9KQNYz1fqQ==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.30.1.tgz", + "integrity": "sha512-r7FQIXD7gB0WJ5mokTUgUWPl0eYIH0wnxqeSAhuIwvnnpjdVB8cRRClyKLQr7lgzjctkbp5KmswWszlwYln03Q==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "darwin" ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.28.1.tgz", - "integrity": "sha512-WsvbOunsUk0wccO/TV4o7IKgloJ942hVFK1CLatwv6TJspcCZb9umQkPdvB7FihmdxgaKR5JyxDjWpCOp4uZlQ==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.30.1.tgz", + "integrity": "sha512-x78BavIwSH6sqfP2xeI1hd1GpHL8J4W2BXcVM/5KYKoAD3nNsfitQhvWSw+TFtQTLZ9OmlF+FEInEHyubut2OA==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "darwin" ] }, "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.28.1.tgz", - "integrity": "sha512-HTDPdY1caUcU4qK23FeeGxCdJF64cKkqajU0iBnTVxS8F7H/7BewvYoG+va1KPSL63kQ1PGNyiwKOfReavzvNA==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.30.1.tgz", + "integrity": "sha512-HYTlUAjbO1z8ywxsDFWADfTRfTIIy/oUlfIDmlHYmjUP2QRDTzBuWXc9O4CXM+bo9qfiCclmHk1x4ogBjOUpUQ==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "freebsd" ] }, "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.28.1.tgz", - "integrity": "sha512-m/uYasxkUevcFTeRSM9TeLyPe2QDuqtjkeoTpP9SW0XxUWfcYrGDMkO/m2tTw+4NMAF9P2fU3Mw4ahNvo7QmsQ==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.30.1.tgz", + "integrity": "sha512-1MEdGqogQLccphhX5myCJqeGNYTNcmTyaic9S7CG3JhwuIByJ7J05vGbZxsizQthP1xpVx7kd3o31eOogfEirw==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "freebsd" ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.28.1.tgz", - "integrity": "sha512-QAg11ZIt6mcmzpNE6JZBpKfJaKkqTm1A9+y9O+frdZJEuhQxiugM05gnCWiANHj4RmbgeVJpTdmKRmH/a+0QbA==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.30.1.tgz", + "integrity": "sha512-PaMRNBSqCx7K3Wc9QZkFx5+CX27WFpAMxJNiYGAXfmMIKC7jstlr32UhTgK6T07OtqR+wYlWm9IxzennjnvdJg==", "cpu": [ "arm" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.28.1.tgz", - "integrity": "sha512-dRP9PEBfolq1dmMcFqbEPSd9VlRuVWEGSmbxVEfiq2cs2jlZAl0YNxFzAQS2OrQmsLBLAATDMb3Z6MFv5vOcXg==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.30.1.tgz", + "integrity": "sha512-B8Rcyj9AV7ZlEFqvB5BubG5iO6ANDsRKlhIxySXcF1axXYUyqwBok+XZPgIYGBgs7LDXfWfifxhw0Ik57T0Yug==", "cpu": [ "arm" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.28.1.tgz", - "integrity": "sha512-uGr8khxO+CKT4XU8ZUH1TTEUtlktK6Kgtv0+6bIFSeiSlnGJHG1tSFSjm41uQ9sAO/5ULx9mWOz70jYLyv1QkA==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.30.1.tgz", + "integrity": "sha512-hqVyueGxAj3cBKrAI4aFHLV+h0Lv5VgWZs9CUGqr1z0fZtlADVV1YPOij6AhcK5An33EXaxnDLmJdQikcn5NEw==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.28.1.tgz", - "integrity": "sha512-QF54q8MYGAqMLrX2t7tNpi01nvq5RI59UBNx+3+37zoKX5KViPo/gk2QLhsuqok05sSCRluj0D00LzCwBikb0A==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.30.1.tgz", + "integrity": "sha512-i4Ab2vnvS1AE1PyOIGp2kXni69gU2DAUVt6FSXeIqUCPIR3ZlheMW3oP2JkukDfu3PsexYRbOiJrY+yVNSk9oA==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-loongarch64-gnu": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.28.1.tgz", - "integrity": "sha512-vPul4uodvWvLhRco2w0GcyZcdyBfpfDRgNKU+p35AWEbJ/HPs1tOUrkSueVbBS0RQHAf/A+nNtDpvw95PeVKOA==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.30.1.tgz", + "integrity": "sha512-fARcF5g296snX0oLGkVxPmysetwUk2zmHcca+e9ObOovBR++9ZPOhqFUM61UUZ2EYpXVPN1redgqVoBB34nTpQ==", "cpu": [ "loong64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.28.1.tgz", - "integrity": "sha512-pTnTdBuC2+pt1Rmm2SV7JWRqzhYpEILML4PKODqLz+C7Ou2apEV52h19CR7es+u04KlqplggmN9sqZlekg3R1A==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.30.1.tgz", + "integrity": "sha512-GLrZraoO3wVT4uFXh67ElpwQY0DIygxdv0BNW9Hkm3X34wu+BkqrDrkcsIapAY+N2ATEbvak0XQ9gxZtCIA5Rw==", "cpu": [ "ppc64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.28.1.tgz", - "integrity": "sha512-vWXy1Nfg7TPBSuAncfInmAI/WZDd5vOklyLJDdIRKABcZWojNDY0NJwruY2AcnCLnRJKSaBgf/GiJfauu8cQZA==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.30.1.tgz", + "integrity": "sha512-0WKLaAUUHKBtll0wvOmh6yh3S0wSU9+yas923JIChfxOaaBarmb/lBKPF0w/+jTVozFnOXJeRGZ8NvOxvk/jcw==", "cpu": [ "riscv64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.28.1.tgz", - "integrity": "sha512-/yqC2Y53oZjb0yz8PVuGOQQNOTwxcizudunl/tFs1aLvObTclTwZ0JhXF2XcPT/zuaymemCDSuuUPXJJyqeDOg==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.30.1.tgz", + "integrity": "sha512-GWFs97Ruxo5Bt+cvVTQkOJ6TIx0xJDD/bMAOXWJg8TCSTEK8RnFeOeiFTxKniTc4vMIaWvCplMAFBt9miGxgkA==", "cpu": [ "s390x" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.28.1.tgz", - "integrity": "sha512-fzgeABz7rrAlKYB0y2kSEiURrI0691CSL0+KXwKwhxvj92VULEDQLpBYLHpF49MSiPG4sq5CK3qHMnb9tlCjBw==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.30.1.tgz", + "integrity": "sha512-UtgGb7QGgXDIO+tqqJ5oZRGHsDLO8SlpE4MhqpY9Llpzi5rJMvrK6ZGhsRCST2abZdBqIBeXW6WPD5fGK5SDwg==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.28.1.tgz", - "integrity": "sha512-xQTDVzSGiMlSshpJCtudbWyRfLaNiVPXt1WgdWTwWz9n0U12cI2ZVtWe/Jgwyv/6wjL7b66uu61Vg0POWVfz4g==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.30.1.tgz", + "integrity": "sha512-V9U8Ey2UqmQsBT+xTOeMzPzwDzyXmnAoO4edZhL7INkwQcaW1Ckv3WJX3qrrp/VHaDkEWIBWhRwP47r8cdrOow==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.28.1.tgz", - "integrity": "sha512-wSXmDRVupJstFP7elGMgv+2HqXelQhuNf+IS4V+nUpNVi/GUiBgDmfwD0UGN3pcAnWsgKG3I52wMOBnk1VHr/A==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.30.1.tgz", + "integrity": "sha512-WabtHWiPaFF47W3PkHnjbmWawnX/aE57K47ZDT1BXTS5GgrBUEpvOzq0FI0V/UYzQJgdb8XlhVNH8/fwV8xDjw==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.28.1.tgz", - "integrity": "sha512-ZkyTJ/9vkgrE/Rk9vhMXhf8l9D+eAhbAVbsGsXKy2ohmJaWg0LPQLnIxRdRp/bKyr8tXuPlXhIoGlEB5XpJnGA==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.30.1.tgz", + "integrity": "sha512-pxHAU+Zv39hLUTdQQHUVHf4P+0C47y/ZloorHpzs2SXMRqeAWmGghzAhfOlzFHHwjvgokdFAhC4V+6kC1lRRfw==", "cpu": [ "ia32" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.28.1.tgz", - "integrity": "sha512-ZvK2jBafvttJjoIdKm/Q/Bh7IJ1Ose9IBOwpOXcOvW3ikGTQGmKDgxTC6oCAzW6PynbkKP8+um1du81XJHZ0JA==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.30.1.tgz", + "integrity": "sha512-D6qjsXGcvhTjv0kI4fU8tUuBDF/Ueee4SVX79VfNDXZa64TfCW1Slkb6Z7O1p7vflqZjcmOVdZlqf8gvJxc6og==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/@rspack/binding": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@rspack/binding/-/binding-1.1.6.tgz", - "integrity": "sha512-vfeBEgGOYVwqj5cQjGyvdfrr/BEihAHlyIsobL98FZjTF0uig+bj2yJUH5Ib5F0BpIUKVG3Pw0IjlUBqcVpZsQ==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/@rspack/binding/-/binding-1.1.8.tgz", + "integrity": "sha512-+/JzXx1HctfgPj+XtsCTbRkxiaOfAXGZZLEvs7jgp04WgWRSZ5u97WRCePNPvy+sCfOEH/2zw2ZK36Z7oQRGhQ==", "dev": true, + "license": "MIT", "optionalDependencies": { - "@rspack/binding-darwin-arm64": "1.1.6", - "@rspack/binding-darwin-x64": "1.1.6", - "@rspack/binding-linux-arm64-gnu": "1.1.6", - "@rspack/binding-linux-arm64-musl": "1.1.6", - "@rspack/binding-linux-x64-gnu": "1.1.6", - "@rspack/binding-linux-x64-musl": "1.1.6", - "@rspack/binding-win32-arm64-msvc": "1.1.6", - "@rspack/binding-win32-ia32-msvc": "1.1.6", - "@rspack/binding-win32-x64-msvc": "1.1.6" + "@rspack/binding-darwin-arm64": "1.1.8", + "@rspack/binding-darwin-x64": "1.1.8", + "@rspack/binding-linux-arm64-gnu": "1.1.8", + "@rspack/binding-linux-arm64-musl": "1.1.8", + "@rspack/binding-linux-x64-gnu": "1.1.8", + "@rspack/binding-linux-x64-musl": "1.1.8", + "@rspack/binding-win32-arm64-msvc": "1.1.8", + "@rspack/binding-win32-ia32-msvc": "1.1.8", + "@rspack/binding-win32-x64-msvc": "1.1.8" } }, "node_modules/@rspack/binding-darwin-arm64": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@rspack/binding-darwin-arm64/-/binding-darwin-arm64-1.1.6.tgz", - "integrity": "sha512-x9dxm2yyiMuL1FBwvWNNMs2/mEUJmRoSRgYb8pblR7HDaTRORrjBFCqhaYlGyAqtQaeUy7o2VAQlE0BavIiFYA==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/@rspack/binding-darwin-arm64/-/binding-darwin-arm64-1.1.8.tgz", + "integrity": "sha512-I7avr471ghQ3LAqKm2fuXuJPLgQ9gffn5Q4nHi8rsukuZUtiLDPfYzK1QuupEp2JXRWM1gG5lIbSUOht3cD6Ug==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "darwin" ] }, "node_modules/@rspack/binding-darwin-x64": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@rspack/binding-darwin-x64/-/binding-darwin-x64-1.1.6.tgz", - "integrity": "sha512-o0seilveftGiDjy3VPxug20HmAgYyQbNEuagR3i93/t/PT/eWXHnik+C1jjwqcivZL1Zllqvy4tbZw393aROEQ==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/@rspack/binding-darwin-x64/-/binding-darwin-x64-1.1.8.tgz", + "integrity": "sha512-vfqf/c+mcx8rr1M8LnqKmzDdnrgguflZnjGerBLjNerAc+dcUp3lCvNxRIvZ2TkSZZBW8BpCMgjj3n70CZ4VLQ==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "darwin" ] }, "node_modules/@rspack/binding-linux-arm64-gnu": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@rspack/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.6.tgz", - "integrity": "sha512-4atnoknJx/c3KaQElsMIxHMpPf2jcRRdWsH/SdqJIRSrkWWakMK9Yv4TFwH680I4HDTMf1XLboMVScHzW8e+Mg==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/@rspack/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.8.tgz", + "integrity": "sha512-lZlO/rAJSeozi+qtVLkGSXfe+riPawCwM4FsrflELfNlvvEXpANwtrdJ+LsaNVXcgvhh50ZX2KicTdmx9G2b6Q==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rspack/binding-linux-arm64-musl": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@rspack/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.6.tgz", - "integrity": "sha512-7QMtwUtgFpt3/Y3/X18fSyN+kk4H8ZnZ8tDzQskVWc/j2AQYShZq56XQYqrhClzwujcCVAHauIQ2eiuJ2ASGag==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/@rspack/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.8.tgz", + "integrity": "sha512-bX7exULSZwy8xtDh6Z65b6sRC4uSxGuyvSLCEKyhmG6AnJkg0gQMxk3hoO0hWnyGEZgdJEn+jEhk0fjl+6ZRAQ==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rspack/binding-linux-x64-gnu": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@rspack/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.6.tgz", - "integrity": "sha512-MTjDEfPn4TwHoqs5d5Fck06kmXiTHZctGIcRVfrpg0RK0r1NLEHN+oosavRZ9c9H70f34+NmcHk+/qvV4c8lWg==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/@rspack/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.8.tgz", + "integrity": "sha512-2Prw2USgTJ3aLdLExfik8pAwAHbX4MZrACBGEmR7Vbb56kLjC+++fXkciRc50pUDK4JFr1VQ7eNZrJuDR6GG6Q==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rspack/binding-linux-x64-musl": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@rspack/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.6.tgz", - "integrity": "sha512-LqDw7PTVr/4ZuGA0izgDQfamfr72USFHltR1Qhy2YVC3JmDmhG/pQi13LHcOLVaGH1xoeyCmEPNJpVizzDxSjg==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/@rspack/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.8.tgz", + "integrity": "sha512-bnVGB/mQBKEdzOU/CPmcOE3qEXxGOGGW7/i6iLl2MamVOykJq8fYjL9j86yi6L0r009ja16OgWckykQGc4UqGw==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rspack/binding-win32-arm64-msvc": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@rspack/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.6.tgz", - "integrity": "sha512-RHApLM93YN0WdHpS35u2cm7VCqZ8Yg3CrNRL16VJtyT9e6MBqeScoe4XIgIWKPm7edFyedYAjLX0wQOApwfjkg==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/@rspack/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.8.tgz", + "integrity": "sha512-u+na3gxhzeksm4xZyAzn1+XWo5a5j7hgWA/KcFPDQ8qQNkRknx4jnQMxVtcZ9pLskAYV4AcOV/AIximx7zvv8A==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/@rspack/binding-win32-ia32-msvc": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@rspack/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.1.6.tgz", - "integrity": "sha512-Y6lx4q0eJawRfMPBo/AclTJAPTZ325DSPFBQJB3TnWh9Z2X7P7pQcYc8PHDmfDuYRIdg5WRsQRvVxihSvF7v8w==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/@rspack/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.1.8.tgz", + "integrity": "sha512-FijUxym1INd5fFHwVCLuVP8XEAb4Sk1sMwEEQUlugiDra9ZsLaPw4OgPGxbxkD6SB0DeUz9Zq46Xbcf6d3OgfA==", "cpu": [ "ia32" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/@rspack/binding-win32-x64-msvc": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@rspack/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.6.tgz", - "integrity": "sha512-UuCsfhC/yNuU7xLASOxNXcmsXi2ZvBX14GkxvcdChw6q7IIGNYUKXo1zgR8C1PE/6qDSxmLxbRMS+71d0H3HQg==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/@rspack/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.8.tgz", + "integrity": "sha512-SBzIcND4qpDt71jlu1MCDxt335tqInT3YID9V4DoQ4t8wgM/uad7EgKOWKTK6vc2RRaOIShfS2XzqjNUxPXh4w==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/@rspack/cli": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@rspack/cli/-/cli-1.1.6.tgz", - "integrity": "sha512-404JTAadncCp81sDa7nGZdsT7r1Ry8fALR8Wkp9VMTUhWEFlbDGQvOTyali24pfyJxJTdsarSabmNhbDO5okJw==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/@rspack/cli/-/cli-1.1.8.tgz", + "integrity": "sha512-LiodoRmv1eAYtGQftPHX5Cr0uBmVUywOXZvnrVipXZupRsl1DNYO6Ao2kYlvSPYi77ymhOBRoxrMqbUOUklbIg==", "dev": true, + "license": "MIT", "dependencies": { "@discoveryjs/json-ext": "^0.5.7", "@rspack/dev-server": "1.0.9", @@ -1739,13 +2132,14 @@ } }, "node_modules/@rspack/core": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@rspack/core/-/core-1.1.6.tgz", - "integrity": "sha512-q0VLphOF5VW2FEG7Vbdq3Ke4I74FbELE/8xmKghSalFtULLZ44SoSz8lyotfMim9GXIRFhDokAaH8WICmPxG+g==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/@rspack/core/-/core-1.1.8.tgz", + "integrity": "sha512-pcZtcj5iXLCuw9oElTYC47bp/RQADm/MMEb3djHdwJuSlFWfWPQi5QFgJ/lJAxIW9UNHnTFrYtytycfjpuoEcA==", "dev": true, + "license": "MIT", "dependencies": { "@module-federation/runtime-tools": "0.5.1", - "@rspack/binding": "1.1.6", + "@rspack/binding": "1.1.8", "@rspack/lite-tapable": "1.0.1", "caniuse-lite": "^1.0.30001616" }, @@ -1961,6 +2355,7 @@ "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.0.tgz", "integrity": "sha512-pemlzrSESWbdAloYml3bAJMEfNh1Z7EduzqPKprCH5S341frlpYnUEW0H72dLxa6IsYr+mPno20GiSm+h9dEdQ==", "dev": true, + "license": "MIT", "dependencies": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", @@ -1976,10 +2371,11 @@ } }, "node_modules/@testing-library/user-event": { - "version": "14.5.2", - "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.5.2.tgz", - "integrity": "sha512-YAh82Wh4TIrxYLmfGcixwD18oIjyC1pFQC2Y01F2lzV2HTMiYrI0nze0FD0ocB//CKS/7jIUgae+adPqxK5yCQ==", + "version": "14.6.0", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.0.tgz", + "integrity": "sha512-+jsfK7kVJbqnCYtLTln8Ja/NmVrZRwBJHmHR9IxIVccMWSOZ6Oy0FkDJNeyVu4QSpMNmRfy10Xb76ObRDlWWBQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=12", "npm": ">=6" @@ -1992,13 +2388,16 @@ "version": "0.23.0", "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/@types/aria-query": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/@types/body-parser": { "version": "1.19.5", @@ -2126,6 +2525,15 @@ "@types/node": "*" } }, + "node_modules/@types/http-server": { + "version": "0.12.4", + "resolved": "https://registry.npmjs.org/@types/http-server/-/http-server-0.12.4.tgz", + "integrity": "sha512-vsn4pvP2oRFALLuM5Rca6qUmSPG7u0VNjOuqvL57l3bKldQRWdUZPeSiARhzagDxgfNCHn/o8WlWk4KinBauUg==", + "dev": true, + "dependencies": { + "@types/connect": "*" + } + }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", @@ -2150,6 +2558,17 @@ "@types/istanbul-lib-report": "*" } }, + "node_modules/@types/jsdom": { + "version": "21.1.7", + "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-21.1.7.tgz", + "integrity": "sha512-yOriVnggzrnQ3a9OKOCxaVuSug3w3/SbOj5i7VwXWZEyUNl3bLF9V3MfxGbZKuwqJOQyRfqXyROBB1CoZLFWzA==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/tough-cookie": "*", + "parse5": "^7.0.0" + } + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -2184,10 +2603,11 @@ "dev": true }, "node_modules/@types/node": { - "version": "22.10.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz", - "integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==", + "version": "22.10.7", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.7.tgz", + "integrity": "sha512-V09KvXxFiutGp6B7XkpaDXlNadZxrzajcY50EuoLIpQ6WWYCSvf19lVIazzfIzQvhUN2HjX12spLojTnhuKlGg==", "dev": true, + "license": "MIT", "dependencies": { "undici-types": "~6.20.0" } @@ -2219,6 +2639,15 @@ "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", "dev": true }, + "node_modules/@types/s3rver": { + "version": "3.7.4", + "resolved": "https://registry.npmjs.org/@types/s3rver/-/s3rver-3.7.4.tgz", + "integrity": "sha512-CMCmdNszxS2FsIznWvBMVCl6fpvr5ueaFCaY0iSoH7Ud5maGcLghukpDvsXBnIcp92cv2HeVnVqI1p8yPcab9Q==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/send": { "version": "0.17.4", "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", @@ -2253,7 +2682,9 @@ "version": "8.1.5", "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.5.tgz", "integrity": "sha512-mQkU2jY8jJEF7YHjHvsQO8+3ughTL1mcnn96igfhONmR+fUPSKIkefQYpSe8bsly2Ep7oQbn/6VG5/9/0qcArQ==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/@types/sockjs": { "version": "0.3.36", @@ -2285,11 +2716,19 @@ "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", "dev": true }, + "node_modules/@types/triple-beam": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", + "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", + "dev": true + }, "node_modules/@types/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@types/which/-/which-2.0.2.tgz", "integrity": "sha512-113D3mDkZDjo+EeUEHCFy0qniNc1ZpecGiAU7WSo7YDoSzolZIQKpYFHrPpjkB2nuyahcKfrmLXeQlh7gqJYdw==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/@types/ws": { "version": "8.5.10", @@ -2321,25 +2760,27 @@ "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", "dev": true, "optional": true, + "peer": true, "dependencies": { "@types/node": "*" } }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.18.0.tgz", - "integrity": "sha512-NR2yS7qUqCL7AIxdJUQf2MKKNDVNaig/dEB0GBLU7D+ZdHgK1NoH/3wsgO3OnPVipn51tG3MAwaODEGil70WEw==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.20.0.tgz", + "integrity": "sha512-naduuphVw5StFfqp4Gq4WhIBE2gN1GEmMUExpJYknZJdRnc+2gDzB8Z3+5+/Kv33hPQRDGzQO/0opHE72lZZ6A==", "dev": true, + "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "8.18.0", - "@typescript-eslint/type-utils": "8.18.0", - "@typescript-eslint/utils": "8.18.0", - "@typescript-eslint/visitor-keys": "8.18.0", + "@typescript-eslint/scope-manager": "8.20.0", + "@typescript-eslint/type-utils": "8.20.0", + "@typescript-eslint/utils": "8.20.0", + "@typescript-eslint/visitor-keys": "8.20.0", "graphemer": "^1.4.0", "ignore": "^5.3.1", "natural-compare": "^1.4.0", - "ts-api-utils": "^1.3.0" + "ts-api-utils": "^2.0.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2355,15 +2796,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.18.0.tgz", - "integrity": "sha512-hgUZ3kTEpVzKaK3uNibExUYm6SKKOmTU2BOxBSvOYwtJEPdVQ70kZJpPjstlnhCHcuc2WGfSbpKlb/69ttyN5Q==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.20.0.tgz", + "integrity": "sha512-gKXG7A5HMyjDIedBi6bUrDcun8GIjnI8qOwVLiY3rx6T/sHP/19XLJOnIq/FgQvWLHja5JN/LSE7eklNBr612g==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.18.0", - "@typescript-eslint/types": "8.18.0", - "@typescript-eslint/typescript-estree": "8.18.0", - "@typescript-eslint/visitor-keys": "8.18.0", + "@typescript-eslint/scope-manager": "8.20.0", + "@typescript-eslint/types": "8.20.0", + "@typescript-eslint/typescript-estree": "8.20.0", + "@typescript-eslint/visitor-keys": "8.20.0", "debug": "^4.3.4" }, "engines": { @@ -2379,13 +2821,14 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.18.0.tgz", - "integrity": "sha512-PNGcHop0jkK2WVYGotk/hxj+UFLhXtGPiGtiaWgVBVP1jhMoMCHlTyJA+hEj4rszoSdLTK3fN4oOatrL0Cp+Xw==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.20.0.tgz", + "integrity": "sha512-J7+VkpeGzhOt3FeG1+SzhiMj9NzGD/M6KoGn9f4dbz3YzK9hvbhVTmLj/HiTp9DazIzJ8B4XcM80LrR9Dm1rJw==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.18.0", - "@typescript-eslint/visitor-keys": "8.18.0" + "@typescript-eslint/types": "8.20.0", + "@typescript-eslint/visitor-keys": "8.20.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2396,15 +2839,16 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.18.0.tgz", - "integrity": "sha512-er224jRepVAVLnMF2Q7MZJCq5CsdH2oqjP4dT7K6ij09Kyd+R21r7UVJrF0buMVdZS5QRhDzpvzAxHxabQadow==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.20.0.tgz", + "integrity": "sha512-bPC+j71GGvA7rVNAHAtOjbVXbLN5PkwqMvy1cwGeaxUoRQXVuKCebRoLzm+IPW/NtFFpstn1ummSIasD5t60GA==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/typescript-estree": "8.18.0", - "@typescript-eslint/utils": "8.18.0", + "@typescript-eslint/typescript-estree": "8.20.0", + "@typescript-eslint/utils": "8.20.0", "debug": "^4.3.4", - "ts-api-utils": "^1.3.0" + "ts-api-utils": "^2.0.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2419,10 +2863,11 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.18.0.tgz", - "integrity": "sha512-FNYxgyTCAnFwTrzpBGq+zrnoTO4x0c1CKYY5MuUTzpScqmY5fmsh2o3+57lqdI3NZucBDCzDgdEbIaNfAjAHQA==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.20.0.tgz", + "integrity": "sha512-cqaMiY72CkP+2xZRrFt3ExRBu0WmVitN/rYPZErA80mHjHx/Svgp8yfbzkJmDoQ/whcytOPO9/IZXnOc+wigRA==", "dev": true, + "license": "MIT", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, @@ -2432,19 +2877,20 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.18.0.tgz", - "integrity": "sha512-rqQgFRu6yPkauz+ms3nQpohwejS8bvgbPyIDq13cgEDbkXt4LH4OkDMT0/fN1RUtzG8e8AKJyDBoocuQh8qNeg==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.20.0.tgz", + "integrity": "sha512-Y7ncuy78bJqHI35NwzWol8E0X7XkRVS4K4P4TCyzWkOJih5NDvtoRDW4Ba9YJJoB2igm9yXDdYI/+fkiiAxPzA==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.18.0", - "@typescript-eslint/visitor-keys": "8.18.0", + "@typescript-eslint/types": "8.20.0", + "@typescript-eslint/visitor-keys": "8.20.0", "debug": "^4.3.4", "fast-glob": "^3.3.2", "is-glob": "^4.0.3", "minimatch": "^9.0.4", "semver": "^7.6.0", - "ts-api-utils": "^1.3.0" + "ts-api-utils": "^2.0.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2458,15 +2904,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.18.0.tgz", - "integrity": "sha512-p6GLdY383i7h5b0Qrfbix3Vc3+J2k6QWw6UMUeY5JGfm3C5LbZ4QIZzJNoNOfgyRe0uuYKjvVOsO/jD4SJO+xg==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.20.0.tgz", + "integrity": "sha512-dq70RUw6UK9ei7vxc4KQtBRk7qkHZv447OUZ6RPQMQl71I3NZxQJX/f32Smr+iqWrB02pHKn2yAdHBb0KNrRMA==", "dev": true, + "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "8.18.0", - "@typescript-eslint/types": "8.18.0", - "@typescript-eslint/typescript-estree": "8.18.0" + "@typescript-eslint/scope-manager": "8.20.0", + "@typescript-eslint/types": "8.20.0", + "@typescript-eslint/typescript-estree": "8.20.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2481,12 +2928,13 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.18.0.tgz", - "integrity": "sha512-pCh/qEA8Lb1wVIqNvBke8UaRjJ6wrAWkJO5yyIbs8Yx6TNGYyfNjOo61tLv+WwLvoLPp4BQ8B7AHKijl8NGUfw==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.20.0.tgz", + "integrity": "sha512-v/BpkeeYAsPkKCkR8BDwcno0llhzWVqPOamQrAEMdpZav2Y9OVjd9dwJyBLJWwf335B5DmlifECIkZRJCaGaHA==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.18.0", + "@typescript-eslint/types": "8.20.0", "eslint-visitor-keys": "^4.2.0" }, "engines": { @@ -2502,6 +2950,7 @@ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.0.tgz", "integrity": "sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==", "dev": true, + "license": "Apache-2.0", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, @@ -2510,19 +2959,20 @@ } }, "node_modules/@vitest/browser": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/browser/-/browser-2.1.8.tgz", - "integrity": "sha512-OWVvEJThRgxlNMYNVLEK/9qVkpRcLvyuKLngIV3Hob01P56NjPHprVBYn+rx4xAJudbM9yrCrywPIEuA3Xyo8A==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@vitest/browser/-/browser-3.0.2.tgz", + "integrity": "sha512-EVXRQTEmwyCNh6qDXIf/fGWp3YXa3KtsMCOXmlD4Yeq62pJaqJ5+iUIY1XLN7TO2iXatGDdLZYbHbR6YQT4FDw==", "dev": true, + "license": "MIT", "dependencies": { "@testing-library/dom": "^10.4.0", - "@testing-library/user-event": "^14.5.2", - "@vitest/mocker": "2.1.8", - "@vitest/utils": "2.1.8", - "magic-string": "^0.30.12", - "msw": "^2.6.4", + "@testing-library/user-event": "^14.6.0", + "@vitest/mocker": "3.0.2", + "@vitest/utils": "3.0.2", + "magic-string": "^0.30.17", + "msw": "^2.7.0", "sirv": "^3.0.0", - "tinyrainbow": "^1.2.0", + "tinyrainbow": "^2.0.0", "ws": "^8.18.0" }, "funding": { @@ -2530,7 +2980,7 @@ }, "peerDependencies": { "playwright": "*", - "vitest": "2.1.8", + "vitest": "3.0.2", "webdriverio": "*" }, "peerDependenciesMeta": { @@ -2560,36 +3010,38 @@ } }, "node_modules/@vitest/expect": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.8.tgz", - "integrity": "sha512-8ytZ/fFHq2g4PJVAtDX57mayemKgDR6X3Oa2Foro+EygiOJHUXhCqBAAKQYYajZpFoIfvBCF1j6R6IYRSIUFuw==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.0.2.tgz", + "integrity": "sha512-dKSHLBcoZI+3pmP5hiZ7I5grNru2HRtEW8Z5Zp4IXog8QYcxhlox7JUPyIIFWfN53+3HW3KPLIl6nSzUGgKSuQ==", "dev": true, + "license": "MIT", "dependencies": { - "@vitest/spy": "2.1.8", - "@vitest/utils": "2.1.8", + "@vitest/spy": "3.0.2", + "@vitest/utils": "3.0.2", "chai": "^5.1.2", - "tinyrainbow": "^1.2.0" + "tinyrainbow": "^2.0.0" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/mocker": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-2.1.8.tgz", - "integrity": "sha512-7guJ/47I6uqfttp33mgo6ga5Gr1VnL58rcqYKyShoRK9ebu8T5Rs6HN3s1NABiBeVTdWNrwUMcHH54uXZBN4zA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.0.2.tgz", + "integrity": "sha512-Hr09FoBf0jlwwSyzIF4Xw31OntpO3XtZjkccpcBf8FeVW3tpiyKlkeUzxS/txzHqpUCNIX157NaTySxedyZLvA==", "dev": true, + "license": "MIT", "dependencies": { - "@vitest/spy": "2.1.8", + "@vitest/spy": "3.0.2", "estree-walker": "^3.0.3", - "magic-string": "^0.30.12" + "magic-string": "^0.30.17" }, "funding": { "url": "https://opencollective.com/vitest" }, "peerDependencies": { "msw": "^2.4.9", - "vite": "^5.0.0" + "vite": "^5.0.0 || ^6.0.0" }, "peerDependenciesMeta": { "msw": { @@ -2601,49 +3053,53 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.1.8.tgz", - "integrity": "sha512-9HiSZ9zpqNLKlbIDRWOnAWqgcA7xu+8YxXSekhr0Ykab7PAYFkhkwoqVArPOtJhPmYeE2YHgKZlj3CP36z2AJQ==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.0.2.tgz", + "integrity": "sha512-yBohcBw/T/p0/JRgYD+IYcjCmuHzjC3WLAKsVE4/LwiubzZkE8N49/xIQ/KGQwDRA8PaviF8IRO8JMWMngdVVQ==", "dev": true, + "license": "MIT", "dependencies": { - "tinyrainbow": "^1.2.0" + "tinyrainbow": "^2.0.0" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/runner": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.1.8.tgz", - "integrity": "sha512-17ub8vQstRnRlIU5k50bG+QOMLHRhYPAna5tw8tYbj+jzjcspnwnwtPtiOlkuKC4+ixDPTuLZiqiWWQ2PSXHVg==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.0.2.tgz", + "integrity": "sha512-GHEsWoncrGxWuW8s405fVoDfSLk6RF2LCXp6XhevbtDjdDme1WV/eNmUueDfpY1IX3MJaCRelVCEXsT9cArfEg==", "dev": true, + "license": "MIT", "dependencies": { - "@vitest/utils": "2.1.8", - "pathe": "^1.1.2" + "@vitest/utils": "3.0.2", + "pathe": "^2.0.1" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/snapshot": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.1.8.tgz", - "integrity": "sha512-20T7xRFbmnkfcmgVEz+z3AU/3b0cEzZOt/zmnvZEctg64/QZbSDJEVm9fLnnlSi74KibmRsO9/Qabi+t0vCRPg==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.0.2.tgz", + "integrity": "sha512-h9s67yD4+g+JoYG0zPCo/cLTabpDqzqNdzMawmNPzDStTiwxwkyYM1v5lWE8gmGv3SVJ2DcxA2NpQJZJv9ym3g==", "dev": true, + "license": "MIT", "dependencies": { - "@vitest/pretty-format": "2.1.8", - "magic-string": "^0.30.12", - "pathe": "^1.1.2" + "@vitest/pretty-format": "3.0.2", + "magic-string": "^0.30.17", + "pathe": "^2.0.1" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/spy": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.1.8.tgz", - "integrity": "sha512-5swjf2q95gXeYPevtW0BLk6H8+bPlMb4Vw/9Em4hFxDcaOxS+e0LOX4yqNxoHzMR2akEB2xfpnWUzkZokmgWDg==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.0.2.tgz", + "integrity": "sha512-8mI2iUn+PJFMT44e3ISA1R+K6ALVs47W6eriDTfXe6lFqlflID05MB4+rIFhmDSLBj8iBsZkzBYlgSkinxLzSQ==", "dev": true, + "license": "MIT", "dependencies": { "tinyspy": "^3.0.2" }, @@ -2652,24 +3108,25 @@ } }, "node_modules/@vitest/ui": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/ui/-/ui-2.1.8.tgz", - "integrity": "sha512-5zPJ1fs0ixSVSs5+5V2XJjXLmNzjugHRyV11RqxYVR+oMcogZ9qTuSfKW+OcTV0JeFNznI83BNylzH6SSNJ1+w==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@vitest/ui/-/ui-3.0.2.tgz", + "integrity": "sha512-R0E4nG0OAafsCKwKnENLdjpMbxAyDqT/hdbJp71eeAR1wE+C7IFv1G158sRj5gUfJ7pM7IxtcwIqa34beYzLhg==", "dev": true, + "license": "MIT", "dependencies": { - "@vitest/utils": "2.1.8", + "@vitest/utils": "3.0.2", "fflate": "^0.8.2", - "flatted": "^3.3.1", - "pathe": "^1.1.2", + "flatted": "^3.3.2", + "pathe": "^2.0.1", "sirv": "^3.0.0", "tinyglobby": "^0.2.10", - "tinyrainbow": "^1.2.0" + "tinyrainbow": "^2.0.0" }, "funding": { "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "vitest": "2.1.8" + "vitest": "3.0.2" } }, "node_modules/@vitest/ui/node_modules/sirv": { @@ -2677,6 +3134,7 @@ "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.0.tgz", "integrity": "sha512-BPwJGUeDaDCHihkORDchNyyTvWFhcusy1XMmhEVTQTwGeybFbp8YEmB+njbPnth1FibULBSBVwCQni25XlCUDg==", "dev": true, + "license": "MIT", "dependencies": { "@polka/url": "^1.0.0-next.24", "mrmime": "^2.0.0", @@ -2687,17 +3145,34 @@ } }, "node_modules/@vitest/utils": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.1.8.tgz", - "integrity": "sha512-dwSoui6djdwbfFmIgbIjX2ZhIoG7Ex/+xpxyiEgIGzjliY8xGkcpITKTlp6B4MgtGkF2ilvm97cPM96XZaAgcA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.0.2.tgz", + "integrity": "sha512-Qu01ZYZlgHvDP02JnMBRpX43nRaZtNpIzw3C1clDXmn8eakgX6iQVGzTQ/NjkIr64WD8ioqOjkaYRVvHQI5qiw==", "dev": true, + "license": "MIT", "dependencies": { - "@vitest/pretty-format": "2.1.8", + "@vitest/pretty-format": "3.0.2", "loupe": "^3.1.2", - "tinyrainbow": "^1.2.0" + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/web-worker": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@vitest/web-worker/-/web-worker-3.0.2.tgz", + "integrity": "sha512-38hmHAp2tB7H9VPUzFOb4rKFStjE0lRCE8k4ljunTDfkvndi68/6XK8UKADAikwDKcmEdsXldJIJxAFhr9RfVw==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.0" }, "funding": { "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "vitest": "3.0.2" } }, "node_modules/@wdio/config": { @@ -2705,6 +3180,8 @@ "resolved": "https://registry.npmjs.org/@wdio/config/-/config-9.2.8.tgz", "integrity": "sha512-EGMmBPGJbz6RmgMjebRWkWu3fGyeTIRcusF4UA4f2tiUEKY8nbzUO/ZyDjVQNR+YVB40q0jcqAqpszYRrIzzeg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "9.1.3", "@wdio/types": "9.2.2", @@ -2723,6 +3200,8 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", @@ -2743,6 +3222,8 @@ "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "@isaacs/cliui": "^8.0.2" }, @@ -2757,13 +3238,17 @@ "version": "10.4.3", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/@wdio/config/node_modules/path-scurry": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "lru-cache": "^10.2.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" @@ -2780,6 +3265,8 @@ "resolved": "https://registry.npmjs.org/@wdio/logger/-/logger-9.1.3.tgz", "integrity": "sha512-cumRMK/gE1uedBUw3WmWXOQ7HtB6DR8EyKQioUz2P0IJtRRpglMBdZV7Svr3b++WWawOuzZHMfbTkJQmaVt8Gw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "chalk": "^5.1.2", "loglevel": "^1.6.0", @@ -2795,6 +3282,8 @@ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -2807,6 +3296,8 @@ "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": "^12.17.0 || ^14.13 || >=16.0.0" }, @@ -2819,6 +3310,8 @@ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "ansi-regex": "^6.0.1" }, @@ -2833,13 +3326,17 @@ "version": "9.2.2", "resolved": "https://registry.npmjs.org/@wdio/protocols/-/protocols-9.2.2.tgz", "integrity": "sha512-0GMUSHCbYm+J+rnRU6XPtaUgVCRICsiH6W5zCXpePm3wLlbmg/mvZ+4OnNErssbpIOulZuAmC2jNmut2AEfWSw==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/@wdio/repl": { "version": "9.0.8", "resolved": "https://registry.npmjs.org/@wdio/repl/-/repl-9.0.8.tgz", "integrity": "sha512-3iubjl4JX5zD21aFxZwQghqC3lgu+mSs8c3NaiYYNCC+IT5cI/8QuKlgh9s59bu+N3gG988jqMJeCYlKuUv/iw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0" }, @@ -2852,6 +3349,8 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.10.tgz", "integrity": "sha512-/jrvh5h6NXhEauFFexRin69nA0uHJ5gwk4iDivp/DeoEua3uwCUto6PC86IpRITBOs4+6i2I56K5x5b6WYGXHA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "undici-types": "~6.19.2" } @@ -2860,13 +3359,17 @@ "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/@wdio/types": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/@wdio/types/-/types-9.2.2.tgz", "integrity": "sha512-nHZ9Ne9iRQFJ1TOYKUn4Fza69IshTTzk6RYmSZ51ImGs9uMZu0+S0Jm9REdly+VLN3FzxG6g2QSe0/F3uNVPdw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0" }, @@ -2879,6 +3382,8 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.10.tgz", "integrity": "sha512-/jrvh5h6NXhEauFFexRin69nA0uHJ5gwk4iDivp/DeoEua3uwCUto6PC86IpRITBOs4+6i2I56K5x5b6WYGXHA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "undici-types": "~6.19.2" } @@ -2887,13 +3392,17 @@ "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/@wdio/utils": { "version": "9.2.8", "resolved": "https://registry.npmjs.org/@wdio/utils/-/utils-9.2.8.tgz", "integrity": "sha512-rKm5FXkpsCyeqh8tdirtRUHvgNytWNMiaVKdctsvKOJvqnDVPAAQcz9Wmgo7bSwoLwtSHcDaRoxY7olV7J4QnA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "@puppeteer/browsers": "^2.2.0", "@wdio/logger": "9.1.3", @@ -3110,6 +3619,8 @@ "resolved": "https://registry.npmjs.org/@zip.js/zip.js/-/zip.js-2.7.54.tgz", "integrity": "sha512-qMrJVg2hoEsZJjMJez9yI2+nZlBUxgYzGV3mqcb2B/6T1ihXp0fWBDYlVHlHquuorgNUQP5a8qSmX6HF5rFJNg==", "dev": true, + "optional": true, + "peer": true, "engines": { "bun": ">=0.7.0", "deno": ">=1.0.0", @@ -3141,6 +3652,8 @@ "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "event-target-shim": "^5.0.0" }, @@ -3363,6 +3876,8 @@ "resolved": "https://registry.npmjs.org/archiver/-/archiver-7.0.1.tgz", "integrity": "sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "archiver-utils": "^5.0.2", "async": "^3.2.4", @@ -3381,6 +3896,8 @@ "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-5.0.2.tgz", "integrity": "sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "glob": "^10.0.0", "graceful-fs": "^4.2.0", @@ -3399,6 +3916,8 @@ "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.8.x" } @@ -3408,6 +3927,8 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", @@ -3428,6 +3949,8 @@ "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "@isaacs/cliui": "^8.0.2" }, @@ -3442,13 +3965,17 @@ "version": "10.4.3", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/archiver-utils/node_modules/path-scurry": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "lru-cache": "^10.2.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" @@ -3465,6 +3992,8 @@ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", @@ -3481,6 +4010,8 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "safe-buffer": "~5.2.0" } @@ -3490,6 +4021,8 @@ "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.8.x" } @@ -3499,6 +4032,8 @@ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", @@ -3515,6 +4050,8 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "safe-buffer": "~5.2.0" } @@ -3689,6 +4226,7 @@ "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", "dev": true, + "license": "MIT", "engines": { "node": ">=12" } @@ -3698,6 +4236,8 @@ "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "tslib": "^2.0.1" }, @@ -3745,15 +4285,17 @@ } }, "node_modules/aws4": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz", - "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==" + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.2.tgz", + "integrity": "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==" }, "node_modules/b4a": { "version": "1.6.6", "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz", "integrity": "sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/balanced-match": { "version": "1.0.2", @@ -3765,7 +4307,8 @@ "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.4.2.tgz", "integrity": "sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==", "dev": true, - "optional": true + "optional": true, + "peer": true }, "node_modules/bare-fs": { "version": "2.3.5", @@ -3773,6 +4316,7 @@ "integrity": "sha512-SlE9eTxifPDJrT6YgemQ1WGFleevzwY+XAP1Xqgl56HtcrisC2CHCZ2tq6dBpcH2TnNxwUEUGhweo+lrQtYuiw==", "dev": true, "optional": true, + "peer": true, "dependencies": { "bare-events": "^2.0.0", "bare-path": "^2.0.0", @@ -3784,7 +4328,8 @@ "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.4.4.tgz", "integrity": "sha512-z3UiI2yi1mK0sXeRdc4O1Kk8aOa/e+FNWZcTiPB/dfTWyLypuE99LibgRaQki914Jq//yAWylcAt+mknKdixRQ==", "dev": true, - "optional": true + "optional": true, + "peer": true }, "node_modules/bare-path": { "version": "2.1.3", @@ -3792,6 +4337,7 @@ "integrity": "sha512-lh/eITfU8hrj9Ru5quUp0Io1kJWIk1bTjzo7JH1P5dWmQ2EL4hFUlfI8FonAhSlgIfhn63p84CDY/x+PisgcXA==", "dev": true, "optional": true, + "peer": true, "dependencies": { "bare-os": "^2.1.0" } @@ -3802,6 +4348,7 @@ "integrity": "sha512-QK6bePvszntxgPKdVXciYzjlWv2Ry1mQuUqyUUzd27G7eLupl6d0K5AGJfnfyFAdgy5tRolHP/zbaUMslLceOg==", "dev": true, "optional": true, + "peer": true, "dependencies": { "streamx": "^2.21.0" } @@ -3824,13 +4371,35 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "optional": true, + "peer": true + }, + "node_modules/basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "dev": true, + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/basic-auth/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true }, "node_modules/basic-ftp": { "version": "5.0.5", "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=10.0.0" } @@ -3935,7 +4504,9 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/brace-expansion": { "version": "2.0.1", @@ -4015,6 +4586,8 @@ "url": "https://feross.org/support" } ], + "optional": true, + "peer": true, "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" @@ -4025,6 +4598,8 @@ "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=8.0.0" } @@ -4052,6 +4627,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/busboy": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.3.1.tgz", + "integrity": "sha512-y7tTxhGKXcyBxRKAni+awqx8uqaJKrSFSNFSeRG5CsWNdmy2BIK+6VGWEW7TZnIO/533mtMEA4rOevQV815YJw==", + "dev": true, + "dependencies": { + "dicer": "0.3.0" + }, + "engines": { + "node": ">=4.5.0" + } + }, "node_modules/bytes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", @@ -4066,10 +4653,24 @@ "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } }, + "node_modules/cache-content-type": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/cache-content-type/-/cache-content-type-1.0.1.tgz", + "integrity": "sha512-IKufZ1o4Ut42YUrZSo8+qnMTrFuKkvyoLXUywKz9GJ5BrhOFGhLdkx9sG4KAnVvbY6kEcSFjLQul+DVmBm2bgA==", + "dev": true, + "dependencies": { + "mime-types": "^2.1.18", + "ylru": "^1.2.0" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/call-bind": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", @@ -4156,6 +4757,7 @@ "resolved": "https://registry.npmjs.org/chai/-/chai-5.1.2.tgz", "integrity": "sha512-aGtmf24DW6MLHHG5gCx4zaI3uBq3KRtxeVs0DjFH6Z0rDNbsvTxFASFvdj79pxjxZ8/5u3PIiN3IwEIQkiiuPw==", "dev": true, + "license": "MIT", "dependencies": { "assertion-error": "^2.0.1", "check-error": "^2.1.1", @@ -4188,6 +4790,7 @@ "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.1.tgz", "integrity": "sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==", "dev": true, + "license": "MIT", "engines": { "node": ">= 16" } @@ -4197,6 +4800,8 @@ "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz", "integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", @@ -4222,6 +4827,8 @@ "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", @@ -4239,6 +4846,8 @@ "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", @@ -4255,6 +4864,8 @@ "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", @@ -4269,6 +4880,8 @@ "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "domelementtype": "^2.3.0" }, @@ -4284,7 +4897,9 @@ "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", "dev": true, - "dependencies": { + "optional": true, + "peer": true, + "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" @@ -4293,23 +4908,13 @@ "url": "https://github.com/fb55/domutils?sponsor=1" } }, - "node_modules/cheerio-select/node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "dev": true, - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, "node_modules/cheerio/node_modules/dom-serializer": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", @@ -4324,6 +4929,8 @@ "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "domelementtype": "^2.3.0" }, @@ -4339,6 +4946,8 @@ "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", @@ -4348,18 +4957,6 @@ "url": "https://github.com/fb55/domutils?sponsor=1" } }, - "node_modules/cheerio/node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "dev": true, - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, "node_modules/cheerio/node_modules/htmlparser2": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", @@ -4372,6 +4969,8 @@ "url": "https://github.com/sponsors/fb55" } ], + "optional": true, + "peer": true, "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", @@ -4379,27 +4978,6 @@ "entities": "^4.5.0" } }, - "node_modules/cheerio/node_modules/parse5": { - "version": "7.2.1", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz", - "integrity": "sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==", - "dev": true, - "dependencies": { - "entities": "^4.5.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/cheerio/node_modules/whatwg-mimetype": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", - "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", - "dev": true, - "engines": { - "node": ">=18" - } - }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -4545,11 +5123,31 @@ "node": ">=0.10.0" } }, + "node_modules/co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", + "dev": true, + "engines": { + "iojs": ">= 1.0.0", + "node": ">= 0.12.0" + } + }, "node_modules/codemirror": { "version": "5.65.16", "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-5.65.16.tgz", "integrity": "sha512-br21LjYmSlVL0vFCPWPfhzUCT34FM/pAdK7rRIZwa0rrtrIdotvP4Oh4GUHsu2E3IrQMCfRkL/fN3ytMNxVQvg==" }, + "node_modules/color": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", + "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.3", + "color-string": "^1.6.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -4566,12 +5164,47 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dev": true, + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/color/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, "node_modules/colorette": { "version": "2.0.20", "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", "dev": true }, + "node_modules/colorspace": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", + "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", + "dev": true, + "dependencies": { + "color": "^3.1.3", + "text-hex": "1.0.x" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -4588,6 +5221,8 @@ "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || >=14" } @@ -4597,6 +5232,8 @@ "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-6.0.2.tgz", "integrity": "sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "crc-32": "^1.2.0", "crc32-stream": "^6.0.0", @@ -4613,6 +5250,8 @@ "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.8.x" } @@ -4622,6 +5261,8 @@ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", @@ -4638,6 +5279,8 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "safe-buffer": "~5.2.0" } @@ -4757,12 +5400,13 @@ } }, "node_modules/cookie": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", - "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.0.2.tgz", + "integrity": "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==", "dev": true, + "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">=18" } }, "node_modules/cookie-signature": { @@ -4771,11 +5415,25 @@ "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", "dev": true }, + "node_modules/cookies": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/cookies/-/cookies-0.9.1.tgz", + "integrity": "sha512-TG2hpqe4ELx54QER/S3HQ9SRVnQnGBtKUz5bLQWtYAQ+o6GpgMs6sYUvaiJjVxb+UXwhRhAEP3m7LbsIZ77Hmw==", + "dev": true, + "dependencies": { + "depd": "~2.0.0", + "keygrip": "~1.1.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/core-js": { - "version": "3.39.0", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.39.0.tgz", - "integrity": "sha512-raM0ew0/jJUqkJ0E6e8UDtl+y/7ktFivgWvqw8dNSQeNWoSDLvQ1H/RN3aPXB9tBd4/FhyR4RDPGhsNIMsAn7g==", + "version": "3.40.0", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.40.0.tgz", + "integrity": "sha512-7vsMc/Lty6AGnn7uFpYT56QesI5D2Y/UkgKounk87OP9Z2H9Z8kj6jzcSGAxFmUtDOS0ntK6lbQz+Nsa0Jj6mQ==", "hasInstallScript": true, + "license": "MIT", "funding": { "type": "opencollective", "url": "https://opencollective.com/core-js" @@ -4787,11 +5445,20 @@ "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", "dev": true }, + "node_modules/corser": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/corser/-/corser-2.0.1.tgz", + "integrity": "sha512-utCYNzRSQIZNPIcGZdQc92UVJYAhtGAteCFg0yRaFm8f0P+CPtyGyHXJcGXnffjCybUCEx3FQ2G7U3/o9eIkVQ==", + "dev": true, + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/crc-32": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", - "dev": true, + "license": "Apache-2.0", "bin": { "crc32": "bin/crc32.njs" }, @@ -4804,6 +5471,8 @@ "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-6.0.0.tgz", "integrity": "sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "crc-32": "^1.2.0", "readable-stream": "^4.0.0" @@ -4817,6 +5486,8 @@ "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.8.x" } @@ -4826,6 +5497,8 @@ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", @@ -4842,6 +5515,8 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "safe-buffer": "~5.2.0" } @@ -4898,19 +5573,25 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/css-shorthand-properties/-/css-shorthand-properties-1.1.1.tgz", "integrity": "sha512-Md+Juc7M3uOdbAFwOYlTrccIZ7oCFuzrhKYQjdeUEW/sE1hv17Jp/Bws+ReOPpGVBTYCBoYo+G17V5Qo8QQ75A==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/css-value": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/css-value/-/css-value-0.0.1.tgz", "integrity": "sha512-FUV3xaJ63buRLgHrLQVlVgQnQdR4yqdLGaDu7g8CQcWjInDfM9plBTPI9FRfpahju1UBSaMckeb2/46ApS/V1Q==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/css-what": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 6" }, @@ -4936,11 +5617,17 @@ "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" }, "node_modules/cssstyle": { - "version": "0.2.37", - "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-0.2.37.tgz", - "integrity": "sha512-FUpKc+1FNBsHUr9IsfSGCovr8VuGOiiuzlgCyppKBjJi2jYTOFLN3oiiNRMIvYqbFzF38mqKj4BgcevzU5/kIA==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.2.1.tgz", + "integrity": "sha512-9+vem03dMXG7gDmZ62uqmRiMRNtinIZ9ZyuF6BdxzfOD+FdN5hretzynkn0ReS2DO2GSw76RWHs0UmJPI2zUjw==", + "dev": true, + "license": "MIT", "dependencies": { - "cssom": "0.3.x" + "@asamuzakjp/css-color": "^2.8.2", + "rrweb-cssom": "^0.8.0" + }, + "engines": { + "node": ">=18" } }, "node_modules/dashdash": { @@ -4959,34 +5646,23 @@ "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 12" } }, "node_modules/data-urls": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz", - "integrity": "sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ==", - "dependencies": { - "abab": "^2.0.0", - "whatwg-mimetype": "^2.2.0", - "whatwg-url": "^7.0.0" - } - }, - "node_modules/data-urls/node_modules/abab": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", - "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", - "deprecated": "Use your platform's native atob() and btoa() methods instead" - }, - "node_modules/data-urls/node_modules/whatwg-url": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz", - "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", + "integrity": "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==", + "dev": true, "dependencies": { - "lodash.sortby": "^4.7.0", - "tr46": "^1.0.1", - "webidl-conversions": "^4.0.2" + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.0.0" + }, + "engines": { + "node": ">=18" } }, "node_modules/data-view-buffer": { @@ -5068,6 +5744,8 @@ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-6.0.0.tgz", "integrity": "sha512-Fv96DCsdOgB6mdGl67MT5JaTNKRzrzill5OH5s8bjYJXVlcXyPYGyPsUkWyGV5p1TXI5esYIYMMeDJL0hEIwaA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -5075,15 +5753,28 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/decimal.js": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz", + "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==", + "dev": true + }, "node_modules/deep-eql": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } }, + "node_modules/deep-equal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", + "integrity": "sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==", + "dev": true + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -5094,6 +5785,8 @@ "resolved": "https://registry.npmjs.org/deepmerge-ts/-/deepmerge-ts-7.1.3.tgz", "integrity": "sha512-qCSH6I0INPxd9Y1VtAiLpnYvz5O//6rCfJXKk0z66Up9/VOSr+1yS8XSKA5IWRxjocFGlzPyaZYe+jxq7OOLtQ==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=16.0.0" } @@ -5189,6 +5882,8 @@ "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "ast-types": "^0.13.4", "escodegen": "^2.1.0", @@ -5206,6 +5901,12 @@ "node": ">=0.4.0" } }, + "node_modules/delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", + "dev": true + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -5240,6 +5941,18 @@ "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", "dev": true }, + "node_modules/dicer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.3.0.tgz", + "integrity": "sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA==", + "dev": true, + "dependencies": { + "streamsearch": "0.1.2" + }, + "engines": { + "node": ">=4.5.0" + } + }, "node_modules/dns-packet": { "version": "5.6.1", "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.1.tgz", @@ -5256,7 +5969,8 @@ "version": "0.5.16", "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/domelementtype": { "version": "2.3.0", @@ -5268,7 +5982,9 @@ "type": "github", "url": "https://github.com/sponsors/fb55" } - ] + ], + "optional": true, + "peer": true }, "node_modules/domexception": { "version": "1.0.1", @@ -5279,6 +5995,11 @@ "webidl-conversions": "^4.0.2" } }, + "node_modules/domexception/node_modules/webidl-conversions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", + "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==" + }, "node_modules/dunder-proto": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.0.tgz", @@ -5318,6 +6039,8 @@ "resolved": "https://registry.npmjs.org/edge-paths/-/edge-paths-3.0.5.tgz", "integrity": "sha512-sB7vSrDnFa4ezWQk9nZ/n0FdpdUuC6R1EOrlU3DL+bovcNFK28rqu2emmAUjujYEJTWIgQGqgVVWUZXMnc8iWg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "@types/which": "^2.0.1", "which": "^2.0.2" @@ -5335,6 +6058,8 @@ "integrity": "sha512-3Ve9cd5ziLByUdigw6zovVeWJjVs8QHVmqOB0sJ0WNeVPcwf4p18GnxMmVvlFmYRloUwf5suNuorea4QzwBIOA==", "dev": true, "hasInstallScript": true, + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "^8.38.0", "@zip.js/zip.js": "^2.7.48", @@ -5353,6 +6078,8 @@ "resolved": "https://registry.npmjs.org/@wdio/logger/-/logger-8.38.0.tgz", "integrity": "sha512-kcHL86RmNbcQP+Gq/vQUGlArfU6IIcbbnNp32rRIraitomZow+iEoc519rdQmSVusDozMS5DZthkgDdxK+vz6Q==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "chalk": "^5.1.2", "loglevel": "^1.6.0", @@ -5368,6 +6095,8 @@ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -5380,6 +6109,8 @@ "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": "^12.17.0 || ^14.13 || >=16.0.0" }, @@ -5392,6 +6123,8 @@ "resolved": "https://registry.npmjs.org/isexe/-/isexe-3.1.1.tgz", "integrity": "sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=16" } @@ -5401,6 +6134,8 @@ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "ansi-regex": "^6.0.1" }, @@ -5416,6 +6151,8 @@ "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz", "integrity": "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "isexe": "^3.1.1" }, @@ -5484,6 +6221,12 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, + "node_modules/enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==", + "dev": true + }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", @@ -5498,6 +6241,8 @@ "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.0.tgz", "integrity": "sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "iconv-lite": "^0.6.3", "whatwg-encoding": "^3.1.1" @@ -5511,6 +6256,8 @@ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" }, @@ -5518,23 +6265,13 @@ "node": ">=0.10.0" } }, - "node_modules/encoding-sniffer/node_modules/whatwg-encoding": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", - "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", - "dev": true, - "dependencies": { - "iconv-lite": "0.6.3" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "once": "^1.4.0" } @@ -5552,6 +6289,18 @@ "node": ">=10.13.0" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "dev": true, + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/es-abstract": { "version": "1.23.5", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.23.5.tgz", @@ -5631,10 +6380,11 @@ } }, "node_modules/es-module-lexer": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz", - "integrity": "sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw==", - "dev": true + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.6.0.tgz", + "integrity": "sha512-qqnD1yMU6tk/jnaMosogGySTZP8YtUgAffA9nMN+E/rjxcfRQ6IEk7IiozUjgxKoFHBGjTLnrHB/YC45r/59EQ==", + "dev": true, + "license": "MIT" }, "node_modules/es-object-atoms": { "version": "1.0.0", @@ -5689,11 +6439,12 @@ } }, "node_modules/esbuild": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.24.0.tgz", - "integrity": "sha512-FuLPevChGDshgSicjisSooU0cemp/sGXR841D5LHMB7mTVOmsEHcAxaH3irL53+8YDIeVNQEySh4DaYU/iuPqQ==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.24.2.tgz", + "integrity": "sha512-+9egpBW8I3CD5XPe0n6BfT5fxLzxrlDzqydF3aviG+9ni1lDC/OvMHcxqEFV0+LANZG5R1bFMWfUrjVsdwxJvA==", "dev": true, "hasInstallScript": true, + "license": "MIT", "bin": { "esbuild": "bin/esbuild" }, @@ -5701,30 +6452,31 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.24.0", - "@esbuild/android-arm": "0.24.0", - "@esbuild/android-arm64": "0.24.0", - "@esbuild/android-x64": "0.24.0", - "@esbuild/darwin-arm64": "0.24.0", - "@esbuild/darwin-x64": "0.24.0", - "@esbuild/freebsd-arm64": "0.24.0", - "@esbuild/freebsd-x64": "0.24.0", - "@esbuild/linux-arm": "0.24.0", - "@esbuild/linux-arm64": "0.24.0", - "@esbuild/linux-ia32": "0.24.0", - "@esbuild/linux-loong64": "0.24.0", - "@esbuild/linux-mips64el": "0.24.0", - "@esbuild/linux-ppc64": "0.24.0", - "@esbuild/linux-riscv64": "0.24.0", - "@esbuild/linux-s390x": "0.24.0", - "@esbuild/linux-x64": "0.24.0", - "@esbuild/netbsd-x64": "0.24.0", - "@esbuild/openbsd-arm64": "0.24.0", - "@esbuild/openbsd-x64": "0.24.0", - "@esbuild/sunos-x64": "0.24.0", - "@esbuild/win32-arm64": "0.24.0", - "@esbuild/win32-ia32": "0.24.0", - "@esbuild/win32-x64": "0.24.0" + "@esbuild/aix-ppc64": "0.24.2", + "@esbuild/android-arm": "0.24.2", + "@esbuild/android-arm64": "0.24.2", + "@esbuild/android-x64": "0.24.2", + "@esbuild/darwin-arm64": "0.24.2", + "@esbuild/darwin-x64": "0.24.2", + "@esbuild/freebsd-arm64": "0.24.2", + "@esbuild/freebsd-x64": "0.24.2", + "@esbuild/linux-arm": "0.24.2", + "@esbuild/linux-arm64": "0.24.2", + "@esbuild/linux-ia32": "0.24.2", + "@esbuild/linux-loong64": "0.24.2", + "@esbuild/linux-mips64el": "0.24.2", + "@esbuild/linux-ppc64": "0.24.2", + "@esbuild/linux-riscv64": "0.24.2", + "@esbuild/linux-s390x": "0.24.2", + "@esbuild/linux-x64": "0.24.2", + "@esbuild/netbsd-arm64": "0.24.2", + "@esbuild/netbsd-x64": "0.24.2", + "@esbuild/openbsd-arm64": "0.24.2", + "@esbuild/openbsd-x64": "0.24.2", + "@esbuild/sunos-x64": "0.24.2", + "@esbuild/win32-arm64": "0.24.2", + "@esbuild/win32-ia32": "0.24.2", + "@esbuild/win32-x64": "0.24.2" } }, "node_modules/escalade": { @@ -5759,6 +6511,8 @@ "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "esprima": "^4.0.1", "estraverse": "^5.2.0", @@ -5776,18 +6530,19 @@ } }, "node_modules/eslint": { - "version": "9.16.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.16.0.tgz", - "integrity": "sha512-whp8mSQI4C8VXd+fLgSM0lh3UlmcFtVwUQjyKCFfsp+2ItAIYhlq/hqGahGqHE6cv9unM41VlqKk2VtKYR2TaA==", + "version": "9.18.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.18.0.tgz", + "integrity": "sha512-+waTfRWQlSbpt3KWE+CjrPPYnbq9kfZIYUqapc0uBXyjTp8aYXZDsUH16m39Ryq3NjAVP4tjuF7KaukeqoCoaA==", "dev": true, + "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.12.1", "@eslint/config-array": "^0.19.0", - "@eslint/core": "^0.9.0", + "@eslint/core": "^0.10.0", "@eslint/eslintrc": "^3.2.0", - "@eslint/js": "9.16.0", - "@eslint/plugin-kit": "^0.2.3", + "@eslint/js": "9.18.0", + "@eslint/plugin-kit": "^0.2.5", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.1", @@ -5795,7 +6550,7 @@ "@types/json-schema": "^7.0.15", "ajv": "^6.12.4", "chalk": "^4.0.0", - "cross-spawn": "^7.0.5", + "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", "eslint-scope": "^8.2.0", @@ -6247,6 +7002,7 @@ "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", "dev": true, + "license": "MIT", "dependencies": { "@types/estree": "^1.0.0" } @@ -6273,7 +7029,9 @@ "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", "dev": true, - "engines": { + "optional": true, + "peer": true, + "engines": { "node": ">=6" } }, @@ -6350,6 +7108,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", "dev": true, + "license": "MIT", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", @@ -6457,6 +7216,8 @@ "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "debug": "^4.1.1", "get-stream": "^5.1.0", @@ -6472,6 +7233,31 @@ "@types/yauzl": "^2.9.1" } }, + "node_modules/extract-zip/node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": "*" + } + }, + "node_modules/extract-zip/node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, "node_modules/extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", @@ -6489,7 +7275,9 @@ "version": "1.3.2", "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/fast-glob": { "version": "3.3.2", @@ -6550,6 +7338,8 @@ "url": "https://paypal.me/naturalintelligence" } ], + "optional": true, + "peer": true, "dependencies": { "strnum": "^1.0.5" }, @@ -6583,10 +7373,19 @@ "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", "dev": true, + "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "pend": "~1.2.0" } }, + "node_modules/fecha": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", + "dev": true + }, "node_modules/fetch-blob": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", @@ -6602,6 +7401,8 @@ "url": "https://paypal.me/jimmywarting" } ], + "optional": true, + "peer": true, "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" @@ -6711,9 +7512,16 @@ } }, "node_modules/flatted": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", - "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.2.tgz", + "integrity": "sha512-AiwGJM8YcNOaobumgtng+6NHuOqC3A7MixFeDafM3X9cIUM+xUXoS5Vfgf+OihAYe20fxqNM9yPBXJzRtZ/4eA==", + "dev": true, + "license": "ISC" + }, + "node_modules/fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==", "dev": true }, "node_modules/follow-redirects": { @@ -6768,11 +7576,27 @@ "node": "*" } }, + "node_modules/form-data": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "fetch-blob": "^3.1.2" }, @@ -6798,6 +7622,29 @@ "node": ">= 0.6" } }, + "node_modules/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/fs-extra/node_modules/universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "dev": true, + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -6854,6 +7701,8 @@ "integrity": "sha512-lGCRqPMuzbRNDWJOQcUqhNqPvNsIFu6yzXF8J/6K3WCYFd2r5ckbeF7h1cxsnjA7YLSEiWzERCt6/gjZ3tW0ug==", "dev": true, "hasInstallScript": true, + "optional": true, + "peer": true, "dependencies": { "@wdio/logger": "^9.0.0", "@zip.js/zip.js": "^2.7.48", @@ -6876,6 +7725,8 @@ "resolved": "https://registry.npmjs.org/isexe/-/isexe-3.1.1.tgz", "integrity": "sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=16" } @@ -6885,6 +7736,8 @@ "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz", "integrity": "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "isexe": "^3.1.1" }, @@ -6931,6 +7784,8 @@ "resolved": "https://registry.npmjs.org/get-port/-/get-port-7.1.0.tgz", "integrity": "sha512-QB9NKEeDg3xxVwCCwJQ9+xycaz6pBB6iQ76wiWMl1927n0Kir6alPiP+yuiICLLU4jpMe08dXfpebuQppFA2zw==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=16" }, @@ -6943,6 +7798,8 @@ "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "pump": "^3.0.0" }, @@ -6987,6 +7844,8 @@ "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.4.tgz", "integrity": "sha512-E1b1lFFLvLgak2whF2xDBcOy6NLVGZBqqjJjsIhvopKfWWEi64pLVTWWehV8KlLerZkfNTA95sTe2OdJKm1OzQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "basic-ftp": "^5.0.2", "data-uri-to-buffer": "^6.0.2", @@ -7001,6 +7860,8 @@ "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 14" } @@ -7019,10 +7880,11 @@ "integrity": "sha512-526NA+3EA+ztAQi0IZpSWiM0fyQXIp7IbRvfJ4wS/TjjQD0uv0fVybXwwqqSOlq33UckivI0yMDlVtboWm3k7A==" }, "node_modules/glob": { - "version": "11.0.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.0.tgz", - "integrity": "sha512-9UiX/Bl6J2yaBbxKoEBRm4Cipxgok8kQYcOPEhScPwebu2I0HoQOuYdIO6S3hLuWoZgpDpwQZMzTFxgpkyT76g==", + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.1.tgz", + "integrity": "sha512-zrQDm8XPnYEKawJScsnM0QzobJxlT/kHOOlRTio8IH/GrmxRE5fjllkzdaHclIuNjUQTJYH2xHNIGfdpJkDJUw==", "dev": true, + "license": "ISC", "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^4.0.1", @@ -7126,13 +7988,16 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz", "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/graphemer": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/graphql": { "version": "16.9.0", @@ -7269,6 +8134,15 @@ "node": ">= 0.4" } }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true, + "bin": { + "he": "bin/he" + } + }, "node_modules/headers-polyfill": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/headers-polyfill/-/headers-polyfill-4.0.3.tgz", @@ -7288,11 +8162,15 @@ } }, "node_modules/html-encoding-sniffer": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz", - "integrity": "sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz", + "integrity": "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==", + "dev": true, "dependencies": { - "whatwg-encoding": "^1.0.1" + "whatwg-encoding": "^3.1.1" + }, + "engines": { + "node": ">=18" } }, "node_modules/html-entities": { @@ -7321,7 +8199,56 @@ "version": "0.3.2", "resolved": "https://registry.npmjs.org/htmlfy/-/htmlfy-0.3.2.tgz", "integrity": "sha512-FsxzfpeDYRqn1emox9VpxMPfGjADoUmmup8D604q497R0VNxiXs4ZZTN2QzkaMA5C9aHGUoe1iQRVSm+HK9xuA==", - "dev": true + "dev": true, + "optional": true, + "peer": true + }, + "node_modules/http-assert": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/http-assert/-/http-assert-1.5.0.tgz", + "integrity": "sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==", + "dev": true, + "dependencies": { + "deep-equal": "~1.0.1", + "http-errors": "~1.8.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/http-assert/node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/http-assert/node_modules/http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "dev": true, + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/http-assert/node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "dev": true, + "engines": { + "node": ">= 0.6" + } }, "node_modules/http-deceiver": { "version": "1.2.7", @@ -7414,6 +8341,69 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/http-server": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/http-server/-/http-server-14.1.1.tgz", + "integrity": "sha512-+cbxadF40UXd9T01zUHgA+rlo2Bg1Srer4+B4NwIHdaGxAGGv59nYRnGGDJ9LBk7alpS0US+J+bLLdQOOkJq4A==", + "dev": true, + "dependencies": { + "basic-auth": "^2.0.1", + "chalk": "^4.1.2", + "corser": "^2.0.1", + "he": "^1.2.0", + "html-encoding-sniffer": "^3.0.0", + "http-proxy": "^1.18.1", + "mime": "^1.6.0", + "minimist": "^1.2.6", + "opener": "^1.5.1", + "portfinder": "^1.0.28", + "secure-compare": "3.0.1", + "union": "~0.5.0", + "url-join": "^4.0.1" + }, + "bin": { + "http-server": "bin/http-server" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/http-server/node_modules/html-encoding-sniffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", + "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", + "dev": true, + "dependencies": { + "whatwg-encoding": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/http-server/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/http-server/node_modules/whatwg-encoding": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", + "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==", + "dev": true, + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -7450,6 +8440,12 @@ "node": ">=10.17.0" } }, + "node_modules/humanize-number": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/humanize-number/-/humanize-number-0.0.2.tgz", + "integrity": "sha512-un3ZAcNQGI7RzaWGZzQDH47HETM4Wrj6z6E4TId8Yeq9w5ZKUVB1nrT2jwFheTUjEmqcgTjXDc959jum+ai1kQ==", + "dev": true + }, "node_modules/hyperdyperid": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/hyperdyperid/-/hyperdyperid-1.2.0.tgz", @@ -7500,7 +8496,9 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "optional": true, + "peer": true }, "node_modules/ignore": { "version": "5.3.1", @@ -7522,52 +8520,290 @@ "xmldom": "^0.1.27" } }, - "node_modules/immediate": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", - "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", - "dev": true - }, - "node_modules/import-fresh": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", - "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", - "dev": true, - "dependencies": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" + "node_modules/ikonate/node_modules/acorn": { + "version": "5.7.4", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz", + "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==", + "bin": { + "acorn": "bin/acorn" }, "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">=0.4.0" } }, - "node_modules/import-meta-resolve": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.1.0.tgz", - "integrity": "sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==", - "dev": true, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" + "node_modules/ikonate/node_modules/cssstyle": { + "version": "0.2.37", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-0.2.37.tgz", + "integrity": "sha512-FUpKc+1FNBsHUr9IsfSGCovr8VuGOiiuzlgCyppKBjJi2jYTOFLN3oiiNRMIvYqbFzF38mqKj4BgcevzU5/kIA==", + "dependencies": { + "cssom": "0.3.x" } }, - "node_modules/imurmurhash": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true, - "engines": { - "node": ">=0.8.19" + "node_modules/ikonate/node_modules/data-urls": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz", + "integrity": "sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ==", + "dependencies": { + "abab": "^2.0.0", + "whatwg-mimetype": "^2.2.0", + "whatwg-url": "^7.0.0" } }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "node_modules/ikonate/node_modules/data-urls/node_modules/abab": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", + "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", + "deprecated": "Use your platform's native atob() and btoa() methods instead" + }, + "node_modules/ikonate/node_modules/data-urls/node_modules/whatwg-url": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz", + "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==", + "dependencies": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, + "node_modules/ikonate/node_modules/escodegen": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz", + "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^4.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=4.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/ikonate/node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/ikonate/node_modules/html-encoding-sniffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz", + "integrity": "sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw==", + "dependencies": { + "whatwg-encoding": "^1.0.1" + } + }, + "node_modules/ikonate/node_modules/jsdom": { + "version": "11.10.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-11.10.0.tgz", + "integrity": "sha512-x5No5FpJgBg3j5aBwA8ka6eGuS5IxbC8FOkmyccKvObtFT0bDMict/LOxINZsZGZSfGdNomLZ/qRV9Bpq/GIBA==", + "dependencies": { + "abab": "^1.0.4", + "acorn": "^5.3.0", + "acorn-globals": "^4.1.0", + "array-equal": "^1.0.0", + "cssom": ">= 0.3.2 < 0.4.0", + "cssstyle": ">= 0.2.37 < 0.3.0", + "data-urls": "^1.0.0", + "domexception": "^1.0.0", + "escodegen": "^1.9.0", + "html-encoding-sniffer": "^1.0.2", + "left-pad": "^1.2.0", + "nwmatcher": "^1.4.3", + "parse5": "4.0.0", + "pn": "^1.1.0", + "request": "^2.83.0", + "request-promise-native": "^1.0.5", + "sax": "^1.2.4", + "symbol-tree": "^3.2.2", + "tough-cookie": "^2.3.3", + "w3c-hr-time": "^1.0.1", + "webidl-conversions": "^4.0.2", + "whatwg-encoding": "^1.0.3", + "whatwg-mimetype": "^2.1.0", + "whatwg-url": "^6.4.0", + "ws": "^4.0.0", + "xml-name-validator": "^3.0.0" + } + }, + "node_modules/ikonate/node_modules/levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", + "dependencies": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/ikonate/node_modules/optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "dependencies": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/ikonate/node_modules/parse5": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz", + "integrity": "sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA==" + }, + "node_modules/ikonate/node_modules/prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/ikonate/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/ikonate/node_modules/tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dependencies": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/ikonate/node_modules/tr46": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", + "integrity": "sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/ikonate/node_modules/type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", + "dependencies": { + "prelude-ls": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/ikonate/node_modules/webidl-conversions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", + "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==" + }, + "node_modules/ikonate/node_modules/whatwg-encoding": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", + "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "dependencies": { + "iconv-lite": "0.4.24" + } + }, + "node_modules/ikonate/node_modules/whatwg-mimetype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", + "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" + }, + "node_modules/ikonate/node_modules/whatwg-url": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz", + "integrity": "sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ==", + "dependencies": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, + "node_modules/ikonate/node_modules/ws": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-4.1.0.tgz", + "integrity": "sha512-ZGh/8kF9rrRNffkLFV4AzhvooEclrOH0xaugmqGsIfFgOE/pIz4fMc4Ef+5HSQqTEug2S9JZIWDR47duDSLfaA==", + "dependencies": { + "async-limiter": "~1.0.0", + "safe-buffer": "~5.1.0" + } + }, + "node_modules/ikonate/node_modules/xml-name-validator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", + "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "dev": true, + "optional": true, + "peer": true + }, + "node_modules/import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "dev": true, + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/import-meta-resolve": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.1.0.tgz", + "integrity": "sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==", + "dev": true, + "optional": true, + "peer": true, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true }, "node_modules/internal-slot": { "version": "1.0.7", @@ -7597,6 +8833,8 @@ "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "jsbn": "1.1.0", "sprintf-js": "^1.1.3" @@ -7609,7 +8847,9 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/ipaddr.js": { "version": "2.1.0", @@ -7636,6 +8876,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", + "dev": true + }, "node_modules/is-async-function": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-async-function/-/is-async-function-2.0.0.tgz", @@ -7933,6 +9179,8 @@ "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -7952,6 +9200,12 @@ "node": ">=0.10.0" } }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "dev": true + }, "node_modules/is-regex": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.0.tgz", @@ -8311,149 +9565,56 @@ "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==" }, "node_modules/jsdom": { - "version": "11.10.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-11.10.0.tgz", - "integrity": "sha512-x5No5FpJgBg3j5aBwA8ka6eGuS5IxbC8FOkmyccKvObtFT0bDMict/LOxINZsZGZSfGdNomLZ/qRV9Bpq/GIBA==", + "version": "26.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-26.0.0.tgz", + "integrity": "sha512-BZYDGVAIriBWTpIxYzrXjv3E/4u8+/pSG5bQdIYCbNCGOvsPkDQfTVLAIXAf9ETdCpduCVTkDe2NNZ8NIwUVzw==", + "dev": true, + "license": "MIT", "dependencies": { - "abab": "^1.0.4", - "acorn": "^5.3.0", - "acorn-globals": "^4.1.0", - "array-equal": "^1.0.0", - "cssom": ">= 0.3.2 < 0.4.0", - "cssstyle": ">= 0.2.37 < 0.3.0", - "data-urls": "^1.0.0", - "domexception": "^1.0.0", - "escodegen": "^1.9.0", - "html-encoding-sniffer": "^1.0.2", - "left-pad": "^1.2.0", - "nwmatcher": "^1.4.3", - "parse5": "4.0.0", - "pn": "^1.1.0", - "request": "^2.83.0", - "request-promise-native": "^1.0.5", - "sax": "^1.2.4", - "symbol-tree": "^3.2.2", - "tough-cookie": "^2.3.3", - "w3c-hr-time": "^1.0.1", - "webidl-conversions": "^4.0.2", - "whatwg-encoding": "^1.0.3", - "whatwg-mimetype": "^2.1.0", - "whatwg-url": "^6.4.0", - "ws": "^4.0.0", - "xml-name-validator": "^3.0.0" - } - }, - "node_modules/jsdom/node_modules/acorn": { - "version": "5.7.4", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz", - "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/jsdom/node_modules/escodegen": { - "version": "1.14.3", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz", - "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==", - "dependencies": { - "esprima": "^4.0.1", - "estraverse": "^4.2.0", - "esutils": "^2.0.2", - "optionator": "^0.8.1" - }, - "bin": { - "escodegen": "bin/escodegen.js", - "esgenerate": "bin/esgenerate.js" - }, - "engines": { - "node": ">=4.0" + "cssstyle": "^4.2.1", + "data-urls": "^5.0.0", + "decimal.js": "^10.4.3", + "form-data": "^4.0.1", + "html-encoding-sniffer": "^4.0.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.6", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.16", + "parse5": "^7.2.1", + "rrweb-cssom": "^0.8.0", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^5.0.0", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^3.1.1", + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.1.0", + "ws": "^8.18.0", + "xml-name-validator": "^5.0.0" }, - "optionalDependencies": { - "source-map": "~0.6.1" - } - }, - "node_modules/jsdom/node_modules/estraverse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", "engines": { - "node": ">=4.0" - } - }, - "node_modules/jsdom/node_modules/levn": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", - "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", - "dependencies": { - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2" + "node": ">=18" }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/jsdom/node_modules/optionator": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", - "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", - "dependencies": { - "deep-is": "~0.1.3", - "fast-levenshtein": "~2.0.6", - "levn": "~0.3.0", - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2", - "word-wrap": "~1.2.3" + "peerDependencies": { + "canvas": "^3.0.0" }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/jsdom/node_modules/prelude-ls": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", - "engines": { - "node": ">= 0.8.0" + "peerDependenciesMeta": { + "canvas": { + "optional": true + } } }, - "node_modules/jsdom/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, "node_modules/jsdom/node_modules/tough-cookie": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", - "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", - "dependencies": { - "psl": "^1.1.28", - "punycode": "^2.1.1" - }, - "engines": { - "node": ">=0.8" - } - }, - "node_modules/jsdom/node_modules/type-check": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", - "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.0.0.tgz", + "integrity": "sha512-FRKsF7cz96xIIeMZ82ehjC3xW2E+O2+v11udrDYewUbszngYhsGa8z6YUMMzO9QJZzzyd0nGGXnML/TReX6W8Q==", + "dev": true, "dependencies": { - "prelude-ls": "~1.1.2" + "tldts": "^6.1.32" }, "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/jsdom/node_modules/ws": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-4.1.0.tgz", - "integrity": "sha512-ZGh/8kF9rrRNffkLFV4AzhvooEclrOH0xaugmqGsIfFgOE/pIz4fMc4Ef+5HSQqTEug2S9JZIWDR47duDSLfaA==", - "dependencies": { - "async-limiter": "~1.0.0", - "safe-buffer": "~5.1.0" + "node": ">=16" } }, "node_modules/json-buffer": { @@ -8491,6 +9652,15 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" }, + "node_modules/jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", + "dev": true, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, "node_modules/jsprim": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz", @@ -8510,6 +9680,8 @@ "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "lie": "~3.3.0", "pako": "~1.0.2", @@ -8521,7 +9693,21 @@ "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", - "dev": true + "dev": true, + "optional": true, + "peer": true + }, + "node_modules/keygrip": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", + "integrity": "sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==", + "dev": true, + "dependencies": { + "tsscmp": "1.0.6" + }, + "engines": { + "node": ">= 0.6" + } }, "node_modules/keyv": { "version": "4.5.4", @@ -8543,6 +9729,203 @@ "node": ">=0.10.0" } }, + "node_modules/koa": { + "version": "2.15.3", + "resolved": "https://registry.npmjs.org/koa/-/koa-2.15.3.tgz", + "integrity": "sha512-j/8tY9j5t+GVMLeioLaxweJiKUayFhlGqNTzf2ZGwL0ZCQijd2RLHK0SLW5Tsko8YyyqCZC2cojIb0/s62qTAg==", + "dev": true, + "dependencies": { + "accepts": "^1.3.5", + "cache-content-type": "^1.0.0", + "content-disposition": "~0.5.2", + "content-type": "^1.0.4", + "cookies": "~0.9.0", + "debug": "^4.3.2", + "delegates": "^1.0.0", + "depd": "^2.0.0", + "destroy": "^1.0.4", + "encodeurl": "^1.0.2", + "escape-html": "^1.0.3", + "fresh": "~0.5.2", + "http-assert": "^1.3.0", + "http-errors": "^1.6.3", + "is-generator-function": "^1.0.7", + "koa-compose": "^4.1.0", + "koa-convert": "^2.0.0", + "on-finished": "^2.3.0", + "only": "~0.0.2", + "parseurl": "^1.3.2", + "statuses": "^1.5.0", + "type-is": "^1.6.16", + "vary": "^1.1.2" + }, + "engines": { + "node": "^4.8.4 || ^6.10.1 || ^7.10.1 || >= 8.1.4" + } + }, + "node_modules/koa-compose": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/koa-compose/-/koa-compose-4.1.0.tgz", + "integrity": "sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==", + "dev": true + }, + "node_modules/koa-convert": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/koa-convert/-/koa-convert-2.0.0.tgz", + "integrity": "sha512-asOvN6bFlSnxewce2e/DK3p4tltyfC4VM7ZwuTuepI7dEQVcvpyFuBcEARu1+Hxg8DIwytce2n7jrZtRlPrARA==", + "dev": true, + "dependencies": { + "co": "^4.6.0", + "koa-compose": "^4.1.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/koa-logger": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/koa-logger/-/koa-logger-3.2.1.tgz", + "integrity": "sha512-MjlznhLLKy9+kG8nAXKJLM0/ClsQp/Or2vI3a5rbSQmgl8IJBQO0KI5FA70BvW+hqjtxjp49SpH2E7okS6NmHg==", + "dev": true, + "dependencies": { + "bytes": "^3.1.0", + "chalk": "^2.4.2", + "humanize-number": "0.0.2", + "passthrough-counter": "^1.0.0" + }, + "engines": { + "node": ">= 7.6.0" + } + }, + "node_modules/koa-logger/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/koa-logger/node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/koa-logger/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/koa-logger/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/koa-logger/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "node_modules/koa-logger/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/koa-logger/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/koa-logger/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/koa/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/koa/node_modules/http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "dev": true, + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/koa/node_modules/http-errors/node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/koa/node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", + "dev": true + }, "node_modules/launch-editor": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/launch-editor/-/launch-editor-2.6.1.tgz", @@ -8558,6 +9941,8 @@ "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "readable-stream": "^2.0.5" }, @@ -8589,6 +9974,8 @@ "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "immediate": "~3.0.5" } @@ -8619,7 +10006,9 @@ "url": "https://github.com/hejny/locate-app/blob/main/README.md#%EF%B8%8F-contributing" } ], - "dependencies": { + "optional": true, + "peer": true, + "dependencies": { "@promptbook/utils": "0.69.5", "type-fest": "4.26.0", "userhome": "1.0.1" @@ -8630,6 +10019,8 @@ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.26.0.tgz", "integrity": "sha512-OduNjVJsFbifKb57UqZ2EMP1i4u64Xwow3NYXUtBbD4vIwJdQd4+xl8YDou1dlm4DVrtwT/7Ky8z8WyCULVfxw==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=16" }, @@ -8666,7 +10057,9 @@ "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/lodash.merge": { "version": "4.6.2", @@ -8683,13 +10076,34 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.zip/-/lodash.zip-4.2.0.tgz", "integrity": "sha512-C7IOaBBK/0gMORRBd8OETNx3kmOkgIWIPvyDpZSCTwUrpYmgZwJkjZeOD8ww4xbOUOs4/attY+pciKvadNfFbg==", - "dev": true + "dev": true, + "optional": true, + "peer": true + }, + "node_modules/logform": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.7.0.tgz", + "integrity": "sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ==", + "dev": true, + "dependencies": { + "@colors/colors": "1.6.0", + "@types/triple-beam": "^1.3.2", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "safe-stable-stringify": "^2.3.1", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } }, "node_modules/loglevel": { "version": "1.9.2", "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz", "integrity": "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 0.6.0" }, @@ -8702,13 +10116,16 @@ "version": "0.8.4", "resolved": "https://registry.npmjs.org/loglevel-plugin-prefix/-/loglevel-plugin-prefix-0.8.4.tgz", "integrity": "sha512-WpG9CcFAOjz/FtNht+QJeGpvVl/cdR6P0z6OcXSkr8wFJOsV2GRj2j10JLfjuA4aYkcKCNIEqRGCyTife9R8/g==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/loupe": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.1.2.tgz", "integrity": "sha512-23I4pFZHmAemUnz8WZXbYRSKYj801VDaNv9ETuMh7IrMc7VuVVSo+Z9iLE3ni30+U48iDWfi30d3twAXBYmnCg==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/lru-cache": { "version": "11.0.0", @@ -8724,15 +10141,17 @@ "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz", "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", "dev": true, + "license": "MIT", "bin": { "lz-string": "bin/bin.js" } }, "node_modules/magic-string": { - "version": "0.30.15", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.15.tgz", - "integrity": "sha512-zXeaYRgZ6ldS1RJJUrMrYgNJ4fdwnyI6tVqoiIhyCyv5IVTK9BU8Ic2l253GGETQHxI4HNUwhJ3fjDhKqEoaAw==", + "version": "0.30.17", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.17.tgz", + "integrity": "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==", "dev": true, + "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" } @@ -8888,6 +10307,18 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dev": true, + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, "node_modules/mrmime": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.0.tgz", @@ -8904,9 +10335,9 @@ "dev": true }, "node_modules/msw": { - "version": "2.6.8", - "resolved": "https://registry.npmjs.org/msw/-/msw-2.6.8.tgz", - "integrity": "sha512-nxXxnH6WALZ9a7rsQp4HU2AaD4iGAiouMmE/MY4al7pXTibgA6OZOuKhmN2WBIM6w9qMKwRtX8p2iOb45B2M/Q==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/msw/-/msw-2.7.0.tgz", + "integrity": "sha512-BIodwZ19RWfCbYTxWTUfTXc+sg4OwjCAgxU1ZsgmggX/7S3LdUifsbUPJs61j0rWb19CZRGY5if77duhc0uXzw==", "dev": true, "hasInstallScript": true, "dependencies": { @@ -8919,12 +10350,12 @@ "@open-draft/until": "^2.1.0", "@types/cookie": "^0.6.0", "@types/statuses": "^2.0.4", - "chalk": "^4.1.2", "graphql": "^16.8.1", "headers-polyfill": "^4.0.2", "is-node-process": "^1.2.0", "outvariant": "^1.4.3", "path-to-regexp": "^6.3.0", + "picocolors": "^1.1.1", "strict-event-emitter": "^0.5.1", "type-fest": "^4.26.1", "yargs": "^17.7.2" @@ -9027,6 +10458,8 @@ "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 0.4.0" } @@ -9054,6 +10487,8 @@ "url": "https://paypal.me/jimmywarting" } ], + "optional": true, + "peer": true, "engines": { "node": ">=10.5.0" } @@ -9063,6 +10498,8 @@ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", @@ -9133,6 +10570,8 @@ "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "boolbase": "^1.0.0" }, @@ -9188,6 +10627,12 @@ "resolved": "https://registry.npmjs.org/nwmatcher/-/nwmatcher-1.4.4.tgz", "integrity": "sha512-3iuY4N5dhgMpCUrOVnuAdGrgxVqV2cJpM+XNccjR2DKOB1RUP0aA+wGXEiNziG/UKboFyGBIoKOaNlJxx8bciQ==" }, + "node_modules/nwsapi": { + "version": "2.2.16", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.16.tgz", + "integrity": "sha512-F1I/bimDpj3ncaNDhfyMWuFqmQDBwDB0Fogc2qpL3BWvkQteFD/8BzWuIRl83rq0DXfm8SGt/HFhLXZyljTXcQ==", + "dev": true + }, "node_modules/oauth-sign": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", @@ -9316,10 +10761,21 @@ "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "wrappy": "1" } }, + "node_modules/one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "dev": true, + "dependencies": { + "fn.name": "1.x.x" + } + }, "node_modules/onetime": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", @@ -9335,6 +10791,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/only": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/only/-/only-0.0.2.tgz", + "integrity": "sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ==", + "dev": true + }, "node_modules/open": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/open/-/open-10.1.0.tgz", @@ -9420,6 +10882,8 @@ "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.1.0.tgz", "integrity": "sha512-Z5FnLVVZSnX7WjBg0mhDtydeRZ1xMcATZThjySQUHqr+0ksP8kqaw23fNKkaaN/Z8gwLUs/W7xdl0I75eP2Xyw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "@tootallnate/quickjs-emscripten": "^0.23.0", "agent-base": "^7.1.2", @@ -9439,6 +10903,8 @@ "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "degenerator": "^5.0.0", "netmask": "^2.0.2" @@ -9465,15 +10931,24 @@ } }, "node_modules/parse5": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz", - "integrity": "sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA==" + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz", + "integrity": "sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==", + "dev": true, + "dependencies": { + "entities": "^4.5.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } }, "node_modules/parse5-htmlparser2-tree-adapter": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "domhandler": "^5.0.3", "parse5": "^7.0.0" @@ -9487,6 +10962,8 @@ "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "domelementtype": "^2.3.0" }, @@ -9497,35 +10974,13 @@ "url": "https://github.com/fb55/domhandler?sponsor=1" } }, - "node_modules/parse5-htmlparser2-tree-adapter/node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "dev": true, - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/parse5-htmlparser2-tree-adapter/node_modules/parse5": { - "version": "7.2.1", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz", - "integrity": "sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==", - "dev": true, - "dependencies": { - "entities": "^4.5.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, "node_modules/parse5-parser-stream": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "parse5": "^7.0.0" }, @@ -9533,30 +10988,6 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, - "node_modules/parse5-parser-stream/node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "dev": true, - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/parse5-parser-stream/node_modules/parse5": { - "version": "7.2.1", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz", - "integrity": "sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==", - "dev": true, - "dependencies": { - "entities": "^4.5.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -9566,6 +10997,12 @@ "node": ">= 0.8" } }, + "node_modules/passthrough-counter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/passthrough-counter/-/passthrough-counter-1.0.0.tgz", + "integrity": "sha512-Wy8PXTLqPAN0oEgBrlnsXPMww3SYJ44tQ8aVrGAI4h4JZYCS0oYqsPqtPR8OhJpv6qFbpbB7XAn0liKV7EXubA==", + "dev": true + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -9612,16 +11049,18 @@ "dev": true }, "node_modules/pathe": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", - "integrity": "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==", - "dev": true + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.2.tgz", + "integrity": "sha512-15Ztpk+nov8DR524R4BF7uEuzESgzUEAV4Ah7CUMNGXdE5ELuvxElxGXndBl32vMSsWa1jpNf22Z+Er3sKwq+w==", + "dev": true, + "license": "MIT" }, "node_modules/pathval": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.0.tgz", "integrity": "sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==", "dev": true, + "license": "MIT", "engines": { "node": ">= 14.16" } @@ -9630,7 +11069,8 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/performance-now": { "version": "2.1.0", @@ -9655,11 +11095,90 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.49.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.49.1.tgz", + "integrity": "sha512-VYL8zLoNTBxVOrJBbDuRgDWa3i+mfQgDTrL8Ah9QXZ7ax4Dsj0MSq5bYgytRnDVVe+njoKnfsYkH3HzqVj5UZA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.49.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.49.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.49.1.tgz", + "integrity": "sha512-BzmpVcs4kE2CH15rWfzpjzVGhWERJfmnXmniSyKeRZUs9Ws65m+RGIi7mjJK/euCegfn3i7jvqWeWyHe9y3Vgg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/pn": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==" }, + "node_modules/portfinder": { + "version": "1.0.32", + "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.32.tgz", + "integrity": "sha512-on2ZJVVDXRADWE6jnQaX0ioEylzgBpQk8r55NE4wjXW1ZxO+BgDlY6DXwj20i0V8eB4SenDQ00WEaxfiIQPcxg==", + "dev": true, + "dependencies": { + "async": "^2.6.4", + "debug": "^3.2.7", + "mkdirp": "^0.5.6" + }, + "engines": { + "node": ">= 0.12.0" + } + }, + "node_modules/portfinder/node_modules/async": { + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", + "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", + "dev": true, + "dependencies": { + "lodash": "^4.17.14" + } + }, + "node_modules/portfinder/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "dependencies": { + "ms": "^2.1.1" + } + }, "node_modules/possible-typed-array-names": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz", @@ -9817,6 +11336,7 @@ "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", "dev": true, + "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1", "ansi-styles": "^5.0.0", @@ -9831,6 +11351,7 @@ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, @@ -9843,6 +11364,8 @@ "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 0.6.0" } @@ -9858,6 +11381,8 @@ "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.4.0" } @@ -9894,6 +11419,8 @@ "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.2", "debug": "^4.3.4", @@ -9913,6 +11440,8 @@ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=12" } @@ -9921,7 +11450,9 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/psl": { "version": "1.9.0", @@ -9933,6 +11464,8 @@ "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -9958,7 +11491,9 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/query-selector-shadow-dom/-/query-selector-shadow-dom-1.0.1.tgz", "integrity": "sha512-lT5yCqEBgfoMYpf3F2xQRK7zEr1rhIIZuceDK6+xRkJQ4NMbHTwXqk4NkwDwQMNqXgG9r9fyHnzwNVs6zV5KRw==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/querystringify": { "version": "2.2.0", @@ -9990,7 +11525,9 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/randombytes": { "version": "2.1.0", @@ -10040,7 +11577,8 @@ "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/readable-stream": { "version": "2.3.8", @@ -10074,6 +11612,8 @@ "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "minimatch": "^5.1.0" } @@ -10083,6 +11623,8 @@ "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -10140,7 +11682,8 @@ "version": "0.14.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/regexp.prototype.flags": { "version": "1.5.3", @@ -10323,6 +11866,8 @@ "resolved": "https://registry.npmjs.org/resq/-/resq-1.11.0.tgz", "integrity": "sha512-G10EBz+zAAy3zUd/CDoBbXRL6ia9kOo3xRHrMDsHljI0GDkhYlyjwoCx5+3eCC4swi1uCoZQhskuJkj7Gp57Bw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "fast-deep-equal": "^2.0.1" } @@ -10331,7 +11876,9 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", "integrity": "sha512-bCK/2Z4zLidyB4ReuIsvALH6w31YfAQDmXMqMx6FyfHqvBxtjC0eRumeSu4Bs3XtXwpyIywtSTrVT99BxY1f9w==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/retry": { "version": "0.13.1", @@ -10356,7 +11903,9 @@ "version": "0.2.5", "resolved": "https://registry.npmjs.org/rgb2hex/-/rgb2hex-0.2.5.tgz", "integrity": "sha512-22MOP1Rh7sAo1BZpDG6R5RFYzR2lYEgwq7HEmyW2qcsOqR2lQKmn+O//xV3YG/0rrhMC6KVX2hU+ZXuaw9a5bw==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/rimraf": { "version": "5.0.10", @@ -10431,10 +11980,11 @@ } }, "node_modules/rollup": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.28.1.tgz", - "integrity": "sha512-61fXYl/qNVinKmGSTHAZ6Yy8I3YIJC/r2m9feHo6SwVAVcLT5MPwOUFe7EuURA/4m0NR8lXG4BBXuo/IZEsjMg==", + "version": "4.30.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.30.1.tgz", + "integrity": "sha512-mlJ4glW020fPuLi7DkM/lN97mYEZGWeqBnrljzN0gs7GLctqX3lNWxKQ7Gl712UAX+6fog/L3jh4gb7R6aVi3w==", "dev": true, + "license": "MIT", "dependencies": { "@types/estree": "1.0.6" }, @@ -10446,28 +11996,35 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.28.1", - "@rollup/rollup-android-arm64": "4.28.1", - "@rollup/rollup-darwin-arm64": "4.28.1", - "@rollup/rollup-darwin-x64": "4.28.1", - "@rollup/rollup-freebsd-arm64": "4.28.1", - "@rollup/rollup-freebsd-x64": "4.28.1", - "@rollup/rollup-linux-arm-gnueabihf": "4.28.1", - "@rollup/rollup-linux-arm-musleabihf": "4.28.1", - "@rollup/rollup-linux-arm64-gnu": "4.28.1", - "@rollup/rollup-linux-arm64-musl": "4.28.1", - "@rollup/rollup-linux-loongarch64-gnu": "4.28.1", - "@rollup/rollup-linux-powerpc64le-gnu": "4.28.1", - "@rollup/rollup-linux-riscv64-gnu": "4.28.1", - "@rollup/rollup-linux-s390x-gnu": "4.28.1", - "@rollup/rollup-linux-x64-gnu": "4.28.1", - "@rollup/rollup-linux-x64-musl": "4.28.1", - "@rollup/rollup-win32-arm64-msvc": "4.28.1", - "@rollup/rollup-win32-ia32-msvc": "4.28.1", - "@rollup/rollup-win32-x64-msvc": "4.28.1", + "@rollup/rollup-android-arm-eabi": "4.30.1", + "@rollup/rollup-android-arm64": "4.30.1", + "@rollup/rollup-darwin-arm64": "4.30.1", + "@rollup/rollup-darwin-x64": "4.30.1", + "@rollup/rollup-freebsd-arm64": "4.30.1", + "@rollup/rollup-freebsd-x64": "4.30.1", + "@rollup/rollup-linux-arm-gnueabihf": "4.30.1", + "@rollup/rollup-linux-arm-musleabihf": "4.30.1", + "@rollup/rollup-linux-arm64-gnu": "4.30.1", + "@rollup/rollup-linux-arm64-musl": "4.30.1", + "@rollup/rollup-linux-loongarch64-gnu": "4.30.1", + "@rollup/rollup-linux-powerpc64le-gnu": "4.30.1", + "@rollup/rollup-linux-riscv64-gnu": "4.30.1", + "@rollup/rollup-linux-s390x-gnu": "4.30.1", + "@rollup/rollup-linux-x64-gnu": "4.30.1", + "@rollup/rollup-linux-x64-musl": "4.30.1", + "@rollup/rollup-win32-arm64-msvc": "4.30.1", + "@rollup/rollup-win32-ia32-msvc": "4.30.1", + "@rollup/rollup-win32-x64-msvc": "4.30.1", "fsevents": "~2.3.2" } }, + "node_modules/rrweb-cssom": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz", + "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==", + "dev": true, + "license": "MIT" + }, "node_modules/run-applescript": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.0.0.tgz", @@ -10503,35 +12060,87 @@ "queue-microtask": "^1.2.2" } }, - "node_modules/safaridriver": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/safaridriver/-/safaridriver-0.1.2.tgz", - "integrity": "sha512-4R309+gWflJktzPXBQCobbWEHlzC4aK3a+Ov3tz2Ib2aBxiwd11phkdIBH1l0EO22x24CJMUQkpKFumRriCSRg==", - "dev": true - }, - "node_modules/safe-array-concat": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.2.tgz", - "integrity": "sha512-vj6RsCsWBCf19jIeHEfkRMw8DPiBb+DMXklQ/1SGDHOMlHdPUkZXFQ2YdplS23zESTijAcurb1aSgJA3AgMu1Q==", + "node_modules/s3rver": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/s3rver/-/s3rver-3.7.1.tgz", + "integrity": "sha512-H9KIX6n8NqcfoE4ziFNbQASBQfjcNJgb+3wbT9L5iotEqfOncFO1c38cfJSFSo7xXTu1zM9HA6t2u9xKNlYRaA==", "dev": true, "dependencies": { - "call-bind": "^1.0.7", - "get-intrinsic": "^1.2.4", - "has-symbols": "^1.0.3", - "isarray": "^2.0.5" + "@koa/router": "^9.0.0", + "busboy": "^0.3.1", + "commander": "^5.0.0", + "fast-xml-parser": "^3.12.19", + "fs-extra": "^8.0.0", + "he": "^1.2.0", + "koa": "^2.7.0", + "koa-logger": "^3.2.0", + "lodash": "^4.17.5", + "statuses": "^2.0.0", + "winston": "^3.0.0" }, - "engines": { - "node": ">=0.4" + "bin": { + "s3rver": "bin/s3rver.js" }, - "funding": { - "url": "https://github.com/sponsors/ljharb" + "engines": { + "node": ">=8.3.0" } }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ + "node_modules/s3rver/node_modules/commander": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", + "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/s3rver/node_modules/fast-xml-parser": { + "version": "3.21.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-3.21.1.tgz", + "integrity": "sha512-FTFVjYoBOZTJekiUsawGsSYV9QL0A+zDYCRj7y34IO6Jg+2IMYEtQa+bbictpdpV8dHxXywqU7C0gRDEOFtBFg==", + "dev": true, + "dependencies": { + "strnum": "^1.0.4" + }, + "bin": { + "xml2js": "cli.js" + }, + "funding": { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + }, + "node_modules/safaridriver": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/safaridriver/-/safaridriver-0.1.2.tgz", + "integrity": "sha512-4R309+gWflJktzPXBQCobbWEHlzC4aK3a+Ov3tz2Ib2aBxiwd11phkdIBH1l0EO22x24CJMUQkpKFumRriCSRg==", + "dev": true, + "optional": true, + "peer": true + }, + "node_modules/safe-array-concat": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.2.tgz", + "integrity": "sha512-vj6RsCsWBCf19jIeHEfkRMw8DPiBb+DMXklQ/1SGDHOMlHdPUkZXFQ2YdplS23zESTijAcurb1aSgJA3AgMu1Q==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.7", + "get-intrinsic": "^1.2.4", + "has-symbols": "^1.0.3", + "isarray": "^2.0.5" + }, + "engines": { + "node": ">=0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ { "type": "github", "url": "https://github.com/sponsors/feross" @@ -10563,15 +12172,36 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "node_modules/sax": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz", - "integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==" + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz", + "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==" + }, + "node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "dev": true, + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=v12.22.7" + } }, "node_modules/schema-utils": { "version": "3.3.0", @@ -10593,6 +12223,12 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/secure-compare": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/secure-compare/-/secure-compare-3.0.1.tgz", + "integrity": "sha512-AckIIV90rPDcBcglUwXPF3kg0P0qmPsPXAj6BBEENQE1p5yA1xfmDJzfi1Tappj37Pv2mVbKpL3Z1T+Nn7k1Qw==", + "dev": true + }, "node_modules/select-hose": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", @@ -10676,6 +12312,8 @@ "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-11.0.3.tgz", "integrity": "sha512-2G2y++21dhj2R7iHAdd0FIzjGwuKZld+7Pl/bTU6YIkrC2ZMbVUjm+luj6A6V34Rv9XfKJDKpTWu9W4Gse1D9g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "type-fest": "^2.12.2" }, @@ -10826,7 +12464,9 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/setprototypeof": { "version": "1.2.0", @@ -10972,6 +12612,15 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dev": true, + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, "node_modules/sirv": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/sirv/-/sirv-2.0.4.tgz", @@ -10991,6 +12640,8 @@ "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 6.0.0", "npm": ">= 3.0.0" @@ -11021,6 +12672,8 @@ "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "ip-address": "^9.0.5", "smart-buffer": "^4.2.0" @@ -11035,6 +12688,8 @@ "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.2", "debug": "^4.3.4", @@ -11088,7 +12743,9 @@ "type": "github", "url": "https://github.com/hejny/spacetrim/blob/main/README.md#%EF%B8%8F-contributing" } - ] + ], + "optional": true, + "peer": true }, "node_modules/spdy": { "version": "4.0.2", @@ -11139,6 +12796,8 @@ "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 10.x" } @@ -11147,7 +12806,9 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/sshpk": { "version": "1.18.0", @@ -11179,6 +12840,15 @@ "integrity": "sha512-LjdcbuBeLcdETCrPn9i8AYAZ1eCtu4ECAWtP7UleOiZ9LzVxRzzUZEoZ8zB24nhkQnDWyET0I+3sWokSDS3E7g==", "dev": true }, + "node_modules/stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "dev": true, + "engines": { + "node": "*" + } + }, "node_modules/stackback": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", @@ -11208,11 +12878,22 @@ "node": ">=0.10.0" } }, + "node_modules/streamsearch": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", + "integrity": "sha512-jos8u++JKm0ARcSUTAZXOVC0mSox7Bhn6sBgty73P1f3JGf7yG2clTbBNHUdde/kdvP2FESam+vM6l8jBrNxHA==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/streamx": { "version": "2.21.0", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.21.0.tgz", "integrity": "sha512-Qz6MsDZXJ6ur9u+b+4xCG18TluU7PGlRfXVAAjNiGsFrBUt/ioyLkxbFaKJygoPs+/kW4VyBj0bSj89Qu0IGyg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "fast-fifo": "^1.3.2", "queue-tick": "^1.0.1", @@ -11461,6 +13142,8 @@ "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.6.tgz", "integrity": "sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "pump": "^3.0.0", "tar-stream": "^3.1.5" @@ -11475,6 +13158,8 @@ "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "b4a": "^1.6.4", "fast-fifo": "^1.2.0", @@ -11550,10 +13235,18 @@ "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.1.1.tgz", "integrity": "sha512-8zll7REEv4GDD3x4/0pW+ppIxSNs7H1J10IKFZsuOMscumCdM2a+toDGLPA3T+1+fLBql4zbt5z83GEQGGV5VA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "b4a": "^1.6.4" } }, + "node_modules/text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==", + "dev": true + }, "node_modules/thingies": { "version": "1.21.0", "resolved": "https://registry.npmjs.org/thingies/-/thingies-1.21.0.tgz", @@ -11570,7 +13263,9 @@ "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/thunky": { "version": "1.1.0", @@ -11585,16 +13280,18 @@ "dev": true }, "node_modules/tinyexec": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.1.tgz", - "integrity": "sha512-WiCJLEECkO18gwqIp6+hJg0//p23HXp4S+gGtAKu3mI2F2/sXC4FvHvXvB0zJVVaTPhx1/tOwdbRsa1sOBIKqQ==", - "dev": true + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" }, "node_modules/tinyglobby": { "version": "0.2.10", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.10.tgz", "integrity": "sha512-Zc+8eJlFMvgatPZTl6A9L/yht8QqdmUNtURHaKZLmKBE12hNPSrqNkUp2cs3M/UKmNVVAMFQYSjYIVHDjW5zew==", "dev": true, + "license": "MIT", "dependencies": { "fdir": "^6.4.2", "picomatch": "^4.0.2" @@ -11604,10 +13301,11 @@ } }, "node_modules/tinyglobby/node_modules/fdir": { - "version": "6.4.2", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.2.tgz", - "integrity": "sha512-KnhMXsKSPZlAhp7+IjUkRZKPb4fUyccpDrdFXbi4QL1qkmFh9kVY09Yox+n4MaOb3lHZ1Tv829C3oaaXoMYPDQ==", + "version": "6.4.3", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.3.tgz", + "integrity": "sha512-PMXmW2y1hDDfTSRc9gaXIuCCRpuoz3Kaz8cUelp3smouvfT632ozg2vrT6lJsHKKOF59YLbOGfAWGUcKEfRMQw==", "dev": true, + "license": "MIT", "peerDependencies": { "picomatch": "^3 || ^4" }, @@ -11622,6 +13320,7 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", "dev": true, + "license": "MIT", "engines": { "node": ">=12" }, @@ -11639,10 +13338,11 @@ } }, "node_modules/tinyrainbow": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-1.2.0.tgz", - "integrity": "sha512-weEDEq7Z5eTHPDh4xjX789+fHfF+P8boiFB+0vbWzpbnbsEr/GRaohi/uMKxg8RZMXnl1ItAi/IUHWMsjDV7kQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", "dev": true, + "license": "MIT", "engines": { "node": ">=14.0.0" } @@ -11652,10 +13352,29 @@ "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-3.0.2.tgz", "integrity": "sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==", "dev": true, + "license": "MIT", "engines": { "node": ">=14.0.0" } }, + "node_modules/tldts": { + "version": "6.1.70", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.70.tgz", + "integrity": "sha512-/W1YVgYVJd9ZDjey5NXadNh0mJXkiUMUue9Zebd0vpdo1sU+H4zFFTaJ1RKD4N6KFoHfcXy6l+Vu7bh+bdWCzA==", + "dev": true, + "dependencies": { + "tldts-core": "^6.1.70" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "6.1.70", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.70.tgz", + "integrity": "sha512-RNnIXDB1FD4T9cpQRErEqw6ZpjLlGdMOitdV+0xtbsnwr4YFka1zpc7D4KD+aAn8oSG5JyFrdasZTE04qDE9Yg==", + "dev": true + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -11702,11 +13421,15 @@ } }, "node_modules/tr46": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", - "integrity": "sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.0.0.tgz", + "integrity": "sha512-tk2G5R2KRwBd+ZN0zaEXpmzdKyOYksXwywulIX95MBODjSzMIuQnQ3m8JxgbhnL1LeVo7lqQKsYa1O3Htl7K5g==", + "dev": true, "dependencies": { - "punycode": "^2.1.0" + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" } }, "node_modules/tree-dump": { @@ -11725,23 +13448,34 @@ "tslib": "2" } }, + "node_modules/triple-beam": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", + "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", + "dev": true, + "engines": { + "node": ">= 14.0.0" + } + }, "node_modules/ts-api-utils": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.3.tgz", - "integrity": "sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.0.0.tgz", + "integrity": "sha512-xCt/TOAc+EOHS1XPnijD3/yzpH6qg2xppZO1YDqGoVsNXfQfzHpOdNuXwrwOU8u4ITXJyDCTyt8w5g1sZv9ynQ==", "dev": true, + "license": "MIT", "engines": { - "node": ">=16" + "node": ">=18.12" }, "peerDependencies": { - "typescript": ">=4.2.0" + "typescript": ">=4.8.4" } }, "node_modules/ts-checker-rspack-plugin": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/ts-checker-rspack-plugin/-/ts-checker-rspack-plugin-1.0.3.tgz", - "integrity": "sha512-K5BUrytoFju1Olu11T49vlYvDEGOguBF1CBCl4o2ARxDGPoJHHf7fBzLlK0YYkUqI5EFA5cMRUC6332M7hQBHw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ts-checker-rspack-plugin/-/ts-checker-rspack-plugin-1.1.1.tgz", + "integrity": "sha512-BlpPqnfAmV0TcDg58H+1qV8Zb57ilv0x+ajjnxrVQ6BWgC8HzAdc+TycqDOJ4sZZYIV+hywQGozZFGklzbCR6A==", "dev": true, + "license": "MIT", "dependencies": { "@babel/code-frame": "^7.16.7", "@rspack/lite-tapable": "^1.0.0", @@ -11793,6 +13527,15 @@ "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", "dev": true }, + "node_modules/tsscmp": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/tsscmp/-/tsscmp-1.0.6.tgz", + "integrity": "sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==", + "dev": true, + "engines": { + "node": ">=0.6.x" + } + }, "node_modules/tsx": { "version": "4.19.2", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.19.2.tgz", @@ -12268,6 +14011,8 @@ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=12.20" }, @@ -12363,10 +14108,11 @@ } }, "node_modules/typescript": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.2.tgz", - "integrity": "sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.3.tgz", + "integrity": "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -12376,14 +14122,15 @@ } }, "node_modules/typescript-eslint": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.18.0.tgz", - "integrity": "sha512-Xq2rRjn6tzVpAyHr3+nmSg1/9k9aIHnJ2iZeOH7cfGOWqTkXTm3kwpQglEuLGdNrYvPF+2gtAs+/KF5rjVo+WQ==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.20.0.tgz", + "integrity": "sha512-Kxz2QRFsgbWj6Xcftlw3Dd154b3cEPFqQC+qMZrMypSijPd4UanKKvoKDrJ4o8AIfZFKAF+7sMaEIR8mTElozA==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.18.0", - "@typescript-eslint/parser": "8.18.0", - "@typescript-eslint/utils": "8.18.0" + "@typescript-eslint/eslint-plugin": "8.20.0", + "@typescript-eslint/parser": "8.20.0", + "@typescript-eslint/utils": "8.20.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -12417,6 +14164,8 @@ "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "buffer": "^5.2.1", "through": "^2.3.8" @@ -12441,6 +14190,8 @@ "url": "https://feross.org/support" } ], + "optional": true, + "peer": true, "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -12451,6 +14202,8 @@ "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.0.tgz", "integrity": "sha512-BUgJXc752Kou3oOIuU1i+yZZypyZRqNPW0vqoMPl8VaoalSfeR0D8/t4iAS3yirs79SSMTxTag+ZC86uswv+Cw==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=18.17" } @@ -12461,6 +14214,18 @@ "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", "dev": true }, + "node_modules/union": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/union/-/union-0.5.0.tgz", + "integrity": "sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA==", + "dev": true, + "dependencies": { + "qs": "^6.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/universalify": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", @@ -12519,6 +14284,12 @@ "punycode": "^2.1.0" } }, + "node_modules/url-join": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", + "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==", + "dev": true + }, "node_modules/url-parse": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", @@ -12533,13 +14304,17 @@ "version": "10.0.0", "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-10.0.0.tgz", "integrity": "sha512-H/A06tKD7sS1O1X2SshBVeA5FLycRpjqiBeqGKmBwBDBy28EnRjORxTNe269KSSr5un5qyWi1iL61wLxpd+ZOg==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/userhome": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/userhome/-/userhome-1.0.1.tgz", "integrity": "sha512-5cnLm4gseXjAclKowC4IjByaGsjtAoV6PrOQOljplNB54ReUYJP8HdAFq2muHinSDAh09PPX/uXDPfdxRHvuSA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 0.8.0" } @@ -12577,552 +14352,161 @@ "node": ">= 0.8" } }, - "node_modules/verror": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", - "engines": [ - "node >=0.6.0" - ], - "dependencies": { - "assert-plus": "^1.0.0", - "core-util-is": "1.0.2", - "extsprintf": "^1.2.0" - } - }, - "node_modules/verror/node_modules/core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==" - }, - "node_modules/vite": { - "version": "5.4.11", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.11.tgz", - "integrity": "sha512-c7jFQRklXua0mTzneGW9QVyxFjUgwcihC4bXEtujIo2ouWCe1Ajt/amn2PCxYnhYfd5k09JX3SB7OYWFKYqj8Q==", - "dev": true, - "dependencies": { - "esbuild": "^0.21.3", - "postcss": "^8.4.43", - "rollup": "^4.20.0" - }, - "bin": { - "vite": "bin/vite.js" - }, - "engines": { - "node": "^18.0.0 || >=20.0.0" - }, - "funding": { - "url": "https://github.com/vitejs/vite?sponsor=1" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - }, - "peerDependencies": { - "@types/node": "^18.0.0 || >=20.0.0", - "less": "*", - "lightningcss": "^1.21.0", - "sass": "*", - "sass-embedded": "*", - "stylus": "*", - "sugarss": "*", - "terser": "^5.4.0" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "less": { - "optional": true - }, - "lightningcss": { - "optional": true - }, - "sass": { - "optional": true - }, - "sass-embedded": { - "optional": true - }, - "stylus": { - "optional": true - }, - "sugarss": { - "optional": true - }, - "terser": { - "optional": true - } - } - }, - "node_modules/vite-node": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.1.8.tgz", - "integrity": "sha512-uPAwSr57kYjAUux+8E2j0q0Fxpn8M9VoyfGiRI8Kfktz9NcYMCenwY5RnZxnF1WTu3TGiYipirIzacLL3VVGFg==", - "dev": true, - "dependencies": { - "cac": "^6.7.14", - "debug": "^4.3.7", - "es-module-lexer": "^1.5.4", - "pathe": "^1.1.2", - "vite": "^5.0.0" - }, - "bin": { - "vite-node": "vite-node.mjs" - }, - "engines": { - "node": "^18.0.0 || >=20.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/vite/node_modules/@esbuild/aix-ppc64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", - "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-arm": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", - "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", - "cpu": [ - "arm" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", - "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", - "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/darwin-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", - "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/darwin-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", - "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/freebsd-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", - "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/freebsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", - "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-arm": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", - "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", - "cpu": [ - "arm" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", - "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-ia32": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", - "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", - "cpu": [ - "ia32" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-loong64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", - "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", - "cpu": [ - "loong64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-mips64el": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", - "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", - "cpu": [ - "mips64el" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-ppc64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", - "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", - "cpu": [ - "ppc64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-riscv64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", - "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", - "cpu": [ - "riscv64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-s390x": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", - "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", - "cpu": [ - "s390x" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", - "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/netbsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", - "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/openbsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", - "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/sunos-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", - "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", - "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-ia32": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", - "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", - "cpu": [ - "ia32" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", - "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", - "cpu": [ - "x64" + "node_modules/verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", + "engines": [ + "node >=0.6.0" ], + "dependencies": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "node_modules/verror/node_modules/core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==" + }, + "node_modules/vite": { + "version": "6.0.7", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.0.7.tgz", + "integrity": "sha512-RDt8r/7qx9940f8FcOIAH9PTViRrghKaK2K1jY3RaAURrEUbm9Du1mJ72G+jlhtG3WwodnfzY8ORQZbBavZEAQ==", "dev": true, - "optional": true, - "os": [ - "win32" - ], + "license": "MIT", + "dependencies": { + "esbuild": "^0.24.2", + "postcss": "^8.4.49", + "rollup": "^4.23.0" + }, + "bin": { + "vite": "bin/vite.js" + }, "engines": { - "node": ">=12" + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } } }, - "node_modules/vite/node_modules/esbuild": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", - "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "node_modules/vite-node": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-3.0.2.tgz", + "integrity": "sha512-hsEQerBAHvVAbv40m3TFQe/lTEbOp7yDpyqMJqr2Tnd+W58+DEYOt+fluQgekOePcsNBmR77lpVAnIU2Xu4SvQ==", "dev": true, - "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.4.0", + "es-module-lexer": "^1.6.0", + "pathe": "^2.0.1", + "vite": "^5.0.0 || ^6.0.0" + }, "bin": { - "esbuild": "bin/esbuild" + "vite-node": "vite-node.mjs" }, "engines": { - "node": ">=12" + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.21.5", - "@esbuild/android-arm": "0.21.5", - "@esbuild/android-arm64": "0.21.5", - "@esbuild/android-x64": "0.21.5", - "@esbuild/darwin-arm64": "0.21.5", - "@esbuild/darwin-x64": "0.21.5", - "@esbuild/freebsd-arm64": "0.21.5", - "@esbuild/freebsd-x64": "0.21.5", - "@esbuild/linux-arm": "0.21.5", - "@esbuild/linux-arm64": "0.21.5", - "@esbuild/linux-ia32": "0.21.5", - "@esbuild/linux-loong64": "0.21.5", - "@esbuild/linux-mips64el": "0.21.5", - "@esbuild/linux-ppc64": "0.21.5", - "@esbuild/linux-riscv64": "0.21.5", - "@esbuild/linux-s390x": "0.21.5", - "@esbuild/linux-x64": "0.21.5", - "@esbuild/netbsd-x64": "0.21.5", - "@esbuild/openbsd-x64": "0.21.5", - "@esbuild/sunos-x64": "0.21.5", - "@esbuild/win32-arm64": "0.21.5", - "@esbuild/win32-ia32": "0.21.5", - "@esbuild/win32-x64": "0.21.5" + "funding": { + "url": "https://opencollective.com/vitest" } }, "node_modules/vitest": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.1.8.tgz", - "integrity": "sha512-1vBKTZskHw/aosXqQUlVWWlGUxSJR8YtiyZDJAFeW2kPAeX6S3Sool0mjspO+kXLuxVWlEDDowBAeqeAQefqLQ==", - "dev": true, - "dependencies": { - "@vitest/expect": "2.1.8", - "@vitest/mocker": "2.1.8", - "@vitest/pretty-format": "^2.1.8", - "@vitest/runner": "2.1.8", - "@vitest/snapshot": "2.1.8", - "@vitest/spy": "2.1.8", - "@vitest/utils": "2.1.8", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.0.2.tgz", + "integrity": "sha512-5bzaHakQ0hmVVKLhfh/jXf6oETDBtgPo8tQCHYB+wftNgFJ+Hah67IsWc8ivx4vFL025Ow8UiuTf4W57z4izvQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "3.0.2", + "@vitest/mocker": "3.0.2", + "@vitest/pretty-format": "^3.0.2", + "@vitest/runner": "3.0.2", + "@vitest/snapshot": "3.0.2", + "@vitest/spy": "3.0.2", + "@vitest/utils": "3.0.2", "chai": "^5.1.2", - "debug": "^4.3.7", + "debug": "^4.4.0", "expect-type": "^1.1.0", - "magic-string": "^0.30.12", - "pathe": "^1.1.2", + "magic-string": "^0.30.17", + "pathe": "^2.0.1", "std-env": "^3.8.0", "tinybench": "^2.9.0", - "tinyexec": "^0.3.1", - "tinypool": "^1.0.1", - "tinyrainbow": "^1.2.0", - "vite": "^5.0.0", - "vite-node": "2.1.8", + "tinyexec": "^0.3.2", + "tinypool": "^1.0.2", + "tinyrainbow": "^2.0.0", + "vite": "^5.0.0 || ^6.0.0", + "vite-node": "3.0.2", "why-is-node-running": "^2.3.0" }, "bin": { "vitest": "vitest.mjs" }, "engines": { - "node": "^18.0.0 || >=20.0.0" + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" }, "funding": { "url": "https://opencollective.com/vitest" }, "peerDependencies": { "@edge-runtime/vm": "*", - "@types/node": "^18.0.0 || >=20.0.0", - "@vitest/browser": "2.1.8", - "@vitest/ui": "2.1.8", + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@vitest/browser": "3.0.2", + "@vitest/ui": "3.0.2", "happy-dom": "*", "jsdom": "*" }, @@ -13156,11 +14540,25 @@ "browser-process-hrtime": "^1.0.0" } }, + "node_modules/w3c-xmlserializer": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", + "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", + "dev": true, + "dependencies": { + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/wait-port": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/wait-port/-/wait-port-1.1.0.tgz", "integrity": "sha512-3e04qkoN3LxTMLakdqeWth8nih8usyg+sf1Bgdf9wwUkp05iuK1eSY/QpLvscT/+F/gA89+LpUmmgBtesbqI2Q==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "chalk": "^4.1.2", "commander": "^9.3.0", @@ -13202,6 +14600,8 @@ "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">= 8" } @@ -13211,6 +14611,8 @@ "resolved": "https://registry.npmjs.org/webdriver/-/webdriver-9.4.1.tgz", "integrity": "sha512-vFDdxMj/9W1+6jhpHSiRYfO8dix23HjAUtLx7aOv9ejEsntC0EzCIAftJ59YsF3Ppu184+FkdDVhnivpkZPTFw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.1.0", "@types/ws": "^8.5.3", @@ -13232,6 +14634,8 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.10.tgz", "integrity": "sha512-/jrvh5h6NXhEauFFexRin69nA0uHJ5gwk4iDivp/DeoEua3uwCUto6PC86IpRITBOs4+6i2I56K5x5b6WYGXHA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "undici-types": "~6.19.2" } @@ -13240,13 +14644,17 @@ "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/webdriverio": { "version": "9.4.1", "resolved": "https://registry.npmjs.org/webdriverio/-/webdriverio-9.4.1.tgz", "integrity": "sha512-XIPtRnxSES4CoxH2BfUY/6QzNgEgJEUjMYu7t7SJR8bVfbLRVXAA1ie9kM0MtdLs4oZ2Pr8rR8fqytsA1CjEWw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "@types/node": "^20.11.30", "@types/sinonjs__fake-timers": "^8.1.5", @@ -13293,6 +14701,8 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.10.tgz", "integrity": "sha512-/jrvh5h6NXhEauFFexRin69nA0uHJ5gwk4iDivp/DeoEua3uwCUto6PC86IpRITBOs4+6i2I56K5x5b6WYGXHA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "undici-types": "~6.19.2" } @@ -13301,12 +14711,18 @@ "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/webidl-conversions": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", - "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==" + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "dev": true, + "engines": { + "node": ">=12" + } }, "node_modules/webpack": { "version": "5.97.1", @@ -13589,26 +15005,49 @@ } }, "node_modules/whatwg-encoding": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", - "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "dev": true, "dependencies": { - "iconv-lite": "0.4.24" + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" } }, "node_modules/whatwg-mimetype": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", - "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "dev": true, + "engines": { + "node": ">=18" + } }, "node_modules/whatwg-url": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz", - "integrity": "sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.1.0.tgz", + "integrity": "sha512-jlf/foYIKywAt3x/XWKZ/3rz8OSJPiWktjmk891alJUEjiVxKX9LEO92qH3hv4aJ0mN3MWPvGMCy8jQi95xK4w==", + "dev": true, "dependencies": { - "lodash.sortby": "^4.7.0", - "tr46": "^1.0.1", - "webidl-conversions": "^4.0.2" + "tr46": "^5.0.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" } }, "node_modules/which": { @@ -13727,6 +15166,70 @@ "integrity": "sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==", "dev": true }, + "node_modules/winston": { + "version": "3.17.0", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.17.0.tgz", + "integrity": "sha512-DLiFIXYC5fMPxaRg832S6F5mJYvePtmO5G9v9IgUFPhXm9/GkXarH/TUrBAVzhTCzAj9anE/+GjrgXp/54nOgw==", + "dev": true, + "dependencies": { + "@colors/colors": "^1.6.0", + "@dabh/diagnostics": "^2.0.2", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.7.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.9.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-transport": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.9.0.tgz", + "integrity": "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A==", + "dev": true, + "dependencies": { + "logform": "^2.7.0", + "readable-stream": "^3.6.2", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-transport/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/winston/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", @@ -13826,7 +15329,9 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/ws": { "version": "8.18.0", @@ -13850,9 +15355,19 @@ } }, "node_modules/xml-name-validator": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", - "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", + "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "dev": true }, "node_modules/xmldom": { "version": "0.1.31", @@ -13920,13 +15435,17 @@ } }, "node_modules/yauzl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.2.0.tgz", + "integrity": "sha512-Ow9nuGZE+qp1u4JIPvg+uCiUr7xGQWdff7JQSk5VGYTAZMDe2q8lxJ10ygv10qmSj031Ty/6FNJpLO4o1Sgc+w==", "dev": true, + "license": "MIT", "dependencies": { "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" + "pend": "~1.2.0" + }, + "engines": { + "node": ">=12" } }, "node_modules/yauzl/node_modules/buffer-crc32": { @@ -13938,6 +15457,15 @@ "node": "*" } }, + "node_modules/ylru": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/ylru/-/ylru-1.4.0.tgz", + "integrity": "sha512-2OQsPNEmBCvXuFlIni/a+Rn+R2pHW9INm0BxXJ4hVDA8TirqMj+J/Rp9ItLatT/5pZqWwefVrTQcHpixsxnVlA==", + "dev": true, + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", @@ -13967,6 +15495,8 @@ "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-6.0.1.tgz", "integrity": "sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "archiver-utils": "^5.0.0", "compress-commons": "^6.0.2", @@ -13981,6 +15511,8 @@ "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.8.x" } @@ -13990,6 +15522,8 @@ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", @@ -14006,6 +15540,8 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "safe-buffer": "~5.2.0" } diff --git a/package.json b/package.json index 235bf26fbb..2473cb3a1e 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "url": "git+https://github.com/google/neuroglancer.git" }, "engines": { - "node": ">=20.11 <21 || >=21.2" + "node": ">=22" }, "browserslist": [ "last 2 Chrome versions", @@ -41,39 +41,53 @@ "version": "tsx ./build_tools/after-version-change.ts" }, "devDependencies": { - "@eslint/js": "^9.16.0", - "@rspack/cli": "^1.1.6", - "@rspack/core": "^1.1.6", + "@eslint/js": "^9.18.0", + "@iodigital/vite-plugin-msw": "^2.0.0", + "@playwright/browser-chromium": "^1.49.1", + "@rspack/cli": "^1.1.8", + "@rspack/core": "^1.1.8", "@types/codemirror": "5.60.15", "@types/gl-matrix": "^2.4.5", + "@types/http-server": "^0.12.4", + "@types/jsdom": "^21.1.7", "@types/lodash-es": "^4.17.12", - "@types/node": "^22.10.2", + "@types/node": "^22.10.7", "@types/nunjucks": "^3.2.6", + "@types/s3rver": "^3.7.4", "@types/yargs": "^17.0.33", - "@vitest/browser": "^2.1.8", - "@vitest/ui": "^2.1.8", + "@vitest/browser": "^3.0.2", + "@vitest/ui": "^3.0.2", + "@vitest/web-worker": "^3.0.2", + "cookie": "^1.0.2", "css-loader": "^7.1.2", - "esbuild": "^0.24.0", - "eslint": "^9.16.0", + "esbuild": "^0.24.2", + "eslint": "^9.18.0", "eslint-import-resolver-typescript": "^3.7.0", "eslint-plugin-import": "^2.31.0", "eslint-rspack-plugin": "^4.2.1", - "glob": "^11.0.0", + "express": "^4.21.2", + "glob": "^11.0.1", + "http-server": "^14.1.1", + "jsdom": "^26.0.0", + "msw": "^2.7.0", "nunjucks": "^3.2.4", + "playwright": "^1.49.1", "prettier": "3.4.2", - "ts-checker-rspack-plugin": "^1.0.3", + "s3rver": "^3.7.1", + "ts-checker-rspack-plugin": "^1.1.1", "tsx": "^4.19.2", - "typescript": "^5.7.2", - "typescript-eslint": "^8.18.0", - "vitest": "^2.1.8", - "webdriverio": "^9.4.1", + "typescript": "^5.7.3", + "typescript-eslint": "^8.20.0", + "vitest": "^3.0.2", "webpack-bundle-analyzer": "^4.10.2", "webpack-merge": "^6.0.1", - "yargs": "^17.7.2" + "yargs": "^17.7.2", + "yauzl": "^3.2.0" }, "dependencies": { "codemirror": "^5.61.1", - "core-js": "^3.39.0", + "core-js": "^3.40.0", + "crc-32": "^1.2.2", "gl-matrix": "3.1.0", "ikonate": "github:mikolajdobrucki/ikonate#a86b4107c6ec717e7877f880a930d1ccf0b59d89", "lodash-es": "^4.17.21", @@ -99,286 +113,357 @@ "#src/third_party/jpgjs/jpg.js": "./src/third_party/jpgjs/jpg.js", "#src/*.js": "./src/*.ts", "#src/*": "./src/*", + "#tests/fixtures/msw": { + "node": "./tests/fixtures/msw_node.ts", + "default": "./tests/fixtures/msw_browser.ts" + }, + "#tests/fixtures/gl": { + "node": "./tests/fixtures/gl_node.ts", + "default": "./tests/fixtures/gl_browser.ts" + }, + "#tests/*.js": "./tests/*.ts", "#testdata/*": "./testdata/*", "#datasource/boss/backend": { "neuroglancer/datasource/boss:enabled": "./src/datasource/boss/backend.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/boss:disabled": "./src/datasource/boss/backend.ts", + "neuroglancer/datasource/boss:disabled": "./src/util/false.ts", "default": "./src/datasource/boss/backend.ts" }, "#datasource/boss/async_computation": { "neuroglancer/datasource/boss:enabled": "./src/datasource/boss/async_computation.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/boss:disabled": "./src/datasource/boss/async_computation.ts", + "neuroglancer/datasource/boss:disabled": "./src/util/false.ts", "default": "./src/datasource/boss/async_computation.ts" }, "#datasource/boss/register_default": { "neuroglancer/datasource/boss:enabled": "./src/datasource/boss/register_default.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/boss:disabled": "./src/datasource/boss/register_default.ts", + "neuroglancer/datasource/boss:disabled": "./src/util/false.ts", "default": "./src/datasource/boss/register_default.ts" }, "#datasource/boss/register_credentials_provider": { "neuroglancer/python": "./src/util/false.ts", "neuroglancer/datasource/boss:enabled": "./src/datasource/boss/register_credentials_provider.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/boss:disabled": "./src/datasource/boss/register_credentials_provider.ts", + "neuroglancer/datasource/boss:disabled": "./src/util/false.ts", "default": "./src/datasource/boss/register_credentials_provider.ts" }, "#datasource/brainmaps/backend": { "neuroglancer/datasource/brainmaps:enabled": "./src/datasource/brainmaps/backend.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/brainmaps:disabled": "./src/datasource/brainmaps/backend.ts", + "neuroglancer/datasource/brainmaps:disabled": "./src/util/false.ts", "default": "./src/datasource/brainmaps/backend.ts" }, "#datasource/brainmaps/async_computation": { "neuroglancer/datasource/brainmaps:enabled": "./src/datasource/brainmaps/async_computation.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/brainmaps:disabled": "./src/datasource/brainmaps/async_computation.ts", + "neuroglancer/datasource/brainmaps:disabled": "./src/util/false.ts", "default": "./src/datasource/brainmaps/async_computation.ts" }, "#datasource/brainmaps/register_default": { "neuroglancer/datasource/brainmaps:enabled": "./src/datasource/brainmaps/register_default.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/brainmaps:disabled": "./src/datasource/brainmaps/register_default.ts", + "neuroglancer/datasource/brainmaps:disabled": "./src/util/false.ts", "default": "./src/datasource/brainmaps/register_default.ts" }, "#datasource/brainmaps/register_credentials_provider": { "neuroglancer/python": "./src/util/false.ts", "neuroglancer/datasource/brainmaps:enabled": "./src/datasource/brainmaps/register_credentials_provider.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/brainmaps:disabled": "./src/datasource/brainmaps/register_credentials_provider.ts", + "neuroglancer/datasource/brainmaps:disabled": "./src/util/false.ts", "default": "./src/datasource/brainmaps/register_credentials_provider.ts" }, "#datasource/deepzoom/backend": { "neuroglancer/datasource/deepzoom:enabled": "./src/datasource/deepzoom/backend.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/deepzoom:disabled": "./src/datasource/deepzoom/backend.ts", + "neuroglancer/datasource/deepzoom:disabled": "./src/util/false.ts", "default": "./src/datasource/deepzoom/backend.ts" }, "#datasource/deepzoom/async_computation": { "neuroglancer/datasource/deepzoom:enabled": "./src/datasource/deepzoom/async_computation.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/deepzoom:disabled": "./src/datasource/deepzoom/async_computation.ts", + "neuroglancer/datasource/deepzoom:disabled": "./src/util/false.ts", "default": "./src/datasource/deepzoom/async_computation.ts" }, "#datasource/deepzoom/register_default": { "neuroglancer/datasource/deepzoom:enabled": "./src/datasource/deepzoom/register_default.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/deepzoom:disabled": "./src/datasource/deepzoom/register_default.ts", + "neuroglancer/datasource/deepzoom:disabled": "./src/util/false.ts", "default": "./src/datasource/deepzoom/register_default.ts" }, "#datasource/dvid/backend": { "neuroglancer/datasource/dvid:enabled": "./src/datasource/dvid/backend.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/dvid:disabled": "./src/datasource/dvid/backend.ts", + "neuroglancer/datasource/dvid:disabled": "./src/util/false.ts", "default": "./src/datasource/dvid/backend.ts" }, "#datasource/dvid/async_computation": { "neuroglancer/datasource/dvid:enabled": "./src/datasource/dvid/async_computation.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/dvid:disabled": "./src/datasource/dvid/async_computation.ts", + "neuroglancer/datasource/dvid:disabled": "./src/util/false.ts", "default": "./src/datasource/dvid/async_computation.ts" }, "#datasource/dvid/register_default": { "neuroglancer/datasource/dvid:enabled": "./src/datasource/dvid/register_default.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/dvid:disabled": "./src/datasource/dvid/register_default.ts", + "neuroglancer/datasource/dvid:disabled": "./src/util/false.ts", "default": "./src/datasource/dvid/register_default.ts" }, "#datasource/dvid/register_credentials_provider": { "neuroglancer/python": "./src/util/false.ts", "neuroglancer/datasource/dvid:enabled": "./src/datasource/dvid/register_credentials_provider.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/dvid:disabled": "./src/datasource/dvid/register_credentials_provider.ts", + "neuroglancer/datasource/dvid:disabled": "./src/util/false.ts", "default": "./src/datasource/dvid/register_credentials_provider.ts" }, "#datasource/graphene/backend": { "neuroglancer/datasource/graphene:enabled": "./src/datasource/graphene/backend.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/graphene:disabled": "./src/datasource/graphene/backend.ts", + "neuroglancer/datasource/graphene:disabled": "./src/util/false.ts", "default": "./src/datasource/graphene/backend.ts" }, "#datasource/graphene/async_computation": { "neuroglancer/datasource/graphene:enabled": "./src/datasource/graphene/async_computation.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/graphene:disabled": "./src/datasource/graphene/async_computation.ts", + "neuroglancer/datasource/graphene:disabled": "./src/util/false.ts", "default": "./src/datasource/graphene/async_computation.ts" }, "#datasource/graphene/register_default": { "neuroglancer/datasource/graphene:enabled": "./src/datasource/graphene/register_default.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/graphene:disabled": "./src/datasource/graphene/register_default.ts", + "neuroglancer/datasource/graphene:disabled": "./src/util/false.ts", "default": "./src/datasource/graphene/register_default.ts" }, - "#datasource/middleauth/register_credentials_provider": { - "neuroglancer/python": "./src/util/false.ts", - "neuroglancer/datasource/middleauth:enabled": "./src/datasource/middleauth/register_credentials_provider.ts", - "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/middleauth:disabled": "./src/datasource/middleauth/register_credentials_provider.ts", - "default": "./src/datasource/middleauth/register_credentials_provider.ts" - }, "#datasource/n5/backend": { "neuroglancer/datasource/n5:enabled": "./src/datasource/n5/backend.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/n5:disabled": "./src/datasource/n5/backend.ts", + "neuroglancer/datasource/n5:disabled": "./src/util/false.ts", "default": "./src/datasource/n5/backend.ts" }, "#datasource/n5/async_computation": { "neuroglancer/datasource/n5:enabled": "./src/datasource/n5/async_computation.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/n5:disabled": "./src/datasource/n5/async_computation.ts", + "neuroglancer/datasource/n5:disabled": "./src/util/false.ts", "default": "./src/datasource/n5/async_computation.ts" }, "#datasource/n5/register_default": { "neuroglancer/datasource/n5:enabled": "./src/datasource/n5/register_default.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/n5:disabled": "./src/datasource/n5/register_default.ts", + "neuroglancer/datasource/n5:disabled": "./src/util/false.ts", "default": "./src/datasource/n5/register_default.ts" }, - "#datasource/ngauth/register_credentials_provider": { - "neuroglancer/python": "./src/util/false.ts", - "neuroglancer/datasource/ngauth:enabled": "./src/datasource/ngauth/register_credentials_provider.ts", - "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/ngauth:disabled": "./src/datasource/ngauth/register_credentials_provider.ts", - "default": "./src/datasource/ngauth/register_credentials_provider.ts" - }, "#datasource/nggraph/backend": { "neuroglancer/datasource/nggraph:enabled": "./src/datasource/nggraph/backend.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/nggraph:disabled": "./src/datasource/nggraph/backend.ts", + "neuroglancer/datasource/nggraph:disabled": "./src/util/false.ts", "default": "./src/datasource/nggraph/backend.ts" }, "#datasource/nggraph/register_default": { "neuroglancer/datasource/nggraph:enabled": "./src/datasource/nggraph/register_default.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/nggraph:disabled": "./src/datasource/nggraph/register_default.ts", + "neuroglancer/datasource/nggraph:disabled": "./src/util/false.ts", "default": "./src/datasource/nggraph/register_default.ts" }, "#datasource/nifti/backend": { "neuroglancer/datasource/nifti:enabled": "./src/datasource/nifti/backend.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/nifti:disabled": "./src/datasource/nifti/backend.ts", + "neuroglancer/datasource/nifti:disabled": "./src/util/false.ts", "default": "./src/datasource/nifti/backend.ts" }, "#datasource/nifti/register_default": { "neuroglancer/datasource/nifti:enabled": "./src/datasource/nifti/register_default.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/nifti:disabled": "./src/datasource/nifti/register_default.ts", + "neuroglancer/datasource/nifti:disabled": "./src/util/false.ts", "default": "./src/datasource/nifti/register_default.ts" }, "#datasource/obj/backend": { "neuroglancer/datasource/obj:enabled": "./src/datasource/obj/backend.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/obj:disabled": "./src/datasource/obj/backend.ts", + "neuroglancer/datasource/obj:disabled": "./src/util/false.ts", "default": "./src/datasource/obj/backend.ts" }, "#datasource/obj/async_computation": { "neuroglancer/datasource/obj:enabled": "./src/datasource/obj/async_computation.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/obj:disabled": "./src/datasource/obj/async_computation.ts", + "neuroglancer/datasource/obj:disabled": "./src/util/false.ts", "default": "./src/datasource/obj/async_computation.ts" }, "#datasource/obj/register_default": { "neuroglancer/datasource/obj:enabled": "./src/datasource/obj/register_default.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/obj:disabled": "./src/datasource/obj/register_default.ts", + "neuroglancer/datasource/obj:disabled": "./src/util/false.ts", "default": "./src/datasource/obj/register_default.ts" }, "#datasource/precomputed/backend": { "neuroglancer/datasource/precomputed:enabled": "./src/datasource/precomputed/backend.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/precomputed:disabled": "./src/datasource/precomputed/backend.ts", + "neuroglancer/datasource/precomputed:disabled": "./src/util/false.ts", "default": "./src/datasource/precomputed/backend.ts" }, "#datasource/precomputed/async_computation": { "neuroglancer/datasource/precomputed:enabled": "./src/datasource/precomputed/async_computation.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/precomputed:disabled": "./src/datasource/precomputed/async_computation.ts", + "neuroglancer/datasource/precomputed:disabled": "./src/util/false.ts", "default": "./src/datasource/precomputed/async_computation.ts" }, "#datasource/precomputed/register_default": { "neuroglancer/datasource/precomputed:enabled": "./src/datasource/precomputed/register_default.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/precomputed:disabled": "./src/datasource/precomputed/register_default.ts", + "neuroglancer/datasource/precomputed:disabled": "./src/util/false.ts", "default": "./src/datasource/precomputed/register_default.ts" }, "#datasource/python/backend": { "neuroglancer/python": "./src/datasource/python/backend.ts", "default": "./src/util/false.ts" }, + "#datasource/python/register_default": { + "neuroglancer/python": "./src/datasource/python/register_default.ts", + "default": "./src/util/false.ts" + }, "#datasource/render/backend": { "neuroglancer/datasource/render:enabled": "./src/datasource/render/backend.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/render:disabled": "./src/datasource/render/backend.ts", + "neuroglancer/datasource/render:disabled": "./src/util/false.ts", "default": "./src/datasource/render/backend.ts" }, "#datasource/render/async_computation": { "neuroglancer/datasource/render:enabled": "./src/datasource/render/async_computation.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/render:disabled": "./src/datasource/render/async_computation.ts", + "neuroglancer/datasource/render:disabled": "./src/util/false.ts", "default": "./src/datasource/render/async_computation.ts" }, "#datasource/render/register_default": { "neuroglancer/datasource/render:enabled": "./src/datasource/render/register_default.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/render:disabled": "./src/datasource/render/register_default.ts", + "neuroglancer/datasource/render:disabled": "./src/util/false.ts", "default": "./src/datasource/render/register_default.ts" }, "#datasource/vtk/backend": { "neuroglancer/datasource/vtk:enabled": "./src/datasource/vtk/backend.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/vtk:disabled": "./src/datasource/vtk/backend.ts", + "neuroglancer/datasource/vtk:disabled": "./src/util/false.ts", "default": "./src/datasource/vtk/backend.ts" }, "#datasource/vtk/async_computation": { "neuroglancer/datasource/vtk:enabled": "./src/datasource/vtk/async_computation.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/vtk:disabled": "./src/datasource/vtk/async_computation.ts", + "neuroglancer/datasource/vtk:disabled": "./src/util/false.ts", "default": "./src/datasource/vtk/async_computation.ts" }, "#datasource/vtk/register_default": { "neuroglancer/datasource/vtk:enabled": "./src/datasource/vtk/register_default.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/vtk:disabled": "./src/datasource/vtk/register_default.ts", + "neuroglancer/datasource/vtk:disabled": "./src/util/false.ts", "default": "./src/datasource/vtk/register_default.ts" }, "#datasource/zarr/backend": { "neuroglancer/datasource/zarr:enabled": "./src/datasource/zarr/backend.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/zarr:disabled": "./src/datasource/zarr/backend.ts", + "neuroglancer/datasource/zarr:disabled": "./src/util/false.ts", "default": "./src/datasource/zarr/backend.ts" }, "#datasource/zarr/async_computation": { "neuroglancer/datasource/zarr:enabled": "./src/datasource/zarr/async_computation.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/zarr:disabled": "./src/datasource/zarr/async_computation.ts", + "neuroglancer/datasource/zarr:disabled": "./src/util/false.ts", "default": "./src/datasource/zarr/async_computation.ts" }, "#datasource/zarr/register_default": { "neuroglancer/datasource/zarr:enabled": "./src/datasource/zarr/register_default.ts", "neuroglancer/datasource:none_by_default": "./src/util/false.ts", - "neuroglancer/datasource/zarr:disabled": "./src/datasource/zarr/register_default.ts", + "neuroglancer/datasource/zarr:disabled": "./src/util/false.ts", "default": "./src/datasource/zarr/register_default.ts" }, + "#kvstore/byte_range/register": { + "neuroglancer/kvstore/byte_range:enabled": "./src/kvstore/byte_range/register.ts", + "neuroglancer/kvstore:none_by_default": "./src/util/false.ts", + "neuroglancer/kvstore/byte_range:disabled": "./src/util/false.ts", + "default": "./src/kvstore/byte_range/register.ts" + }, + "#kvstore/gcs/register": { + "neuroglancer/kvstore/gcs:enabled": "./src/kvstore/gcs/register.ts", + "neuroglancer/kvstore:none_by_default": "./src/util/false.ts", + "neuroglancer/kvstore/gcs:disabled": "./src/util/false.ts", + "default": "./src/kvstore/gcs/register.ts" + }, + "#kvstore/gzip/register": { + "neuroglancer/kvstore/gzip:enabled": "./src/kvstore/gzip/register.ts", + "neuroglancer/kvstore:none_by_default": "./src/util/false.ts", + "neuroglancer/kvstore/gzip:disabled": "./src/util/false.ts", + "default": "./src/kvstore/gzip/register.ts" + }, + "#kvstore/http/register": { + "neuroglancer/kvstore/http:enabled": "./src/kvstore/http/register.ts", + "neuroglancer/kvstore:none_by_default": "./src/util/false.ts", + "neuroglancer/kvstore/http:disabled": "./src/util/false.ts", + "default": "./src/kvstore/http/register.ts" + }, + "#kvstore/middleauth/register": { + "neuroglancer/kvstore/middleauth:enabled": "./src/kvstore/middleauth/register.ts", + "neuroglancer/kvstore:none_by_default": "./src/util/false.ts", + "neuroglancer/kvstore/middleauth:disabled": "./src/util/false.ts", + "default": "./src/kvstore/middleauth/register.ts" + }, + "#kvstore/middleauth/register_credentials_provider": { + "neuroglancer/python": "./src/util/false.ts", + "neuroglancer/kvstore/middleauth:enabled": "./src/kvstore/middleauth/register_credentials_provider.ts", + "neuroglancer/kvstore:none_by_default": "./src/util/false.ts", + "neuroglancer/kvstore/middleauth:disabled": "./src/util/false.ts", + "default": "./src/kvstore/middleauth/register_credentials_provider.ts" + }, + "#kvstore/ngauth/register": { + "neuroglancer/kvstore/ngauth:enabled": "./src/kvstore/ngauth/register.ts", + "neuroglancer/kvstore:none_by_default": "./src/util/false.ts", + "neuroglancer/kvstore/ngauth:disabled": "./src/util/false.ts", + "default": "./src/kvstore/ngauth/register.ts" + }, + "#kvstore/ngauth/register_credentials_provider": { + "neuroglancer/python": "./src/util/false.ts", + "neuroglancer/kvstore/ngauth:enabled": "./src/kvstore/ngauth/register_credentials_provider.ts", + "neuroglancer/kvstore:none_by_default": "./src/util/false.ts", + "neuroglancer/kvstore/ngauth:disabled": "./src/util/false.ts", + "default": "./src/kvstore/ngauth/register_credentials_provider.ts" + }, + "#kvstore/s3/register": { + "neuroglancer/kvstore/s3:enabled": "./src/kvstore/s3/register.ts", + "neuroglancer/kvstore:none_by_default": "./src/util/false.ts", + "neuroglancer/kvstore/s3:disabled": "./src/util/false.ts", + "default": "./src/kvstore/s3/register.ts" + }, + "#kvstore/zip/register_frontend": { + "neuroglancer/kvstore/zip:enabled": "./src/kvstore/zip/register_frontend.ts", + "neuroglancer/kvstore:none_by_default": "./src/util/false.ts", + "neuroglancer/kvstore/zip:disabled": "./src/util/false.ts", + "default": "./src/kvstore/zip/register_frontend.ts" + }, + "#kvstore/zip/register_backend": { + "neuroglancer/kvstore/zip:enabled": "./src/kvstore/zip/register_backend.ts", + "neuroglancer/kvstore:none_by_default": "./src/util/false.ts", + "neuroglancer/kvstore/zip:disabled": "./src/util/false.ts", + "default": "./src/kvstore/zip/register_backend.ts" + }, "#layer/annotation": { "neuroglancer/layer/annotation:enabled": "./src/layer/annotation/index.ts", "neuroglancer/layer:none_by_default": "./src/util/false.ts", + "neuroglancer/layer/annotation:disabled": "./src/util/false.ts", "default": "./src/layer/annotation/index.ts" }, "#layer/image": { "neuroglancer/layer/image:enabled": "./src/layer/image/index.ts", "neuroglancer/layer:none_by_default": "./src/util/false.ts", + "neuroglancer/layer/image:disabled": "./src/util/false.ts", "default": "./src/layer/image/index.ts" }, "#layer/segmentation": { "neuroglancer/layer/segmentation:enabled": "./src/layer/segmentation/index.ts", "neuroglancer/layer:none_by_default": "./src/util/false.ts", + "neuroglancer/layer/segmentation:disabled": "./src/util/false.ts", "default": "./src/layer/segmentation/index.ts" }, "#layer/single_mesh": { "neuroglancer/layer/single_mesh:enabled": "./src/layer/single_mesh/index.ts", "neuroglancer/layer:none_by_default": "./src/util/false.ts", + "neuroglancer/layer/single_mesh:disabled": "./src/util/false.ts", "default": "./src/layer/single_mesh/index.ts" }, "#main": { diff --git a/pyproject.toml b/pyproject.toml index 3474d58d80..c78059e51e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ exclude = [ "/guide_video_recorder/", "^docs/", "^build/", + "^testdata/", ] [[tool.mypy.overrides]] diff --git a/python/neuroglancer/write_annotations.py b/python/neuroglancer/write_annotations.py index 6e1d3f87c6..8a204e0fc1 100644 --- a/python/neuroglancer/write_annotations.py +++ b/python/neuroglancer/write_annotations.py @@ -174,11 +174,11 @@ def _add_two_point_obj( def _add_obj(self, coords: Sequence[float], id: Optional[int], **kwargs): encoded = np.zeros(shape=(), dtype=self.dtype) - encoded[()]["geometry"] = coords + encoded[()]["geometry"] = coords # type: ignore[call-overload] for i, p in enumerate(self.properties): if p.id in kwargs: - encoded[()][f"property{i}"] = kwargs.pop(p.id) + encoded[()][f"property{i}"] = kwargs.pop(p.id) # type: ignore[call-overload] related_ids = [] for relationship in self.relationships: diff --git a/python/tests/on_demand_mesh_generator_test.py b/python/tests/on_demand_mesh_generator_test.py index b1f265d043..8e28af49b5 100644 --- a/python/tests/on_demand_mesh_generator_test.py +++ b/python/tests/on_demand_mesh_generator_test.py @@ -20,7 +20,7 @@ import pytest from neuroglancer import local_volume, test_util, viewer_state -testdata_dir = os.path.join(os.path.dirname(__file__), "..", "testdata", "mesh") +testdata_dir = os.path.join(os.path.dirname(__file__), "..", "..", "testdata", "mesh") @pytest.mark.xfail( diff --git a/python/tests/zarr_test.py b/python/tests/zarr_test.py index 14e1fb60f4..860c7dbadb 100644 --- a/python/tests/zarr_test.py +++ b/python/tests/zarr_test.py @@ -19,7 +19,9 @@ import numpy as np import pytest -TEST_DATA_DIR = pathlib.Path(__file__).parent.parent / "testdata" +TEST_DATA_DIR = ( + pathlib.Path(__file__).parent.parent.parent / "testdata" / "datasource" / "zarr" +) @pytest.mark.parametrize( diff --git a/setup.py b/setup.py index 7fd8e08317..35c0fc3cad 100755 --- a/setup.py +++ b/setup.py @@ -130,7 +130,7 @@ class BundleClientCommand( ): editable_mode: bool = False - user_options = [ + user_options = [ # type: ignore[assignment] ( "client-bundle-type=", None, diff --git a/src/annotation/backend.ts b/src/annotation/backend.ts index aa7f43ce49..1c8444e9a1 100644 --- a/src/annotation/backend.ts +++ b/src/annotation/backend.ts @@ -170,8 +170,8 @@ class AnnotationMetadataChunkSource extends ChunkSource { return chunk; } - download(chunk: AnnotationMetadataChunk, abortSignal: AbortSignal) { - return this.parent!.downloadMetadata(chunk, abortSignal); + download(chunk: AnnotationMetadataChunk, signal: AbortSignal) { + return this.parent!.downloadMetadata(chunk, signal); } } @@ -205,11 +205,11 @@ class AnnotationSubsetGeometryChunkSource extends ChunkSource { } return chunk; } - download(chunk: AnnotationSubsetGeometryChunk, abortSignal: AbortSignal) { + download(chunk: AnnotationSubsetGeometryChunk, signal: AbortSignal) { return this.parent!.downloadSegmentFilteredGeometry( chunk, this.relationshipIndex, - abortSignal, + signal, ); } } @@ -220,12 +220,12 @@ export interface AnnotationSource { // TypeScript supports mixins with abstract classes. downloadMetadata( chunk: AnnotationMetadataChunk, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise; downloadSegmentFilteredGeometry( chunk: AnnotationSubsetGeometryChunk, relationshipIndex: number, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise; } diff --git a/src/async_computation/request.ts b/src/async_computation/request.ts index f641b6fefc..9497ffeb78 100644 --- a/src/async_computation/request.ts +++ b/src/async_computation/request.ts @@ -85,14 +85,14 @@ export function requestAsyncComputation< Signature extends (...args: any) => any, >( request: AsyncComputationSpec, - abortSignal: AbortSignal | undefined, + signal: AbortSignal | undefined, transfer: Transferable[] | undefined, ...args: Parameters ): Promise> { const id = nextTaskId++; const msg = { t: request.id, id, args: args }; - abortSignal?.throwIfAborted(); + signal?.throwIfAborted(); const promise = new Promise>((resolve, reject) => { tasks.set(id, { resolve, reject }); @@ -102,16 +102,16 @@ export function requestAsyncComputation< freeWorkers.pop()!.postMessage(msg, transfer as Transferable[]); } else { let cleanup: (() => void) | undefined; - if (abortSignal !== undefined) { + if (signal !== undefined) { function abortHandler() { pendingTasks.delete(id); const task = tasks.get(id)!; tasks.delete(id); - task.reject(abortSignal!.reason); + task.reject(signal!.reason); } - abortSignal.addEventListener("abort", abortHandler, { once: true }); + signal.addEventListener("abort", abortHandler, { once: true }); cleanup = () => { - abortSignal.removeEventListener("abort", abortHandler); + signal.removeEventListener("abort", abortHandler); }; } pendingTasks.set(id, { msg, transfer, cleanup }); diff --git a/src/chunk_manager/backend.ts b/src/chunk_manager/backend.ts index 70f02db41d..62457ac015 100644 --- a/src/chunk_manager/backend.ts +++ b/src/chunk_manager/backend.ts @@ -412,12 +412,12 @@ export interface ChunkSourceBase { * Note: This method must be defined by subclasses. * * @param chunk Chunk to download. - * @param abortSignal Used to abort download. + * @param signal Used to abort download. * * TODO(jbms): Move this back to the class definition above and declare this abstract once mixins * are compatible with abstract classes. */ - download(chunk: Chunk, abortSignal: AbortSignal): Promise; + download(chunk: Chunk, signal: AbortSignal): Promise; } export class ChunkSource extends ChunkSourceBase { @@ -462,7 +462,7 @@ function startChunkDownload(chunk: Chunk) { function cancelChunkDownload(chunk: Chunk) { const controller = chunk.downloadAbortController!; chunk.downloadAbortController = undefined; - controller.abort(); + controller.abort(new DOMException("chunk download cancelled", "AbortError")); } class ChunkPriorityQueue { @@ -890,6 +890,11 @@ export class ChunkQueueManager extends SharedObjectCounterpart { this.scheduleUpdate(); } + markRecentlyUsed(chunk: Chunk) { + this.removeChunkFromQueues_(chunk); + this.addChunkToQueues_(chunk); + } + private processGPUPromotions_() { const queueManager = this; function evictFromGPUMemory(chunk: Chunk) { diff --git a/src/chunk_manager/frontend.ts b/src/chunk_manager/frontend.ts index 8d552cbefc..3df76ce075 100644 --- a/src/chunk_manager/frontend.ts +++ b/src/chunk_manager/frontend.ts @@ -196,9 +196,9 @@ export class ChunkQueueManager extends SharedObject { } private handleFetch_(source: ChunkSource, update: any) { - const { resolve, reject, abortSignal } = update.promise; - if (abortSignal.aborted) { - reject(abortSignal.reason); + const { resolve, reject, signal } = update.promise; + if (signal.aborted) { + reject(signal.reason); return; } @@ -348,15 +348,12 @@ registerRPC("Chunk.update", function (x) { updateChunk(this, x); }); -registerPromiseRPC( - "Chunk.retrieve", - function (x, abortSignal): RPCPromise { - return new Promise<{ value: any }>((resolve, reject) => { - x.promise = { resolve, reject, abortSignal }; - updateChunk(this, x); - }); - }, -); +registerPromiseRPC("Chunk.retrieve", function (x, signal): RPCPromise { + return new Promise<{ value: any }>((resolve, reject) => { + x.promise = { resolve, reject, signal }; + updateChunk(this, x); + }); +}); registerRPC(CHUNK_LAYER_STATISTICS_RPC_ID, function (x) { const chunkManager = this.get(x.id) as ChunkManager; diff --git a/src/chunk_manager/generic_file_source.ts b/src/chunk_manager/generic_file_source.ts index 6029a19814..308e18ffe4 100644 --- a/src/chunk_manager/generic_file_source.ts +++ b/src/chunk_manager/generic_file_source.ts @@ -21,243 +21,25 @@ import type { ChunkManager } from "#src/chunk_manager/backend.js"; import { Chunk, ChunkSourceBase } from "#src/chunk_manager/backend.js"; -import { ChunkPriorityTier, ChunkState } from "#src/chunk_manager/base.js"; -import { raceWithAbort, SharedAbortController } from "#src/util/abort.js"; -import type { Borrowed, Owned } from "#src/util/disposable.js"; +import { ChunkState } from "#src/chunk_manager/base.js"; +import type { SharedKvStoreContextCounterpart } from "#src/kvstore/backend.js"; +import type { ReadResponse } from "#src/kvstore/index.js"; +import type { Owned } from "#src/util/disposable.js"; import { stableStringify } from "#src/util/json.js"; +import type { AsyncMemoize } from "#src/util/memoize.js"; +import { asyncMemoizeWithProgress } from "#src/util/memoize.js"; import { getObjectId } from "#src/util/object_id.js"; -import type { SpecialProtocolCredentialsProvider } from "#src/util/special_protocol_request.js"; -import { fetchSpecialOk } from "#src/util/special_protocol_request.js"; - -export type PriorityGetter = () => { - priorityTier: ChunkPriorityTier; - priority: number; -}; - -interface FileDataRequester { - resolve: (data: Data) => void; - reject: (error: any) => void; - getPriority: PriorityGetter; - cleanup: () => void; -} - -class GenericSharedDataChunk extends Chunk { - decodedKey?: Key; - data?: Data; - requesters?: Set>; - - initialize(key: string) { - super.initialize(key); - this.requesters = new Set>(); - } - - downloadSucceeded() { - super.downloadSucceeded(); - const { requesters, data } = this; - this.requesters = undefined; - for (const requester of requesters!) { - requester.resolve(data!); - } - } - - downloadFailed(error: any) { - super.downloadFailed(error); - const { requesters } = this; - this.requesters = undefined; - for (const requester of requesters!) { - requester.reject(error); - } - } - - freeSystemMemory() { - this.data = undefined; - } -} - -export interface GenericSharedDataSourceOptions { - encodeKey?: (key: Key) => string; - download: ( - key: Key, - abortSignal: AbortSignal, - ) => Promise<{ size: number; data: Data }>; - sourceQueueLevel?: number; -} - -export class GenericSharedDataSource extends ChunkSourceBase { - declare chunks: Map>; - - private encodeKeyFunction: (key: Key) => string; - - private downloadFunction: ( - key: Key, - abortSignal: AbortSignal, - ) => Promise<{ size: number; data: Data }>; - - constructor( - chunkManager: Owned, - options: GenericSharedDataSourceOptions, - ) { - super(chunkManager); - this.registerDisposer(chunkManager); - const { encodeKey = stableStringify } = options; - this.downloadFunction = options.download; - this.encodeKeyFunction = encodeKey; - const { sourceQueueLevel = 0 } = options; - this.sourceQueueLevel = sourceQueueLevel; - - // This source is unusual in that it updates its own chunk priorities. - this.registerDisposer( - this.chunkManager.recomputeChunkPrioritiesLate.add(() => { - this.updateChunkPriorities(); - }), - ); - } - - updateChunkPriorities() { - const { chunkManager } = this; - for (const chunk of this.chunks.values()) { - const { requesters } = chunk; - if (requesters !== undefined) { - for (const requester of requesters) { - const { priorityTier, priority } = requester.getPriority(); - if (priorityTier === ChunkPriorityTier.RECENT) continue; - chunkManager.requestChunk( - chunk, - priorityTier, - priority, - ChunkState.SYSTEM_MEMORY_WORKER, - ); - } - } - } - } - - async download( - chunk: GenericSharedDataChunk, - abortSignal: AbortSignal, - ) { - const { size, data } = await this.downloadFunction( - chunk.decodedKey!, - abortSignal, - ); - chunk.systemMemoryBytes = size; - chunk.data = data; - } - - /** - * Precondition: priorityTier <= ChunkPriorityTier.LAST_ORDERED_TIER - */ - getData(key: Key, getPriority: PriorityGetter, abortSignal: AbortSignal) { - const encodedKey = this.encodeKeyFunction(key); - let chunk = this.chunks.get(encodedKey); - if (chunk === undefined) { - chunk = this.getNewChunk_>( - GenericSharedDataChunk, - ); - chunk.decodedKey = key; - chunk.initialize(encodedKey); - this.addChunk(chunk); - } - return new Promise((resolve, reject) => { - // If the data is already available or the request has already failed, resolve/reject the - // promise immediately. - switch (chunk!.state) { - case ChunkState.FAILED: - reject(chunk!.error); - return; - - case ChunkState.SYSTEM_MEMORY_WORKER: - resolve(chunk!.data!); - return; - } - function handleAbort() { - const { requesters } = chunk!; - if (requesters !== undefined) { - requesters.delete(requester); - chunk!.chunkManager!.scheduleUpdateChunkPriorities(); - } - reject(abortSignal.reason); - } - - const requester: FileDataRequester = { - resolve, - reject, - getPriority, - cleanup: () => abortSignal.removeEventListener("abort", handleAbort), - }; - chunk!.requesters!.add(requester); - abortSignal.addEventListener("abort", handleAbort, { once: true }); - this.chunkManager.scheduleUpdateChunkPriorities(); - }); - } - - static get( - chunkManager: Borrowed, - memoizeKey: string, - options: GenericSharedDataSourceOptions, - ) { - return chunkManager.memoize.get( - `getFileSource:${memoizeKey}`, - () => new GenericSharedDataSource(chunkManager.addRef(), options), - ); - } - - static getData( - chunkManager: Borrowed, - memoizeKey: string, - options: GenericSharedDataSourceOptions, - key: Key, - getPriority: PriorityGetter, - abortSignal: AbortSignal, - ) { - const source = GenericSharedDataSource.get( - chunkManager, - memoizeKey, - options, - ); - const result = source.getData(key, getPriority, abortSignal); - source.dispose(); - return result; - } - - static getUrl( - chunkManager: Borrowed, - credentialsProvider: SpecialProtocolCredentialsProvider, - decodeFunction: ( - buffer: ArrayBuffer, - abortSignal: AbortSignal, - ) => Promise<{ size: number; data: Data }>, - url: string, - getPriority: PriorityGetter, - abortSignal: AbortSignal, - ) { - return GenericSharedDataSource.getData( - chunkManager, - `${getObjectId(decodeFunction)}`, - { - download: (url: string, abortSignal: AbortSignal) => - fetchSpecialOk(credentialsProvider, url, { signal: abortSignal }) - .then((response) => response.arrayBuffer()) - .then((response) => decodeFunction(response, abortSignal)), - }, - url, - getPriority, - abortSignal, - ); - } -} +import type { ProgressOptions } from "#src/util/progress_listener.js"; class AsyncCacheChunk extends Chunk { - promise: Promise | undefined; - outstandingRequests: number = 0; - sharedAbortController: SharedAbortController | undefined; + asyncMemoize: AsyncMemoize | undefined; initialize(key: string) { super.initialize(key); } freeSystemMemory() { - this.promise = undefined; + this.asyncMemoize = undefined; } } @@ -265,7 +47,7 @@ export interface SimpleAsyncCacheOptions { encodeKey?: (key: Key) => string; get: ( key: Key, - abortSignal: AbortSignal, + progressOptions: ProgressOptions, ) => Promise<{ size: number; data: Value }>; } @@ -284,10 +66,10 @@ export class SimpleAsyncCache extends ChunkSourceBase { encodeKeyFunction: (key: Key) => string; downloadFunction: ( key: Key, - abortSignal: AbortSignal, + progressOptions: ProgressOptions, ) => Promise<{ size: number; data: Value }>; - get(key: Key, abortSignal?: AbortSignal): Promise { + get(key: Key, options: Partial): Promise { const encodedKey = this.encodeKeyFunction(key); let chunk = this.chunks.get(encodedKey); if (chunk === undefined) { @@ -295,23 +77,12 @@ export class SimpleAsyncCache extends ChunkSourceBase { chunk.initialize(encodedKey); this.addChunk(chunk); } - if ( - chunk.promise === undefined || - chunk.sharedAbortController?.signal.aborted - ) { - let completed = false; - const sharedAbortController = (chunk!.sharedAbortController = - new SharedAbortController()); - sharedAbortController.signal.addEventListener("abort", () => { - if (!completed) { - chunk!.promise = undefined; - } - }); - chunk.promise = (async () => { + if (chunk.asyncMemoize === undefined) { + chunk.asyncMemoize = asyncMemoizeWithProgress(async (progressOptions) => { try { const { data, size } = await this.downloadFunction( key, - sharedAbortController.signal, + progressOptions, ); chunk.systemMemoryBytes = size; chunk!.queueManager.updateChunkState( @@ -322,15 +93,13 @@ export class SimpleAsyncCache extends ChunkSourceBase { } catch (e) { chunk!.queueManager.updateChunkState(chunk!, ChunkState.FAILED); throw e; - } finally { - completed = true; - sharedAbortController[Symbol.dispose](); } - })(); + }); + } + if (chunk.state === ChunkState.SYSTEM_MEMORY) { + chunk.chunkManager.queueManager.markRecentlyUsed(chunk); } - chunk!.sharedAbortController!.addConsumer(abortSignal); - chunk!.sharedAbortController!.start(); - return raceWithAbort(chunk.promise, abortSignal); + return chunk.asyncMemoize(options); } } @@ -344,3 +113,38 @@ export function makeSimpleAsyncCache( () => new SimpleAsyncCache(chunkManager.addRef(), options), ); } + +export function getCachedDecodedUrl( + sharedKvStoreContext: SharedKvStoreContextCounterpart, + url: string, + decodeFunction: ( + readResponse: ReadResponse | undefined, + options: ProgressOptions, + ) => Promise<{ size: number; data: Data }>, + options: Partial, +): Promise { + const cache = sharedKvStoreContext.chunkManager.memoize.get( + `getCachedDecodedUrl:${getObjectId(decodeFunction)}`, + () => { + const cache = new SimpleAsyncCache( + sharedKvStoreContext.chunkManager.addRef(), + { + get: async (url: string, options: ProgressOptions) => { + const readResponse = await sharedKvStoreContext.kvStoreContext.read( + url, + options, + ); + try { + return decodeFunction(readResponse, options); + } catch (e) { + throw new Error("Error reading ${url}", { cause: e }); + } + }, + }, + ); + cache.registerDisposer(sharedKvStoreContext.addRef()); + return cache; + }, + ); + return cache.get(url, options); +} diff --git a/src/chunk_worker.bundle.js b/src/chunk_worker.bundle.js index a5cb4b9bbc..a463171535 100644 --- a/src/chunk_worker.bundle.js +++ b/src/chunk_worker.bundle.js @@ -4,9 +4,11 @@ import "#src/util/polyfills.js"; import "#src/shared_watchable_value.js"; import "#src/chunk_manager/backend.js"; +import "#src/kvstore/backend.js"; import "#src/sliceview/backend.js"; import "#src/perspective_view/backend.js"; import "#src/volume_rendering/backend.js"; import "#src/annotation/backend.js"; import "#src/datasource/enabled_backend_modules.js"; +import "#src/kvstore/enabled_backend_modules.js"; import "#src/worker_rpc_context.js"; diff --git a/src/credentials_provider/default_manager.ts b/src/credentials_provider/default_manager.ts index dea60b764c..c95fb6a874 100644 --- a/src/credentials_provider/default_manager.ts +++ b/src/credentials_provider/default_manager.ts @@ -18,7 +18,23 @@ * @file CredentialsManager for globally registering a CredentialsProvider */ -import { CachingMapBasedCredentialsManager } from "#src/credentials_provider/index.js"; +import { + CachingMapBasedCredentialsManager, + type ProviderGetter, +} from "#src/credentials_provider/index.js"; -export const defaultCredentialsManager = - new CachingMapBasedCredentialsManager(); +const providers = new Map>(); +export function registerDefaultCredentialsProvider( + key: string, + providerGetter: ProviderGetter, +) { + providers.set(key, providerGetter); +} + +export function getDefaultCredentialsManager() { + const manager = new CachingMapBasedCredentialsManager(); + for (const [key, provider] of providers) { + manager.register(key, provider); + } + return manager; +} diff --git a/src/credentials_provider/http_request.ts b/src/credentials_provider/http_request.ts index d3998a66f5..bdf8bec3f0 100644 --- a/src/credentials_provider/http_request.ts +++ b/src/credentials_provider/http_request.ts @@ -18,18 +18,20 @@ import type { CredentialsProvider, CredentialsWithGeneration, } from "#src/credentials_provider/index.js"; +import type { FetchOk } from "#src/util/http_request.js"; import { fetchOk, HttpError, pickDelay } from "#src/util/http_request.js"; +import type { ProgressListener } from "#src/util/progress_listener.js"; const maxCredentialsAttempts = 3; -export async function fetchWithCredentials( +export async function fetchOkWithCredentials( credentialsProvider: CredentialsProvider, input: RequestInfo | ((credentials: Credentials) => RequestInfo), - init: RequestInit, + init: RequestInit & { progressListener?: ProgressListener }, applyCredentials: ( credentials: Credentials, - requestInit: RequestInit, - ) => RequestInit, + requestInit: RequestInit & { progressListener?: ProgressListener }, + ) => RequestInit & { progressListener?: ProgressListener }, errorHandler: (httpError: HttpError, credentials: Credentials) => "refresh", ): Promise { let credentials: CredentialsWithGeneration | undefined; @@ -43,10 +45,10 @@ export async function fetchWithCredentials( setTimeout(resolve, pickDelay(credentialsAttempt - 2)), ); } - credentials = await credentialsProvider.get( - credentials, - init.signal ?? undefined, - ); + credentials = await credentialsProvider.get(credentials, { + signal: init.signal ?? undefined, + progressListener: init.progressListener, + }); try { return await fetchOk( typeof input === "function" ? input(credentials.credentials) : input, @@ -63,3 +65,21 @@ export async function fetchWithCredentials( } } } + +export function fetchOkWithCredentialsAdapter( + credentialsProvider: CredentialsProvider, + applyCredentials: ( + credentials: Credentials, + requestInit: RequestInit & { progressListener?: ProgressListener }, + ) => RequestInit & { progressListener?: ProgressListener }, + errorHandler: (httpError: HttpError, credentials: Credentials) => "refresh", +): FetchOk { + return (input, init = {}) => + fetchOkWithCredentials( + credentialsProvider, + input, + init, + applyCredentials, + errorHandler, + ); +} diff --git a/src/credentials_provider/index.ts b/src/credentials_provider/index.ts index 41ec25ede0..1290e24036 100644 --- a/src/credentials_provider/index.ts +++ b/src/credentials_provider/index.ts @@ -18,10 +18,11 @@ * @file Generic facility for providing authentication/authorization credentials. */ -import { raceWithAbort, SharedAbortController } from "#src/util/abort.js"; import type { Owned } from "#src/util/disposable.js"; import { RefCounted } from "#src/util/disposable.js"; -import { StringMemoize } from "#src/util/memoize.js"; +import type { AsyncMemoizeWithProgress } from "#src/util/memoize.js"; +import { asyncMemoizeWithProgress, StringMemoize } from "#src/util/memoize.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; /** * Wraps an arbitrary JSON credentials object with a generation number. @@ -42,68 +43,50 @@ export abstract class CredentialsProvider extends RefCounted { */ abstract get: ( invalidCredentials?: CredentialsWithGeneration, - abortSignal?: AbortSignal | undefined, + options?: Partial, ) => Promise>; } export function makeCachedCredentialsGetter( getUncached: ( invalidCredentials: CredentialsWithGeneration | undefined, - abortSignal: AbortSignal, + options: ProgressOptions, ) => Promise>, ) { let cachedCredentials: CredentialsWithGeneration | undefined; let pendingCredentials: - | Promise> + | AsyncMemoizeWithProgress> | undefined; - let pendingAbortController: SharedAbortController | undefined; - return ( + return async ( invalidCredentials?: CredentialsWithGeneration, - abortSignal?: AbortSignal, + options?: Partial, ) => { + // Check if a new credential request needs to be made. if ( - pendingCredentials !== undefined && - (cachedCredentials === undefined || - invalidCredentials === undefined || - cachedCredentials.generation !== invalidCredentials.generation) + pendingCredentials === undefined || + (invalidCredentials !== undefined && + cachedCredentials?.generation === invalidCredentials.generation) ) { - if (cachedCredentials === undefined) { - pendingAbortController!.addConsumer(abortSignal); - } - return raceWithAbort(pendingCredentials, abortSignal); + cachedCredentials = undefined; + pendingCredentials = asyncMemoizeWithProgress(async (progressOptions) => { + cachedCredentials = await getUncached( + invalidCredentials, + progressOptions, + ); + return cachedCredentials; + }); } - cachedCredentials = undefined; - pendingAbortController = new SharedAbortController(); - pendingCredentials = getUncached( - invalidCredentials, - pendingAbortController.signal, - ).then( - (credentials) => { - cachedCredentials = credentials; - pendingAbortController![Symbol.dispose](); - pendingAbortController = undefined; - return credentials; - }, - (reason) => { - pendingAbortController![Symbol.dispose](); - if (pendingAbortController?.signal.aborted) { - pendingAbortController = undefined; - pendingCredentials = undefined; - } - throw reason; - }, - ); - return pendingCredentials; + return pendingCredentials(options ?? {}); }; } export function makeCredentialsGetter( - getWithoutGeneration: (abortSignal: AbortSignal) => Promise, + getWithoutGeneration: (options: ProgressOptions) => Promise, ) { let generation = 0; return makeCachedCredentialsGetter( - (_invalidCredentials, abortSignal) => - getWithoutGeneration(abortSignal).then((credentials) => ({ + (_invalidCredentials, options) => + getWithoutGeneration(options).then((credentials) => ({ generation: ++generation, credentials, })), @@ -129,13 +112,7 @@ export type ProviderGetter = ( * CredentialsManager that supports registration. */ export class MapBasedCredentialsManager implements CredentialsManager { - providers = new Map< - string, - ( - parameters: any, - credentialsManager: CredentialsManager, - ) => Owned> - >(); + providers = new Map>(); topLevelManager: CredentialsManager = this; register( key: string, @@ -213,7 +190,10 @@ export class AnonymousFirstCredentialsProvider< } get = makeCachedCredentialsGetter( - (invalidCredentials?: CredentialsWithGeneration) => { + ( + invalidCredentials: CredentialsWithGeneration | undefined, + options: ProgressOptions, + ) => { if (this.anonymous && invalidCredentials === undefined) { return Promise.resolve({ generation: -10, @@ -221,7 +201,7 @@ export class AnonymousFirstCredentialsProvider< }); } this.anonymous = false; - return this.baseProvider.get(invalidCredentials); + return this.baseProvider.get(invalidCredentials, options); }, ); } diff --git a/src/credentials_provider/interactive_credentials_provider.ts b/src/credentials_provider/interactive_credentials_provider.ts index 9f66c18d70..b10e7f7bdd 100644 --- a/src/credentials_provider/interactive_credentials_provider.ts +++ b/src/credentials_provider/interactive_credentials_provider.ts @@ -22,15 +22,15 @@ export function getCredentialsWithStatus( description: string; requestDescription?: string; supportsImmediate?: boolean; - get: (abortSignal: AbortSignal, immediate: boolean) => Promise; + get: (signal: AbortSignal, immediate: boolean) => Promise; }, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise { const { requestDescription = "login" } = options; const status = new StatusMessage(/*delay=*/ true); let abortController: AbortController | undefined; return new Promise((resolve, reject) => { - const disposeAbortCallback = scopedAbortCallback(abortSignal, (reason) => { + const disposeAbortCallback = scopedAbortCallback(signal, (reason) => { if (abortController !== undefined) { abortController.abort(reason); abortController = undefined; @@ -92,6 +92,7 @@ export function getCredentialsWithStatus( } else { writeLoginStatus(); status.setVisible(true); + status.setModal(true); } }); } diff --git a/src/credentials_provider/oauth2.ts b/src/credentials_provider/oauth2.ts index 2eec784848..3fee4084fa 100644 --- a/src/credentials_provider/oauth2.ts +++ b/src/credentials_provider/oauth2.ts @@ -14,8 +14,12 @@ * limitations under the License. */ -import { fetchWithCredentials } from "#src/credentials_provider/http_request.js"; +import { + fetchOkWithCredentials, + fetchOkWithCredentialsAdapter, +} from "#src/credentials_provider/http_request.js"; import type { CredentialsProvider } from "#src/credentials_provider/index.js"; +import type { FetchOk, HttpError } from "#src/util/http_request.js"; import { fetchOk } from "#src/util/http_request.js"; /** @@ -27,7 +31,41 @@ export interface OAuth2Credentials { email?: string; } -export function fetchWithOAuth2Credentials( +function applyCredentials( + credentials: OAuth2Credentials, + init: RequestInit, +): RequestInit { + if (!credentials.accessToken) return init; + const headers = new Headers(init.headers); + headers.set( + "Authorization", + `${credentials.tokenType} ${credentials.accessToken}`, + ); + return { ...init, headers }; +} + +function errorHandler( + error: HttpError, + credentials: OAuth2Credentials, +): "refresh" { + const { status } = error; + if (status === 401) { + // 401: Authorization needed. OAuth2 token may have expired. + return "refresh"; + } + if (status === 403 && !credentials.accessToken) { + // Anonymous access denied. Request credentials. + return "refresh"; + } + if (error instanceof Error && credentials.email !== undefined) { + error.message += ` (Using credentials for ${JSON.stringify( + credentials.email, + )})`; + } + throw error; +} + +export function fetchOkWithOAuth2Credentials( credentialsProvider: CredentialsProvider | undefined, input: RequestInfo, init: RequestInit, @@ -35,35 +73,22 @@ export function fetchWithOAuth2Credentials( if (credentialsProvider === undefined) { return fetchOk(input, init); } - return fetchWithCredentials( + return fetchOkWithCredentials( credentialsProvider, input, init, - (credentials, init) => { - if (!credentials.accessToken) return init; - const headers = new Headers(init.headers); - headers.set( - "Authorization", - `${credentials.tokenType} ${credentials.accessToken}`, - ); - return { ...init, headers }; - }, - (error, credentials) => { - const { status } = error; - if (status === 401) { - // 401: Authorization needed. OAuth2 token may have expired. - return "refresh"; - } - if (status === 403 && !credentials.accessToken) { - // Anonymous access denied. Request credentials. - return "refresh"; - } - if (error instanceof Error && credentials.email !== undefined) { - error.message += ` (Using credentials for ${JSON.stringify( - credentials.email, - )})`; - } - throw error; - }, + applyCredentials, + errorHandler, + ); +} + +export function fetchOkWithOAuth2CredentialsAdapter( + credentialsProvider: CredentialsProvider | undefined, +): FetchOk { + if (credentialsProvider === undefined) return fetchOk; + return fetchOkWithCredentialsAdapter( + credentialsProvider, + applyCredentials, + errorHandler, ); } diff --git a/src/credentials_provider/shared.ts b/src/credentials_provider/shared.ts index 1703a0f94f..60153312f9 100644 --- a/src/credentials_provider/shared.ts +++ b/src/credentials_provider/shared.ts @@ -19,14 +19,18 @@ */ import type { + CredentialsManager, CredentialsProvider, CredentialsWithGeneration, } from "#src/credentials_provider/index.js"; import { CREDENTIALS_PROVIDER_RPC_ID, CREDENTIALS_PROVIDER_GET_RPC_ID, + CREDENTIALS_MANAGER_RPC_ID, + CREDENTIALS_MANAGER_GET_RPC_ID, } from "#src/credentials_provider/shared_common.js"; import type { Owned } from "#src/util/disposable.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; import type { RPC, RPCPromise } from "#src/worker_rpc.js"; import { registerPromiseRPC, @@ -50,9 +54,9 @@ export class SharedCredentialsProvider get( invalidCredentials?: CredentialsWithGeneration, - abortSignal?: AbortSignal, + options?: Partial, ): Promise> { - return this.provider.get(invalidCredentials, abortSignal); + return this.provider.get(invalidCredentials, options); } } @@ -61,11 +65,53 @@ registerPromiseRPC( function ( this: RPC, x: { providerId: number; invalidCredentials: any }, - abortSignal: AbortSignal, + progressOptions, ): RPCPromise> { const obj = >this.get(x.providerId); - return obj.get(x.invalidCredentials, abortSignal).then((credentials) => ({ - value: credentials, - })); + return obj + .get(x.invalidCredentials, progressOptions) + .then((credentials) => ({ + value: credentials, + })); + }, +); + +@registerSharedObjectOwner(CREDENTIALS_MANAGER_RPC_ID) +export class SharedCredentialsManager + extends SharedObject + implements CredentialsManager +{ + constructor( + public base: CredentialsManager, + rpc: RPC, + ) { + super(); + this.initializeCounterpart(rpc); + } + + getCredentialsProvider(key: string, parameters?: any) { + return this.base.getCredentialsProvider(key, parameters); + } +} + +registerPromiseRPC( + CREDENTIALS_MANAGER_GET_RPC_ID, + async function ( + this: RPC, + x: { + managerId: number; + key: string; + parameters: any; + invalidCredentials: any; + }, + progressOptions, + ): RPCPromise> { + const manager = this.get(x.managerId) as SharedCredentialsManager; + const provider = manager.base.getCredentialsProvider(x.key, x.parameters); + const credentials = await provider.get( + x.invalidCredentials, + progressOptions, + ); + return { value: credentials }; }, ); diff --git a/src/credentials_provider/shared_common.ts b/src/credentials_provider/shared_common.ts index a9b435cc9a..ce47289b67 100644 --- a/src/credentials_provider/shared_common.ts +++ b/src/credentials_provider/shared_common.ts @@ -16,3 +16,5 @@ export const CREDENTIALS_PROVIDER_RPC_ID = "CredentialsProvider"; export const CREDENTIALS_PROVIDER_GET_RPC_ID = "CredentialsProvider.get"; +export const CREDENTIALS_MANAGER_RPC_ID = "CredentialsManager"; +export const CREDENTIALS_MANAGER_GET_RPC_ID = "CredentialsManager.get"; diff --git a/src/credentials_provider/shared_counterpart.ts b/src/credentials_provider/shared_counterpart.ts index 22436a53b2..c9ca969bca 100644 --- a/src/credentials_provider/shared_counterpart.ts +++ b/src/credentials_provider/shared_counterpart.ts @@ -20,15 +20,23 @@ */ import type { - CredentialsProvider, + CredentialsManager, CredentialsWithGeneration, MaybeOptionalCredentialsProvider, } from "#src/credentials_provider/index.js"; -import { makeCachedCredentialsGetter } from "#src/credentials_provider/index.js"; import { + CachingCredentialsManager, + makeCachedCredentialsGetter, + CredentialsProvider, +} from "#src/credentials_provider/index.js"; +import { + CREDENTIALS_MANAGER_GET_RPC_ID, + CREDENTIALS_MANAGER_RPC_ID, CREDENTIALS_PROVIDER_GET_RPC_ID, CREDENTIALS_PROVIDER_RPC_ID, } from "#src/credentials_provider/shared_common.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; +import type { RPC } from "#src/worker_rpc.js"; import { registerSharedObject, SharedObjectCounterpart, @@ -41,13 +49,13 @@ export class SharedCredentialsProviderCounterpart { get = makeCachedCredentialsGetter( ( - invalidCredentials?: CredentialsWithGeneration, - abortSignal?: AbortSignal, + invalidCredentials: CredentialsWithGeneration | undefined, + options: ProgressOptions, ): Promise> => this.rpc!.promiseInvoke( CREDENTIALS_PROVIDER_GET_RPC_ID, { providerId: this.rpcId, invalidCredentials: invalidCredentials }, - abortSignal, + { signal: options.signal, progressListener: options.progressListener }, ), ); } @@ -67,3 +75,57 @@ export function WithSharedCredentialsProviderCounterpart() { } }; } + +class ProxyCredentialsProvider< + Credentials, +> extends CredentialsProvider { + constructor( + public rpc: RPC, + public managerId: number, + public key: string, + public parameters?: any, + ) { + super(); + } + get = makeCachedCredentialsGetter( + ( + invalidCredentials: CredentialsWithGeneration | undefined, + options: ProgressOptions, + ): Promise> => + this.rpc.promiseInvoke( + CREDENTIALS_MANAGER_GET_RPC_ID, + { + managerId: this.managerId, + key: this.key, + parameters: this.parameters, + invalidCredentials: invalidCredentials, + }, + { signal: options.signal, progressListener: options.progressListener }, + ), + ); +} + +@registerSharedObject(CREDENTIALS_MANAGER_RPC_ID) +export class SharedCredentialsManagerCounterpart + extends SharedObjectCounterpart + implements CredentialsManager +{ + private impl: CachingCredentialsManager = + new CachingCredentialsManager(this.makeBaseCredentialsManager()); + + private makeBaseCredentialsManager(): CredentialsManager { + return { + getCredentialsProvider: (key: string, parameters?: any) => + new ProxyCredentialsProvider( + this.rpc!, + this.rpcId!, + key, + parameters, + ), + }; + } + + getCredentialsProvider(key: string, parameters?: any) { + return this.impl.getCredentialsProvider(key, parameters); + } +} diff --git a/src/data_management_context.ts b/src/data_management_context.ts new file mode 100644 index 0000000000..d55274c81a --- /dev/null +++ b/src/data_management_context.ts @@ -0,0 +1,79 @@ +/** + * @license + * Copyright 2016 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { FrameNumberCounter } from "#src/chunk_manager/frontend.js"; +import { + CapacitySpecification, + ChunkManager, + ChunkQueueManager, +} from "#src/chunk_manager/frontend.js"; +import { RefCounted } from "#src/util/disposable.js"; +import type { GL } from "#src/webgl/context.js"; +import { RPC } from "#src/worker_rpc.js"; + +export class DataManagementContext extends RefCounted { + worker: Worker; + chunkQueueManager: ChunkQueueManager; + chunkManager: ChunkManager; + + get rpc(): RPC { + return this.chunkQueueManager.rpc!; + } + + constructor( + public gl: GL, + public frameNumberCounter: FrameNumberCounter, + ) { + super(); + // Note: For compatibility with multiple bundlers, a browser-compatible URL + // must be used with `new URL`, which means a Node.js subpath import like + // "#src/chunk_worker.bundle.js" cannot be used. + this.worker = new Worker( + /* webpackChunkName: "neuroglancer_chunk_worker" */ + new URL("./chunk_worker.bundle.js", import.meta.url), + { type: "module" }, + ); + this.chunkQueueManager = this.registerDisposer( + new ChunkQueueManager( + new RPC(this.worker, /*waitUntilReady=*/ true), + this.gl, + this.frameNumberCounter, + { + gpuMemory: new CapacitySpecification({ + defaultItemLimit: 1e6, + defaultSizeLimit: 1e9, + }), + systemMemory: new CapacitySpecification({ + defaultItemLimit: 1e7, + defaultSizeLimit: 2e9, + }), + download: new CapacitySpecification({ + defaultItemLimit: 100, + defaultSizeLimit: Number.POSITIVE_INFINITY, + }), + compute: new CapacitySpecification({ + defaultItemLimit: 128, + defaultSizeLimit: 5e8, + }), + }, + ), + ); + this.chunkQueueManager.registerDisposer(() => this.worker.terminate()); + this.chunkManager = this.registerDisposer( + new ChunkManager(this.chunkQueueManager), + ); + } +} diff --git a/src/datasource/boss/api.ts b/src/datasource/boss/api.ts index cfed87d1fb..8ca9ff6817 100644 --- a/src/datasource/boss/api.ts +++ b/src/datasource/boss/api.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { fetchWithCredentials } from "#src/credentials_provider/http_request.js"; +import { fetchOkWithCredentials } from "#src/credentials_provider/http_request.js"; import type { CredentialsProvider } from "#src/credentials_provider/index.js"; import { fetchOk } from "#src/util/http_request.js"; @@ -41,7 +41,7 @@ export async function fetchWithBossCredentials( // has been cancelled throw error; } - return fetchWithCredentials( + return fetchOkWithCredentials( credentialsProvider, input, init, diff --git a/src/datasource/boss/backend.ts b/src/datasource/boss/backend.ts index 3b16ca3159..fba370af85 100644 --- a/src/datasource/boss/backend.ts +++ b/src/datasource/boss/backend.ts @@ -67,7 +67,7 @@ export class BossVolumeChunkSource extends BossSource( ) { chunkDecoder = chunkDecoders.get(this.parameters.encoding)!; - async download(chunk: VolumeChunk, abortSignal: AbortSignal) { + async download(chunk: VolumeChunk, signal: AbortSignal) { const { parameters } = this; let url = `${parameters.baseUrl}/latest/cutout/${parameters.collection}/${parameters.experiment}/${parameters.channel}/${parameters.resolution}`; { @@ -88,11 +88,11 @@ export class BossVolumeChunkSource extends BossSource( this.credentialsProvider, url, { - signal: abortSignal, + signal: signal, headers: { Accept: acceptHeaders.get(parameters.encoding)! }, }, ); - await this.chunkDecoder(chunk, abortSignal, await response.arrayBuffer()); + await this.chunkDecoder(chunk, signal, await response.arrayBuffer()); } } @@ -119,23 +119,23 @@ export class BossMeshSource extends BossSource( MeshSource, MeshSourceParameters, ) { - download(chunk: ManifestChunk, abortSignal: AbortSignal) { + download(chunk: ManifestChunk, signal: AbortSignal) { const { parameters } = this; return fetchWithBossCredentials( this.credentialsProvider, `${parameters.baseUrl}${chunk.objectId}`, - { signal: abortSignal }, + { signal: signal }, ) .then((response) => response.arrayBuffer()) .then((response) => decodeManifestChunk(chunk, response)); } - downloadFragment(chunk: FragmentChunk, abortSignal: AbortSignal) { + downloadFragment(chunk: FragmentChunk, signal: AbortSignal) { const { parameters } = this; return fetchWithBossCredentials( this.credentialsProvider, `${parameters.baseUrl}${chunk.fragmentId}`, - { signal: abortSignal }, + { signal: signal }, ) .then((response) => response.arrayBuffer()) .then((response) => decodeFragmentChunk(chunk, response)); diff --git a/src/datasource/boss/credentials_provider.ts b/src/datasource/boss/credentials_provider.ts index 939a44e400..0a05e79fcc 100644 --- a/src/datasource/boss/credentials_provider.ts +++ b/src/datasource/boss/credentials_provider.ts @@ -33,6 +33,7 @@ import { verifyObjectProperty, verifyString, } from "#src/util/json.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; import { getRandomHexString } from "#src/util/random.js"; export type BossToken = string; @@ -61,7 +62,7 @@ function makeAuthRequestUrl(options: { function waitForAuthResponseMessage( source: Window, state: string, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise { return new Promise((resolve, reject) => { window.addEventListener( @@ -104,7 +105,7 @@ function waitForAuthResponseMessage( console.error("Response received: ", event.data); } }, - { signal: abortSignal }, + { signal: signal }, ); }); } @@ -115,7 +116,7 @@ function waitForAuthResponseMessage( */ export async function authenticateKeycloakOIDC( options: { realm: string; clientId: string; authServer: string }, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise { const state = getRandomHexString(); const nonce = getRandomHexString(); @@ -127,7 +128,7 @@ export async function authenticateKeycloakOIDC( nonce: nonce, }); const abortController = new AbortController(); - abortSignal = AbortSignal.any([abortController.signal, abortSignal]); + signal = AbortSignal.any([abortController.signal, signal]); try { const newWindow = open(url); if (newWindow === null) { @@ -136,7 +137,7 @@ export async function authenticateKeycloakOIDC( monitorAuthPopupWindow(newWindow, abortController); return await raceWithAbort( waitForAuthResponseMessage(newWindow, state, abortController.signal), - abortSignal, + signal, ); } finally { abortController.abort(); @@ -148,8 +149,11 @@ export class BossCredentialsProvider extends CredentialsProvider { super(); } - get = makeCredentialsGetter((abortSignal) => - getCredentialsWithStatus( + get = makeCredentialsGetter(async (options) => { + using _span = new ProgressSpan(options.progressListener, { + message: `Requesting Boss access token from ${this.authServer}`, + }); + return await getCredentialsWithStatus( { description: "Boss", get: (signal) => @@ -162,7 +166,7 @@ export class BossCredentialsProvider extends CredentialsProvider { signal, ), }, - abortSignal, - ), - ); + options.signal, + ); + }); } diff --git a/src/datasource/boss/frontend.ts b/src/datasource/boss/frontend.ts index 69d5302529..ae3cfee2da 100644 --- a/src/datasource/boss/frontend.ts +++ b/src/datasource/boss/frontend.ts @@ -46,8 +46,8 @@ import type { CompletionResult, DataSource, GetDataSourceOptions, + DataSourceProvider, } from "#src/datasource/index.js"; -import { DataSourceProvider } from "#src/datasource/index.js"; import { MeshSource } from "#src/mesh/frontend.js"; import type { SliceViewSingleResolutionSource } from "#src/sliceview/frontend.js"; import type { VolumeSourceOptions } from "#src/sliceview/volume/base.js"; @@ -81,6 +81,7 @@ import { verifyOptionalString, verifyString, } from "#src/util/json.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; class BossVolumeChunkSource extends WithParameters( WithCredentialsProvider()(VolumeChunkSource), @@ -249,6 +250,7 @@ function parseExperimentInfo( credentialsProvider: CredentialsProvider, collection: string, experiment: string, + options: Partial, ): Promise { verifyObject(obj); @@ -261,6 +263,7 @@ function parseExperimentInfo( experiment, collection, ch, + options, ), ), ); @@ -290,6 +293,7 @@ function parseExperimentInfo( hostname, credentialsProvider, experimentInfo, + options, ); }); } @@ -500,19 +504,21 @@ export function getExperimentInfo( credentialsProvider: CredentialsProvider, experiment: string, collection: string, + options: Partial, ): Promise { - return chunkManager.memoize.getUncounted( + return chunkManager.memoize.getAsync( { hostname: hostname, collection: collection, experiment: experiment, type: "boss:getExperimentInfo", }, - () => + options, + (progressOptions) => fetchWithBossCredentials( credentialsProvider, `${hostname}/latest/collection/${collection}/experiment/${experiment}/`, - {}, + progressOptions, ) .then((response) => response.json()) .then((value) => @@ -523,6 +529,7 @@ export function getExperimentInfo( credentialsProvider, collection, experiment, + progressOptions, ), ), ); @@ -535,8 +542,9 @@ export function getChannelInfo( experiment: string, collection: string, channel: string, + options: Partial, ): Promise { - return chunkManager.memoize.getUncounted( + return chunkManager.memoize.getAsync( { hostname: hostname, collection: collection, @@ -544,11 +552,12 @@ export function getChannelInfo( channel: channel, type: "boss:getChannelInfo", }, - () => + options, + (progressOptions) => fetchWithBossCredentials( credentialsProvider, `${hostname}/latest/collection/${collection}/experiment/${experiment}/channel/${channel}/`, - {}, + progressOptions, ) .then((response) => response.json()) .then(parseChannelInfo), @@ -562,9 +571,10 @@ export function getDownsampleInfoForChannel( collection: string, experimentInfo: ExperimentInfo, channel: string, + options: Partial, ): Promise { return chunkManager.memoize - .getUncounted( + .getAsync( { hostname: hostname, collection: collection, @@ -573,11 +583,12 @@ export function getDownsampleInfoForChannel( downsample: true, type: "boss:getDownsampleInfoForChannel", }, - () => + options, + (progressOptions) => fetchWithBossCredentials( credentialsProvider, `${hostname}/latest/downsample/${collection}/${experimentInfo.key}/${channel}`, - {}, + progressOptions, ).then((response) => response.json()), ) .then((downsampleObj) => { @@ -660,6 +671,7 @@ export function getDataSource( hostname: string, credentialsProvider: CredentialsProvider, path: string, + options: Partial, ) { const match = path.match(pathPattern); if (match === null) { @@ -670,15 +682,17 @@ export function getDataSource( const channel = match[3]; const parameters = parseQueryStringParameters(match[4] || ""); // Warning: If additional arguments are added, the cache key should be updated as well. - return chunkManager.memoize.getUncounted( + return chunkManager.memoize.getAsync( { hostname: hostname, path: path, type: "boss:getVolume" }, - async () => { + options, + async (progressOptions) => { const experimentInfo = await getExperimentInfo( chunkManager, hostname, credentialsProvider, experiment, collection, + progressOptions, ); const experimentInfoWithDownsample = await getDownsampleInfoForChannel( chunkManager, @@ -687,6 +701,7 @@ export function getDataSource( collection, experimentInfo, channel, + progressOptions, ); const volume = new BossMultiscaleVolumeChunkSource( chunkManager, @@ -739,14 +754,16 @@ export function getCollections( chunkManager: ChunkManager, hostname: string, credentialsProvider: CredentialsProvider, + options: Partial, ) { - return chunkManager.memoize.getUncounted( + return chunkManager.memoize.getAsync( { hostname: hostname, type: "boss:getCollections" }, - () => + options, + (progressOptions) => fetchWithBossCredentials( credentialsProvider, `${hostname}/latest/collection/`, - {}, + progressOptions, ) .then((response) => response.json()) .then((value) => @@ -762,14 +779,16 @@ export function getExperiments( hostname: string, credentialsProvider: CredentialsProvider, collection: string, + options: Partial, ) { - return chunkManager.memoize.getUncounted( + return chunkManager.memoize.getAsync( { hostname: hostname, collection: collection, type: "boss:getExperiments" }, - () => + options, + (progressOptions) => fetchWithBossCredentials( credentialsProvider, `${hostname}/latest/collection/${collection}/experiment/`, - {}, + progressOptions, ) .then((response) => response.json()) .then((value) => @@ -785,20 +804,22 @@ export function getCoordinateFrame( hostname: string, credentialsProvider: CredentialsProvider, experimentInfo: ExperimentInfo, + options: Partial, ): Promise { const key = experimentInfo.coordFrameKey; - return chunkManager.memoize.getUncounted( + return chunkManager.memoize.getAsync( { hostname: hostname, coordinateframe: key, experimentInfo: experimentInfo, type: "boss:getCoordinateFrame", }, - () => + options, + (progressOptions) => fetchWithBossCredentials( credentialsProvider, `${hostname}/latest/coord/${key}/`, - {}, + progressOptions, ) .then((response) => response.json()) .then((coordinateFrameObj) => @@ -812,6 +833,7 @@ export function collectionExperimentChannelCompleter( hostname: string, credentialsProvider: CredentialsProvider, path: string, + options: Partial, ): Promise { const channelMatch = path.match( /^(?:([^/]+)(?:\/?([^/]*)(?:\/?([^/]*)(?:\/?([^/]*)?))?)?)?$/, @@ -827,19 +849,22 @@ export function collectionExperimentChannelCompleter( if (channelMatch[2] === undefined) { const collectionPrefix = channelMatch[1] || ""; // Try to complete the collection. - return getCollections(chunkManager, hostname, credentialsProvider).then( - (collections) => { - return { - offset: 0, - completions: getPrefixMatchesWithDescriptions( - collectionPrefix, - collections, - (x) => x + "/", - () => undefined, - ), - }; - }, - ); + return getCollections( + chunkManager, + hostname, + credentialsProvider, + options, + ).then((collections) => { + return { + offset: 0, + completions: getPrefixMatchesWithDescriptions( + collectionPrefix, + collections, + (x) => x + "/", + () => undefined, + ), + }; + }); } if (channelMatch[3] === undefined) { const experimentPrefix = channelMatch[2] || ""; @@ -848,6 +873,7 @@ export function collectionExperimentChannelCompleter( hostname, credentialsProvider, channelMatch[1], + options, ).then((experiments) => { return { offset: channelMatch![1].length + 1, @@ -866,6 +892,7 @@ export function collectionExperimentChannelCompleter( credentialsProvider, channelMatch[2], channelMatch[1], + options, ).then((experimentInfo) => { const completions = getPrefixMatchesWithDescriptions( channelMatch![3], @@ -893,18 +920,17 @@ function getAuthServer(endpoint: string): string { return authServer; } -export class BossDataSource extends DataSourceProvider { - constructor(public credentialsManager: CredentialsManager) { - super(); +export class BossDataSource implements DataSourceProvider { + get scheme() { + return "boss"; } - get description() { return "bossDB: Block & Object Storage System"; } - getCredentialsProvider(path: string) { + getCredentialsProvider(credentialsManager: CredentialsManager, path: string) { const authServer = getAuthServer(path); - return this.credentialsManager.getCredentialsProvider( + return credentialsManager.getCredentialsProvider( credentialsKey, authServer, ); @@ -918,13 +944,15 @@ export class BossDataSource extends DataSourceProvider { ); } const credentialsProvider = this.getCredentialsProvider( + options.registry.credentialsManager, options.providerUrl, ); return getDataSource( - options.chunkManager, + options.registry.chunkManager, match[1], credentialsProvider, match[2], + options, ); } @@ -935,13 +963,17 @@ export class BossDataSource extends DataSourceProvider { throw null; } const hostname = match[1]; - const credentialsProvider = this.getCredentialsProvider(match[1]); + const credentialsProvider = this.getCredentialsProvider( + options.registry.credentialsManager, + match[1], + ); const path = match[2]; const completions = await collectionExperimentChannelCompleter( - options.chunkManager, + options.registry.chunkManager, hostname, credentialsProvider, path, + options, ); return applyCompletionOffset(match![1].length + 1, completions); } diff --git a/src/datasource/boss/register_credentials_provider.ts b/src/datasource/boss/register_credentials_provider.ts index fcdbe1eeac..47ec688d32 100644 --- a/src/datasource/boss/register_credentials_provider.ts +++ b/src/datasource/boss/register_credentials_provider.ts @@ -14,11 +14,11 @@ * limitations under the License. */ -import { defaultCredentialsManager } from "#src/credentials_provider/default_manager.js"; +import { registerDefaultCredentialsProvider } from "#src/credentials_provider/default_manager.js"; import { credentialsKey } from "#src/datasource/boss/api.js"; import { BossCredentialsProvider } from "#src/datasource/boss/credentials_provider.js"; -defaultCredentialsManager.register( +registerDefaultCredentialsProvider( credentialsKey, (authServer) => new BossCredentialsProvider(authServer), ); diff --git a/src/datasource/boss/register_default.ts b/src/datasource/boss/register_default.ts index 399f13a1a5..acdf432402 100644 --- a/src/datasource/boss/register_default.ts +++ b/src/datasource/boss/register_default.ts @@ -17,7 +17,4 @@ import { BossDataSource } from "#src/datasource/boss/frontend.js"; import { registerProvider } from "#src/datasource/default_provider.js"; -registerProvider( - "boss", - (options) => new BossDataSource(options.credentialsManager), -); +registerProvider(new BossDataSource()); diff --git a/src/datasource/brainmaps/api.ts b/src/datasource/brainmaps/api.ts index e2cfd481d5..941e7ccffd 100644 --- a/src/datasource/brainmaps/api.ts +++ b/src/datasource/brainmaps/api.ts @@ -16,7 +16,8 @@ import type { CredentialsProvider } from "#src/credentials_provider/index.js"; import type { OAuth2Credentials } from "#src/credentials_provider/oauth2.js"; -import { fetchWithOAuth2Credentials } from "#src/credentials_provider/oauth2.js"; +import { fetchOkWithOAuth2Credentials } from "#src/credentials_provider/oauth2.js"; +import type { RequestInitWithProgress } from "#src/util/http_request.js"; export type { OAuth2Credentials }; @@ -93,25 +94,15 @@ export interface BatchMeshFragmentPayload { batches: BatchMeshFragment[]; } -export interface HttpCall { - method: "GET" | "POST"; - path: string; - payload?: string; - signal?: AbortSignal; -} - export function makeRequest( instance: BrainmapsInstance, credentialsProvider: BrainmapsCredentialsProvider, - httpCall: HttpCall, + path: string, + init: RequestInitWithProgress = {}, ): Promise { - return fetchWithOAuth2Credentials( + return fetchOkWithOAuth2Credentials( credentialsProvider, - `${instance.serverUrl}${httpCall.path}`, - { - signal: httpCall.signal, - method: httpCall.method, - body: httpCall.payload, - }, + `${instance.serverUrl}${path}`, + init, ); } diff --git a/src/datasource/brainmaps/backend.ts b/src/datasource/brainmaps/backend.ts index af3c722b81..51e2de5061 100644 --- a/src/datasource/brainmaps/backend.ts +++ b/src/datasource/brainmaps/backend.ts @@ -142,8 +142,6 @@ function BrainmapsSource< ); } -const tempUint64 = new Uint64(); - @registerSharedObject() export class BrainmapsVolumeChunkSource extends BrainmapsSource( VolumeChunkSource, @@ -177,7 +175,7 @@ export class BrainmapsVolumeChunkSource extends BrainmapsSource( } } - async download(chunk: VolumeChunk, abortSignal: AbortSignal) { + async download(chunk: VolumeChunk, signal: AbortSignal) { const { parameters } = this; // chunkPosition must not be captured, since it will be invalidated by the next call to @@ -200,14 +198,14 @@ export class BrainmapsVolumeChunkSource extends BrainmapsSource( const response = await makeRequest( parameters.instance, this.credentialsProvider, + path, { method: "POST", - payload: JSON.stringify(payload), - path, - signal: abortSignal, + body: JSON.stringify(payload), + signal: signal, }, ); - await this.chunkDecoder(chunk, abortSignal, await response.arrayBuffer()); + await this.chunkDecoder(chunk, signal, await response.arrayBuffer()); } } @@ -223,7 +221,7 @@ function getFragmentCorner( `Couldn't parse fragmentId ${fragmentId} as hex-encoded Uint64`, ); } - return decodeZIndexCompressed(id, xBits, yBits, zBits); + return decodeZIndexCompressed(id.toBigInt(), xBits, yBits, zBits); } interface BrainmapsMultiscaleManifestChunk extends MultiscaleManifestChunk { @@ -485,7 +483,7 @@ async function makeBatchMeshRequest( meshName: string; }, ids: Map, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise { const path = "/v1/objects/meshes:batch"; const batches: BatchMeshFragment[] = []; @@ -511,11 +509,10 @@ async function makeBatchMeshRequest( }; try { return await ( - await makeRequest(parameters.instance, credentialsProvider, { + await makeRequest(parameters.instance, credentialsProvider, path, { method: "POST", - path, - payload: JSON.stringify(payload), - signal: abortSignal, + body: JSON.stringify(payload), + signal: signal, }) ).arrayBuffer(); } finally { @@ -539,26 +536,21 @@ export class BrainmapsMultiscaleMeshSource extends BrainmapsSource( return ""; })(); - download(chunk: BrainmapsMultiscaleManifestChunk, abortSignal: AbortSignal) { + download(chunk: BrainmapsMultiscaleManifestChunk, signal: AbortSignal) { const { parameters } = this; const path = `/v1/objects/${parameters.volumeId}/meshes/` + `${parameters.info.lods[0].info.name}:listfragments?` + `object_id=${chunk.objectId}&return_supervoxel_ids=true` + this.listFragmentsParams; - return makeRequest(parameters.instance, this.credentialsProvider, { - method: "GET", - path, - signal: abortSignal, + return makeRequest(parameters.instance, this.credentialsProvider, path, { + signal: signal, }) .then((response) => response.json()) .then((response) => decodeMultiscaleManifestChunk(chunk, response)); } - async downloadFragment( - chunk: MultiscaleFragmentChunk, - abortSignal: AbortSignal, - ) { + async downloadFragment(chunk: MultiscaleFragmentChunk, signal: AbortSignal) { const { parameters } = this; const manifestChunk = @@ -597,7 +589,6 @@ export class BrainmapsMultiscaleMeshSource extends BrainmapsSource( octree[chunkIndex * 5 + 2] / relativeBlockShape[2], ); const fragmentKey = encodeZIndexCompressed3d( - tempUint64, xBits, yBits, zBits, @@ -641,7 +632,7 @@ export class BrainmapsMultiscaleMeshSource extends BrainmapsSource( meshName, }, ids, - abortSignal, + signal, ) .then((response) => { --requestsInProgress; @@ -772,17 +763,15 @@ export class BrainmapsMeshSource extends BrainmapsSource( return ""; })(); - download(chunk: ManifestChunk, abortSignal: AbortSignal) { + download(chunk: ManifestChunk, signal: AbortSignal) { const { parameters } = this; const path = `/v1/objects/${parameters.volumeId}/meshes/` + `${parameters.meshName}:listfragments?` + `object_id=${chunk.objectId}&return_supervoxel_ids=true` + this.listFragmentsParams; - return makeRequest(parameters.instance, this.credentialsProvider, { - signal: abortSignal, - method: "GET", - path, + return makeRequest(parameters.instance, this.credentialsProvider, path, { + signal, }) .then((response) => response.json()) .then((response) => @@ -790,7 +779,7 @@ export class BrainmapsMeshSource extends BrainmapsSource( ); } - async downloadFragment(chunk: FragmentChunk, abortSignal: AbortSignal) { + async downloadFragment(chunk: FragmentChunk, signal: AbortSignal) { const { parameters } = this; const ids = new Map(); @@ -807,7 +796,7 @@ export class BrainmapsMeshSource extends BrainmapsSource( credentialsProvider, parameters, ids, - abortSignal, + signal, ); decodeBatchMeshResponse(response, (fragment) => { if (!ids.delete(fragment.fullKey)) { @@ -852,7 +841,7 @@ export class BrainmapsSkeletonSource extends BrainmapsSource( SkeletonSource, SkeletonSourceParameters, ) { - download(chunk: SkeletonChunk, abortSignal: AbortSignal) { + download(chunk: SkeletonChunk, signal: AbortSignal) { const { parameters } = this; const payload: SkeletonPayload = { object_id: `${chunk.objectId}`, @@ -862,11 +851,10 @@ export class BrainmapsSkeletonSource extends BrainmapsSource( `/meshes/${parameters.meshName}` + "/skeleton:binary"; applyChangeStack(parameters.changeSpec, payload); - return makeRequest(parameters.instance, this.credentialsProvider, { + return makeRequest(parameters.instance, this.credentialsProvider, path, { method: "POST", - path, - payload: JSON.stringify(payload), - signal: abortSignal, + body: JSON.stringify(payload), + signal, }) .then((response) => response.arrayBuffer()) .then((response) => decodeSkeletonChunk(chunk, response)); @@ -1134,19 +1122,23 @@ export class BrainmapsAnnotationGeometryChunkSource extends BrainmapsSource( AnnotationGeometryChunkSourceBackend, AnnotationSpatialIndexSourceParameters, ) { - async download(chunk: AnnotationGeometryChunk, abortSignal: AbortSignal) { + async download(chunk: AnnotationGeometryChunk, signal: AbortSignal) { const { parameters } = this; return Promise.all( spatialAnnotationTypes.map((spatialAnnotationType) => - makeRequest(parameters.instance, this.credentialsProvider, { - signal: abortSignal, - method: "POST", - path: `/v1/changes/${parameters.volumeId}/${parameters.changestack}/spatials:get`, - payload: JSON.stringify({ - type: spatialAnnotationType, - ignore_payload: true, - }), - }).then((response) => response.json()), + makeRequest( + parameters.instance, + this.credentialsProvider, + `/v1/changes/${parameters.volumeId}/${parameters.changestack}/spatials:get`, + { + signal, + method: "POST", + body: JSON.stringify({ + type: spatialAnnotationType, + ignore_payload: true, + }), + }, + ).then((response) => response.json()), ), ).then((values) => { parseAnnotations(chunk, values); @@ -1162,39 +1154,47 @@ export class BrainmapsAnnotationSource extends BrainmapsSource( downloadSegmentFilteredGeometry( chunk: AnnotationSubsetGeometryChunk, _relationshipIndex: number, - abortSignal: AbortSignal, + signal: AbortSignal, ) { const { parameters } = this; return Promise.all( spatialAnnotationTypes.map((spatialAnnotationType) => - makeRequest(parameters.instance, this.credentialsProvider, { - signal: abortSignal, - method: "POST", - path: `/v1/changes/${parameters.volumeId}/${parameters.changestack}/spatials:get`, - payload: JSON.stringify({ - type: spatialAnnotationType, - object_labels: [chunk.objectId.toString()], - ignore_payload: true, - }), - }).then((response) => response.json()), + makeRequest( + parameters.instance, + this.credentialsProvider, + `/v1/changes/${parameters.volumeId}/${parameters.changestack}/spatials:get`, + { + signal, + method: "POST", + body: JSON.stringify({ + type: spatialAnnotationType, + object_labels: [chunk.objectId.toString()], + ignore_payload: true, + }), + }, + ).then((response) => response.json()), ), ).then((values) => { parseAnnotations(chunk, values); }); } - downloadMetadata(chunk: AnnotationMetadataChunk, abortSignal: AbortSignal) { + downloadMetadata(chunk: AnnotationMetadataChunk, signal: AbortSignal) { const { parameters } = this; const id = chunk.key!; - return makeRequest(parameters.instance, this.credentialsProvider, { - signal: abortSignal, - method: "POST", - path: `/v1/changes/${parameters.volumeId}/${parameters.changestack}/spatials:get`, - payload: JSON.stringify({ - type: getSpatialAnnotationTypeFromId(id), - id: getFullSpatialAnnotationId(parameters, id), - }), - }) + return makeRequest( + parameters.instance, + this.credentialsProvider, + `/v1/changes/${parameters.volumeId}/${parameters.changestack}/spatials:get`, + { + signal, + method: "POST", + body: JSON.stringify({ + type: getSpatialAnnotationTypeFromId(id), + id: getFullSpatialAnnotationId(parameters, id), + }), + }, + ) .then((response) => response.json()) .then( (response) => { @@ -1213,11 +1213,15 @@ export class BrainmapsAnnotationSource extends BrainmapsSource( add(annotation: Annotation) { const { parameters } = this; const brainmapsAnnotation = annotationToBrainmaps(annotation); - return makeRequest(parameters.instance, this.credentialsProvider, { - method: "POST", - path: `/v1/changes/${parameters.volumeId}/${parameters.changestack}/spatials:push`, - payload: JSON.stringify({ annotations: [brainmapsAnnotation] }), - }) + return makeRequest( + parameters.instance, + this.credentialsProvider, + `/v1/changes/${parameters.volumeId}/${parameters.changestack}/spatials:push`, + { + method: "POST", + body: JSON.stringify({ annotations: [brainmapsAnnotation] }), + }, + ) .then((response) => response.json()) .then((response) => { verifyObject(response); @@ -1236,22 +1240,30 @@ export class BrainmapsAnnotationSource extends BrainmapsSource( const { parameters } = this; const brainmapsAnnotation = annotationToBrainmaps(annotation); brainmapsAnnotation.id = getFullSpatialAnnotationId(parameters, id); - return makeRequest(parameters.instance, this.credentialsProvider, { - method: "POST", - path: `/v1/changes/${parameters.volumeId}/${parameters.changestack}/spatials:push`, - payload: JSON.stringify({ annotations: [brainmapsAnnotation] }), - }).then((response) => response.json()); + return makeRequest( + parameters.instance, + this.credentialsProvider, + `/v1/changes/${parameters.volumeId}/${parameters.changestack}/spatials:push`, + { + method: "POST", + body: JSON.stringify({ annotations: [brainmapsAnnotation] }), + }, + ).then((response) => response.json()); } delete(id: AnnotationId) { const { parameters } = this; - return makeRequest(parameters.instance, this.credentialsProvider, { - method: "POST", - path: `/v1/changes/${parameters.volumeId}/${parameters.changestack}/spatials:delete`, - payload: JSON.stringify({ - type: getSpatialAnnotationTypeFromId(id), - ids: [getFullSpatialAnnotationId(parameters, id)], - }), - }).then((response) => response.json()); + return makeRequest( + parameters.instance, + this.credentialsProvider, + `/v1/changes/${parameters.volumeId}/${parameters.changestack}/spatials:delete`, + { + method: "POST", + body: JSON.stringify({ + type: getSpatialAnnotationTypeFromId(id), + ids: [getFullSpatialAnnotationId(parameters, id)], + }), + }, + ).then((response) => response.json()); } } diff --git a/src/datasource/brainmaps/frontend.ts b/src/datasource/brainmaps/frontend.ts index 6bd0cfc1ba..aa5a16ba14 100644 --- a/src/datasource/brainmaps/frontend.ts +++ b/src/datasource/brainmaps/frontend.ts @@ -34,13 +34,16 @@ import { makeIdentityTransformedBoundingBox, } from "#src/coordinate_transform.js"; import { WithCredentialsProvider } from "#src/credentials_provider/chunk_source_frontend.js"; -import type { CredentialsProvider } from "#src/credentials_provider/index.js"; +import type { + CredentialsManager, + CredentialsProvider, +} from "#src/credentials_provider/index.js"; import type { BrainmapsCredentialsProvider, BrainmapsInstance, OAuth2Credentials, } from "#src/datasource/brainmaps/api.js"; -import { makeRequest } from "#src/datasource/brainmaps/api.js"; +import { credentialsKey, makeRequest } from "#src/datasource/brainmaps/api.js"; import type { ChangeSpec, MultiscaleMeshInfo, @@ -58,9 +61,10 @@ import { import type { CompleteUrlOptions, DataSource, + DataSourceRegistry, GetDataSourceOptions, + DataSourceProvider, } from "#src/datasource/index.js"; -import { DataSourceProvider } from "#src/datasource/index.js"; import { VertexPositionFormat } from "#src/mesh/base.js"; import { MeshSource, MultiscaleMeshSource } from "#src/mesh/frontend.js"; import { SkeletonSource } from "#src/skeleton/frontend.js"; @@ -79,7 +83,6 @@ import { MultiscaleVolumeChunkSource as GenericMultiscaleVolumeChunkSource, VolumeChunkSource, } from "#src/sliceview/volume/frontend.js"; -import { StatusMessage } from "#src/status.js"; import { transposeNestedArrays } from "#src/util/array.js"; import type { CompletionWithDescription } from "#src/util/completion.js"; import { @@ -106,7 +109,8 @@ import { verifyPositiveInt, verifyString, } from "#src/util/json.js"; -import { getObjectId } from "#src/util/object_id.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; import { defaultStringCompare } from "#src/util/string.js"; class BrainmapsVolumeChunkSource extends WithParameters( @@ -769,49 +773,65 @@ const supportedQueryParameters = [ }, ]; -export class BrainmapsDataSource extends DataSourceProvider { +function getCredentialsProvider(credentialsManager: CredentialsManager) { + return credentialsManager.getCredentialsProvider( + credentialsKey, + ); +} + +export class BrainmapsDataSource implements DataSourceProvider { constructor( public instance: BrainmapsInstance, - public credentialsProvider: Owned, - ) { - super(); - } + public scheme: string, + ) {} get description() { return this.instance.description; } - private getMultiscaleInfo(chunkManager: ChunkManager, volumeId: string) { - return chunkManager.memoize.getUncounted( + private getMultiscaleInfo( + registry: DataSourceRegistry, + volumeId: string, + options: Partial, + ) { + return registry.chunkManager.memoize.getAsync( { type: "brainmaps:getMultiscaleInfo", volumeId, instance: this.instance, - credentialsProvider: getObjectId(this.credentialsProvider), }, - () => - makeRequest(this.instance, this.credentialsProvider, { - method: "GET", - path: `/v1beta2/volumes/${volumeId}`, - }) - .then((response) => response.json()) - .then((response) => new MultiscaleVolumeInfo(response)), + options, + async (progressOptions) => { + const response = await makeRequest( + this.instance, + getCredentialsProvider(registry.credentialsManager), + `/v1beta2/volumes/${volumeId}`, + progressOptions, + ); + return new MultiscaleVolumeInfo(await response.json()); + }, ); } - private getMeshesInfo(chunkManager: ChunkManager, volumeId: string) { - return chunkManager.memoize.getUncounted( + private getMeshesInfo( + registry: DataSourceRegistry, + volumeId: string, + options: Partial, + ) { + return registry.chunkManager.memoize.getAsync( { type: "brainmaps:getMeshesInfo", volumeId, instance: this.instance, - credentialsProvider: getObjectId(this.credentialsProvider), }, - () => - makeRequest(this.instance, this.credentialsProvider, { - method: "GET", - path: `/v1beta2/objects/${volumeId}/meshes`, - }) + options, + (progressOptions) => + makeRequest( + this.instance, + getCredentialsProvider(registry.credentialsManager), + `/v1beta2/objects/${volumeId}/meshes`, + progressOptions, + ) .then((response) => response.json()) .then((response) => parseMeshesResponse(response)), ); @@ -848,7 +868,7 @@ export class BrainmapsDataSource extends DataSourceProvider { chunkLayoutPreference, jpegQuality, }; - return options.chunkManager.memoize.getUncounted( + return options.registry.chunkManager.memoize.getAsync( { type: "brainmaps:get", instance: this.instance, @@ -856,15 +876,19 @@ export class BrainmapsDataSource extends DataSourceProvider { changeSpec, brainmapsOptions, }, - async () => { + options, + async (progressOptions) => { + const credentialsProvider = getCredentialsProvider( + options.registry.credentialsManager, + ); const [multiscaleVolumeInfo, meshesInfo] = await Promise.all([ - this.getMultiscaleInfo(options.chunkManager, volumeId), - this.getMeshesInfo(options.chunkManager, volumeId), + this.getMultiscaleInfo(options.registry, volumeId, progressOptions), + this.getMeshesInfo(options.registry, volumeId, progressOptions), ]); const volume = new MultiscaleVolumeChunkSource( - options.chunkManager, + options.registry.chunkManager, this.instance, - this.credentialsProvider, + credentialsProvider, volumeId, changeSpec, multiscaleVolumeInfo, @@ -918,10 +942,10 @@ export class BrainmapsDataSource extends DataSourceProvider { const { single } = mesh; if (single !== undefined) { if (single.type === "TRIANGLES") { - meshSource = options.chunkManager.getChunkSource( + meshSource = options.registry.chunkManager.getChunkSource( BrainmapsMeshSource, { - credentialsProvider: this.credentialsProvider, + credentialsProvider, parameters: { instance: this.instance, volumeId: volumeId, @@ -931,10 +955,10 @@ export class BrainmapsDataSource extends DataSourceProvider { }, ); } else { - meshSource = options.chunkManager.getChunkSource( + meshSource = options.registry.chunkManager.getChunkSource( BrainmapsSkeletonSource, { - credentialsProvider: this.credentialsProvider, + credentialsProvider, parameters: { instance: this.instance, volumeId: volumeId, @@ -946,10 +970,10 @@ export class BrainmapsDataSource extends DataSourceProvider { } } else { const multi = mesh.multi!; - meshSource = options.chunkManager.getChunkSource( + meshSource = options.registry.chunkManager.getChunkSource( BrainmapsMultiscaleMeshSource, { - credentialsProvider: this.credentialsProvider, + credentialsProvider, format: { fragmentRelativeVertices: false, vertexPositionFormat: VertexPositionFormat.float32, @@ -994,7 +1018,7 @@ export class BrainmapsDataSource extends DataSourceProvider { default: true, modelSubspaceDimensionIndices: [0, 1, 2], subsource: { - annotation: options.chunkManager.getChunkSource( + annotation: options.registry.chunkManager.getChunkSource( BrainmapsAnnotationSource, { parameters: { @@ -1004,7 +1028,7 @@ export class BrainmapsDataSource extends DataSourceProvider { upperVoxelBound: multiscaleVolumeInfo.scales[0].upperVoxelBound, }, - credentialsProvider: this.credentialsProvider, + credentialsProvider, }, ), }, @@ -1015,112 +1039,110 @@ export class BrainmapsDataSource extends DataSourceProvider { ); } - getProjectList(chunkManager: ChunkManager) { - return chunkManager.memoize.getUncounted( + getProjectList( + registry: DataSourceRegistry, + options: Partial, + ) { + return registry.chunkManager.memoize.getAsync( { instance: this.instance, type: "brainmaps:getProjectList" }, - () => { - const promise = makeRequest(this.instance, this.credentialsProvider, { - method: "GET", - path: "/v1beta2/projects", - }) - .then((response) => response.json()) - .then((projectsResponse) => { - return parseProjectList(projectsResponse); - }); - const description = `${this.instance.description} project list`; - StatusMessage.forPromise(promise, { - delay: true, - initialMessage: `Retrieving ${description}.`, - errorPrefix: `Error retrieving ${description}: `, + options, + async (progressOptions) => { + using _span = new ProgressSpan(progressOptions.progressListener, { + message: `Retrieving ${this.instance.description} project list`, }); - return promise; + const response = await makeRequest( + this.instance, + getCredentialsProvider(registry.credentialsManager), + "/v1beta2/projects", + progressOptions, + ); + return parseProjectList(await response.json()); }, ); } - getDatasetList(chunkManager: ChunkManager, project: string) { - return chunkManager.memoize.getUncounted( + getDatasetList( + registry: DataSourceRegistry, + project: string, + options: Partial, + ) { + return registry.chunkManager.memoize.getAsync( { instance: this.instance, type: `brainmaps:${project}:getDatasetList` }, - () => { - const promise = makeRequest(this.instance, this.credentialsProvider, { - method: "GET", - path: `/v1beta2/datasets?project_id=${project}`, - }) - .then((response) => response.json()) - .then((datasetsResponse) => { - return parseAPIResponseList(datasetsResponse, "datasetIds"); - }); - const description = `${this.instance.description} dataset list`; - StatusMessage.forPromise(promise, { - delay: true, - initialMessage: `Retrieving ${description}`, - errorPrefix: `Error retrieving ${description}`, + options, + async (progressOptions) => { + using _span = new ProgressSpan(progressOptions.progressListener, { + message: `Retrieving ${this.instance.description} dataset list for ${project}`, }); - return promise; + const response = await makeRequest( + this.instance, + getCredentialsProvider(registry.credentialsManager), + `/v1beta2/datasets?project_id=${project}`, + ); + return parseAPIResponseList(await response.json(), "datasetIds"); }, ); } - getVolumeList(chunkManager: ChunkManager, project: string, dataset: string) { - return chunkManager.memoize.getUncounted( + getVolumeList( + registry: DataSourceRegistry, + project: string, + dataset: string, + options: Partial, + ) { + return registry.chunkManager.memoize.getAsync( { instance: this.instance, type: `brainmaps:${project}:${dataset}:getVolumeList`, }, - () => { - const promise = makeRequest(this.instance, this.credentialsProvider, { - method: "GET", - path: `/v1beta2/volumes?project_id=${project}&dataset_id=${dataset}`, - }) - .then((response) => response.json()) - .then((volumesResponse) => { - const fullyQualifyiedVolumeList = parseAPIResponseList( - volumesResponse, - "volumeId", - ); - const splitPoint = project.length + dataset.length + 2; - const volumeList = []; - for (const volume of fullyQualifyiedVolumeList) { - volumeList.push(volume.substring(splitPoint)); - } - return volumeList; - }); - const description = `${this.instance.description} volume list`; - StatusMessage.forPromise(promise, { - delay: true, - initialMessage: `Retrieving ${description}`, - errorPrefix: `Error retrieving ${description}`, + options, + async (progressOptions) => { + using _span = new ProgressSpan(progressOptions.progressListener, { + message: `Retrieving ${this.instance.description} volume list for ${project}:${dataset}`, }); - return promise; + + const response = await makeRequest( + this.instance, + getCredentialsProvider(registry.credentialsManager), + `/v1beta2/volumes?project_id=${project}&dataset_id=${dataset}`, + progressOptions, + ); + const fullyQualifyiedVolumeList = parseAPIResponseList( + await response.json(), + "volumeId", + ); + const splitPoint = project.length + dataset.length + 2; + const volumeList = []; + for (const volume of fullyQualifyiedVolumeList) { + volumeList.push(volume.substring(splitPoint)); + } + return volumeList; }, ); } - getChangeStackList(chunkManager: ChunkManager, volumeId: string) { - return chunkManager.memoize.getUncounted( + getChangeStackList( + registry: DataSourceRegistry, + volumeId: string, + options: Partial, + ) { + return registry.chunkManager.memoize.getAsync( { instance: this.instance, type: "brainmaps:getChangeStackList", volumeId, }, - () => { - const promise: Promise = makeRequest( - this.instance, - this.credentialsProvider, - { - method: "GET", - path: `/v1beta2/changes/${volumeId}/change_stacks`, - }, - ) - .then((response) => response.json()) - .then((response) => parseChangeStackList(response)); - const description = `change stacks for ${volumeId}`; - StatusMessage.forPromise(promise, { - delay: true, - initialMessage: `Retrieving ${description}.`, - errorPrefix: `Error retrieving ${description}: `, + options, + async (progressOptions) => { + using _span = new ProgressSpan(progressOptions.progressListener, { + message: `Retrieving ${this.instance.description} change stack list for ${volumeId}`, }); - return promise; + const response = await makeRequest( + this.instance, + getCredentialsProvider(registry.credentialsManager), + `/v1beta2/changes/${volumeId}/change_stacks`, + progressOptions, + ); + return parseChangeStackList(await response.json()); }, ); } @@ -1143,7 +1165,11 @@ export class BrainmapsDataSource extends DataSourceProvider { } if (meshName !== undefined) { const volumeId = `${project}:${dataset}:${volume}`; - const meshes = await this.getMeshesInfo(options.chunkManager, volumeId); + const meshes = await this.getMeshesInfo( + options.registry, + volumeId, + options, + ); const results: CompletionWithDescription[] = []; const seenMultiscale = new Set(); for (const mesh of meshes) { @@ -1180,8 +1206,9 @@ export class BrainmapsDataSource extends DataSourceProvider { if (changestack !== undefined) { const volumeId = `${project}:${dataset}:${volume}`; const changeStacks = await this.getChangeStackList( - options.chunkManager, + options.registry, volumeId, + options, ); if (changeStacks === undefined) { throw null; @@ -1196,12 +1223,16 @@ export class BrainmapsDataSource extends DataSourceProvider { offset: providerUrl.length - volume.length, completions: getPrefixMatches( volume, - await this.getVolumeList(options.chunkManager, project, dataset), + await this.getVolumeList(options.registry, project, dataset, options), ), }; } if (dataset !== undefined) { - const datasets = await this.getDatasetList(options.chunkManager, project); + const datasets = await this.getDatasetList( + options.registry, + project, + options, + ); return { offset: providerUrl.length - dataset.length, completions: getPrefixMatches( @@ -1211,7 +1242,7 @@ export class BrainmapsDataSource extends DataSourceProvider { }; } - const projects = await this.getProjectList(options.chunkManager); + const projects = await this.getProjectList(options.registry, options); return { offset: 0, completions: getPrefixMatchesWithDescriptions( diff --git a/src/datasource/brainmaps/register_credentials_provider.ts b/src/datasource/brainmaps/register_credentials_provider.ts index 4d3035b159..f31189e993 100644 --- a/src/datasource/brainmaps/register_credentials_provider.ts +++ b/src/datasource/brainmaps/register_credentials_provider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { defaultCredentialsManager } from "#src/credentials_provider/default_manager.js"; +import { registerDefaultCredentialsProvider } from "#src/credentials_provider/default_manager.js"; import { credentialsKey } from "#src/datasource/brainmaps/api.js"; import { BrainmapsCredentialsProvider } from "#src/datasource/brainmaps/credentials_provider.js"; @@ -22,7 +22,7 @@ import { BrainmapsCredentialsProvider } from "#src/datasource/brainmaps/credenti declare const NEUROGLANCER_BRAINMAPS_CLIENT_ID: string | undefined; if (typeof NEUROGLANCER_BRAINMAPS_CLIENT_ID !== "undefined") { - defaultCredentialsManager.register( + registerDefaultCredentialsProvider( credentialsKey, () => new BrainmapsCredentialsProvider(NEUROGLANCER_BRAINMAPS_CLIENT_ID!), ); diff --git a/src/datasource/brainmaps/register_default.ts b/src/datasource/brainmaps/register_default.ts index 03c94c992f..c06935310e 100644 --- a/src/datasource/brainmaps/register_default.ts +++ b/src/datasource/brainmaps/register_default.ts @@ -15,21 +15,13 @@ */ import type { BrainmapsInstance } from "#src/datasource/brainmaps/api.js"; -import { credentialsKey } from "#src/datasource/brainmaps/api.js"; import { BrainmapsDataSource, productionInstance, } from "#src/datasource/brainmaps/frontend.js"; import { registerProvider } from "#src/datasource/default_provider.js"; -registerProvider( - "brainmaps", - (options) => - new BrainmapsDataSource( - productionInstance, - options.credentialsManager.getCredentialsProvider(credentialsKey), - ), -); +registerProvider(new BrainmapsDataSource(productionInstance, "brainmaps")); declare const NEUROGLANCER_BRAINMAPS_SERVERS: | { [key: string]: BrainmapsInstance } @@ -39,13 +31,6 @@ if (typeof NEUROGLANCER_BRAINMAPS_SERVERS !== "undefined") { for (const [key, instance] of Object.entries( NEUROGLANCER_BRAINMAPS_SERVERS, )) { - registerProvider( - `brainmaps-${key}`, - (options) => - new BrainmapsDataSource( - instance, - options.credentialsManager.getCredentialsProvider(credentialsKey), - ), - ); + registerProvider(new BrainmapsDataSource(instance, `brainmaps-${key}`)); } } diff --git a/src/datasource/deepzoom/backend.ts b/src/datasource/deepzoom/backend.ts index a5bf1e7122..fc5a74d8d6 100644 --- a/src/datasource/deepzoom/backend.ts +++ b/src/datasource/deepzoom/backend.ts @@ -18,17 +18,14 @@ import { decodeJpeg } from "#src/async_computation/decode_jpeg_request.js"; import { decodePng } from "#src/async_computation/decode_png_request.js"; import { requestAsyncComputation } from "#src/async_computation/request.js"; import { WithParameters } from "#src/chunk_manager/backend.js"; -import { WithSharedCredentialsProviderCounterpart } from "#src/credentials_provider/shared_counterpart.js"; import { ImageTileEncoding, ImageTileSourceParameters, } from "#src/datasource/deepzoom/base.js"; +import { WithSharedKvStoreContextCounterpart } from "#src/kvstore/backend.js"; import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; import { VolumeChunkSource } from "#src/sliceview/volume/backend.js"; import { transposeArray2d } from "#src/util/array.js"; -import { isNotFoundError } from "#src/util/http_request.js"; -import type { SpecialProtocolCredentials } from "#src/util/special_protocol_request.js"; -import { fetchSpecialOk } from "#src/util/special_protocol_request.js"; import { registerSharedObject } from "#src/worker_rpc.js"; /* This is enough if support for these aren't needed: @@ -39,11 +36,13 @@ import { registerSharedObject } from "#src/worker_rpc.js"; @registerSharedObject() export class DeepzoomImageTileSource extends WithParameters( - WithSharedCredentialsProviderCounterpart()( - VolumeChunkSource, - ), + WithSharedKvStoreContextCounterpart(VolumeChunkSource), ImageTileSourceParameters, ) { + private tileKvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore( + this.parameters.url, + ); + gridShape = (() => { const gridShape = new Uint32Array(2); const { upperVoxelBound, chunkDataSize } = this.spec; @@ -53,7 +52,7 @@ export class DeepzoomImageTileSource extends WithParameters( return gridShape; })(); - async download(chunk: VolumeChunk, abortSignal: AbortSignal): Promise { + async download(chunk: VolumeChunk, signal: AbortSignal): Promise { const { parameters } = this; // /* This block is enough if support for these aren't needed: @@ -63,7 +62,7 @@ export class DeepzoomImageTileSource extends WithParameters( // const {tilesize, overlap} = parameters; // const [x, y] = chunk.chunkGridPosition; // const url = `${parameters.url}/${x}_${y}.${ImageTileEncoding[parameters.encoding].toLowerCase()}`; - // const response: Blob = await (await fetchSpecialOk(this.credentialsProvider, url, {signal: abortSignal})).blob(); + // const response: Blob = await (await fetchSpecialOk(this.credentialsProvider, url, {signal: signal})).blob(); // const tile = await createImageBitmap(response); // const canvas = new OffscreenCanvas(tilesize, tilesize); // const ctx = canvas.getContext("2d")!; @@ -82,73 +81,71 @@ export class DeepzoomImageTileSource extends WithParameters( const [x, y] = chunk.chunkGridPosition; const ox = x === 0 ? 0 : overlap; const oy = y === 0 ? 0 : overlap; - const url = `${parameters.url}/${x}_${y}.${parameters.format}`; - try { - const responseBuffer = await ( - await fetchSpecialOk(this.credentialsProvider, url, { - signal: abortSignal, - }) - ).arrayBuffer(); - - let tilewidth = 0; - let tileheight = 0; - let tiledata: Uint8Array | undefined; - switch (encoding) { - case ImageTileEncoding.PNG: { - const pngbitmap = await requestAsyncComputation( - decodePng, - abortSignal, - [responseBuffer], - new Uint8Array(responseBuffer), - undefined, - undefined, - undefined, - 3, - 1, - false, - ); - ({ width: tilewidth, height: tileheight } = pngbitmap); - tiledata = transposeArray2d( - pngbitmap.uint8Array, - tilewidth * tileheight, - 3, - ); - break; - } + const path = `${this.tileKvStore.path}/${x}_${y}.${parameters.format}`; + const response = await this.tileKvStore.store.read(path, { + signal, + }); + if (response === undefined) { + return; + } + const responseArray = new Uint8Array(await response.response.arrayBuffer()); - case ImageTileEncoding.JPG: - case ImageTileEncoding.JPEG: { - const jpegbitmap = await requestAsyncComputation( - decodeJpeg, - abortSignal, - [responseBuffer], - new Uint8Array(responseBuffer), - undefined, - undefined, - undefined, - 3, - false, - ); - ({ - uint8Array: tiledata, - width: tilewidth, - height: tileheight, - } = jpegbitmap); - break; - } + let tilewidth = 0; + let tileheight = 0; + let tiledata: Uint8Array | undefined; + switch (encoding) { + case ImageTileEncoding.PNG: { + const pngbitmap = await requestAsyncComputation( + decodePng, + signal, + [responseArray.buffer], + responseArray, + undefined, + undefined, + undefined, + 3, + 1, + false, + ); + ({ width: tilewidth, height: tileheight } = pngbitmap); + tiledata = transposeArray2d( + pngbitmap.uint8Array, + tilewidth * tileheight, + 3, + ); + break; } - if (tiledata !== undefined) { - const t2 = tilesize * tilesize; - const twh = tilewidth * tileheight; - const d = (chunk.data = new Uint8Array(t2 * 3)); - for (let k = 0; k < 3; k++) - for (let j = 0; j < tileheight; j++) - for (let i = 0; i < tilewidth; i++) - d[i + j * tilesize + k * t2] = - tiledata[i + ox + (j + oy) * tilewidth + k * twh]; + + case ImageTileEncoding.JPG: + case ImageTileEncoding.JPEG: { + const jpegbitmap = await requestAsyncComputation( + decodeJpeg, + signal, + [responseArray.buffer], + responseArray, + undefined, + undefined, + undefined, + 3, + false, + ); + ({ + uint8Array: tiledata, + width: tilewidth, + height: tileheight, + } = jpegbitmap); + break; } - } catch (e) { - if (!isNotFoundError(e)) throw e; + } + if (tiledata !== undefined) { + const t2 = tilesize * tilesize; + const twh = tilewidth * tileheight; + const d = (chunk.data = new Uint8Array(t2 * 3)); + for (let k = 0; k < 3; k++) + for (let j = 0; j < tileheight; j++) + for (let i = 0; i < tilewidth; i++) + d[i + j * tilesize + k * t2] = + tiledata[i + ox + (j + oy) * tilewidth + k * twh]; } } } diff --git a/src/datasource/deepzoom/frontend.ts b/src/datasource/deepzoom/frontend.ts index 502e2e6cbc..826d43e438 100644 --- a/src/datasource/deepzoom/frontend.ts +++ b/src/datasource/deepzoom/frontend.ts @@ -15,7 +15,6 @@ */ import { makeDataBoundsBoundingBoxAnnotationSet } from "#src/annotation/index.js"; -import type { ChunkManager } from "#src/chunk_manager/frontend.js"; import { WithParameters } from "#src/chunk_manager/frontend.js"; import type { BoundingBox, @@ -26,25 +25,24 @@ import { makeIdentityTransform, makeIdentityTransformedBoundingBox, } from "#src/coordinate_transform.js"; -import { WithCredentialsProvider } from "#src/credentials_provider/chunk_source_frontend.js"; import { ImageTileEncoding, ImageTileSourceParameters, } from "#src/datasource/deepzoom/base.js"; import type { - CompleteUrlOptions, - ConvertLegacyUrlOptions, DataSource, DataSubsourceEntry, - GetDataSourceOptions, - NormalizeUrlOptions, + GetKvStoreBasedDataSourceOptions, + KvStoreBasedDataSourceProvider, } from "#src/datasource/index.js"; -import { DataSourceProvider } from "#src/datasource/index.js"; -import { - parseProviderUrl, - resolvePath, - unparseProviderUrl, -} from "#src/datasource/precomputed/frontend.js"; +import type { + AutoDetectFileOptions, + AutoDetectMatch, + AutoDetectRegistry, +} from "#src/kvstore/auto_detect.js"; +import { WithSharedKvStoreContext } from "#src/kvstore/chunk_source_frontend.js"; +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; +import { ensureEmptyUrlSuffix } from "#src/kvstore/url.js"; import type { SliceViewSingleResolutionSource } from "#src/sliceview/frontend.js"; import type { VolumeSourceOptions } from "#src/sliceview/volume/base.js"; import { @@ -57,7 +55,6 @@ import { } from "#src/sliceview/volume/frontend.js"; import { transposeNestedArrays } from "#src/util/array.js"; import { DataType } from "#src/util/data_type.js"; -import { completeHttpPath } from "#src/util/http_path_completion.js"; import { verifyEnumString, verifyInt, @@ -65,18 +62,11 @@ import { verifyPositiveInt, verifyString, } from "#src/util/json.js"; -import { getObjectId } from "#src/util/object_id.js"; -import type { - SpecialProtocolCredentials, - SpecialProtocolCredentialsProvider, -} from "#src/util/special_protocol_request.js"; -import { - fetchSpecialOk, - parseSpecialUrl, -} from "#src/util/special_protocol_request.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; /*export*/ class DeepzoomImageTileSource extends WithParameters( - WithCredentialsProvider()(VolumeChunkSource), + WithSharedKvStoreContext(VolumeChunkSource), ImageTileSourceParameters, ) {} @@ -150,12 +140,11 @@ interface LevelInfo { url: string; constructor( - chunkManager: ChunkManager, - public credentialsProvider: SpecialProtocolCredentialsProvider, - /*public*/ url: string, + public sharedKvStoreContext: SharedKvStoreContext, + url: string, public info: PyramidalImageInfo, ) { - super(chunkManager); + super(sharedKvStoreContext.chunkManager); this.url = url.substring(0, url.lastIndexOf(".")) + "_files"; } @@ -196,13 +185,10 @@ interface LevelInfo { chunkSource: this.chunkManager.getChunkSource( DeepzoomImageTileSource, { - credentialsProvider: this.credentialsProvider, + sharedKvStoreContext: this.sharedKvStoreContext, spec, parameters: { - url: resolvePath( - this.url, - (array.length - 1 - index).toString(), - ), + url: `${this.url}/${array.length - 1 - index}/`, encoding: this.info.encoding, format: this.info.format, overlap: this.info.overlap, @@ -228,9 +214,9 @@ interface DZIMetaData { } function getDZIMetadata( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, + options: Partial, ): Promise { if (url.endsWith(".json") || url.includes(".json?")) { /* http://openseadragon.github.io/examples/tilesource-dzi/ @@ -240,16 +226,21 @@ function getDZIMetadata( */ throw new Error("DZI-JSON: OpenSeadragon hack not supported yet."); } - return chunkManager.memoize.getUncounted( + return sharedKvStoreContext.chunkManager.memoize.getAsync( { type: "deepzoom:metadata", url, - credentialsProvider: getObjectId(credentialsProvider), }, - async () => { - const text = await fetchSpecialOk(credentialsProvider, url, {}).then( - (response) => response.text(), - ); + options, + async (progressOptions) => { + using _span = new ProgressSpan(progressOptions.progressListener, { + message: `Reading Deep Zoom metadata from ${url}`, + }); + const { response } = await sharedKvStoreContext.kvStoreContext.read(url, { + ...progressOptions, + throwIfMissing: true, + }); + const text = await response.text(); const xml = new DOMParser().parseFromString(text, "text/xml"); const image = xml.documentElement; const size = verifyObject(image.getElementsByTagName("Size").item(0)); @@ -266,16 +257,14 @@ function getDZIMetadata( ); } -async function getImageDataSource( - options: GetDataSourceOptions, - credentialsProvider: SpecialProtocolCredentialsProvider, +function getImageDataSource( + sharedKvStoreContext: SharedKvStoreContext, url: string, metadata: DZIMetaData, -): Promise { +): DataSource { const info = buildPyramidalImageInfo(metadata); const volume = new DeepzoomPyramidalImageTileSource( - options.chunkManager, - credentialsProvider, + sharedKvStoreContext, url, info, ); @@ -296,53 +285,61 @@ async function getImageDataSource( }, }, ]; - return { modelTransform: makeIdentityTransform(modelSpace), subsources }; + return { + modelTransform: makeIdentityTransform(modelSpace), + subsources, + canonicalUrl: `${url}|deepzoom:`, + }; } -export class DeepzoomDataSource extends DataSourceProvider { - get description() { - return "Deep Zoom file-backed data source"; - } - - normalizeUrl(options: NormalizeUrlOptions): string { - const { url, parameters } = parseProviderUrl(options.providerUrl); - return ( - options.providerProtocol + "://" + unparseProviderUrl(url, parameters) - ); +export class DeepzoomDataSource implements KvStoreBasedDataSourceProvider { + get scheme() { + return "deepzoom"; } - - convertLegacyUrl(options: ConvertLegacyUrlOptions): string { - const { url, parameters } = parseProviderUrl(options.providerUrl); - return ( - options.providerProtocol + "://" + unparseProviderUrl(url, parameters) - ); + get description() { + return "Deep Zoom data source"; } - get(options: GetDataSourceOptions): Promise { - const { url: providerUrl, parameters } = parseProviderUrl( - options.providerUrl, - ); - return options.chunkManager.memoize.getUncounted( - { type: "deepzoom:get", providerUrl, parameters }, - async (): Promise => { - const { url, credentialsProvider } = parseSpecialUrl( - providerUrl, - options.credentialsManager, - ); + get(options: GetKvStoreBasedDataSourceOptions): Promise { + ensureEmptyUrlSuffix(options.url); + return options.registry.chunkManager.memoize.getAsync( + { type: "deepzoom:get", url: options.kvStoreUrl }, + options, + async (progressOptions): Promise => { const metadata = await getDZIMetadata( - options.chunkManager, - credentialsProvider, - url, + options.registry.sharedKvStoreContext, + options.kvStoreUrl, + progressOptions, + ); + return getImageDataSource( + options.registry.sharedKvStoreContext, + options.kvStoreUrl, + metadata, ); - return getImageDataSource(options, credentialsProvider, url, metadata); }, ); } - completeUrl(options: CompleteUrlOptions) { - return completeHttpPath( - options.credentialsManager, - options.providerUrl, - options.abortSignal, - ); +} + +async function detectFormat( + options: AutoDetectFileOptions, +): Promise { + const text = new TextDecoder().decode(options.prefix); + const xml = new DOMParser().parseFromString(text, "text/xml"); + if ( + xml.documentElement.tagName === "Image" && + xml.documentElement.namespaceURI === + "http://schemas.microsoft.com/deepzoom/2009" + ) { + return [{ suffix: "deepzoom:", description: "Deep Zoom" }]; } + return []; +} + +export function registerAutoDetect(registry: AutoDetectRegistry) { + registry.registerFileFormat({ + prefixLength: 500, + suffixLength: 0, + match: detectFormat, + }); } diff --git a/src/datasource/deepzoom/register_default.ts b/src/datasource/deepzoom/register_default.ts index eb9c8a387d..cf97508b87 100644 --- a/src/datasource/deepzoom/register_default.ts +++ b/src/datasource/deepzoom/register_default.ts @@ -14,7 +14,18 @@ * limitations under the License. */ -import { DeepzoomDataSource } from "#src/datasource/deepzoom/frontend.js"; -import { registerProvider } from "#src/datasource/default_provider.js"; +import { + DeepzoomDataSource, + registerAutoDetect, +} from "#src/datasource/deepzoom/frontend.js"; +import { + dataSourceAutoDetectRegistry, + registerKvStoreBasedDataProvider, + registerProvider, +} from "#src/datasource/default_provider.js"; +import { KvStoreBasedDataSourceLegacyUrlAdapter } from "#src/datasource/index.js"; -registerProvider("deepzoom", () => new DeepzoomDataSource()); +const provider = new DeepzoomDataSource(); +registerKvStoreBasedDataProvider(provider); +registerProvider(new KvStoreBasedDataSourceLegacyUrlAdapter(provider)); +registerAutoDetect(dataSourceAutoDetectRegistry); diff --git a/src/datasource/default_provider.ts b/src/datasource/default_provider.ts index bc72938543..85bf8b0aa7 100644 --- a/src/datasource/default_provider.ts +++ b/src/datasource/default_provider.ts @@ -18,32 +18,47 @@ * @file Facility for registering default data sources. */ -import type { CredentialsManager } from "#src/credentials_provider/index.js"; -import type { DataSourceProvider } from "#src/datasource/index.js"; -import { DataSourceProviderRegistry } from "#src/datasource/index.js"; -import type { Owned } from "#src/util/disposable.js"; +import type { SharedCredentialsManager } from "#src/credentials_provider/shared.js"; +import type { + DataSourceProvider, + KvStoreBasedDataSourceProvider, +} from "#src/datasource/index.js"; +import { DataSourceRegistry } from "#src/datasource/index.js"; +import { LocalDataSourceProvider } from "#src/datasource/local.js"; +import { AutoDetectRegistry } from "#src/kvstore/auto_detect.js"; +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; export interface ProviderOptions { - credentialsManager: CredentialsManager; + credentialsManager: SharedCredentialsManager; + kvStoreContext: SharedKvStoreContext; } -export type ProviderFactory = ( - options: ProviderOptions, -) => Owned; -const providerFactories = new Map(); +const providers: DataSourceProvider[] = []; +const kvStoreBasedProviders: KvStoreBasedDataSourceProvider[] = []; +export const dataSourceAutoDetectRegistry = new AutoDetectRegistry(); -export function registerProvider(name: string, factory: ProviderFactory) { - providerFactories.set(name, factory); +export function registerProvider(provider: DataSourceProvider) { + providers.push(provider); +} + +export function registerKvStoreBasedDataProvider( + provider: KvStoreBasedDataSourceProvider, +) { + kvStoreBasedProviders.push(provider); } export function getDefaultDataSourceProvider(options: ProviderOptions) { - const provider = new DataSourceProviderRegistry(options.credentialsManager); - for (const [name, factory] of providerFactories) { - try { - provider.register(name, factory(options)); - } catch (e) { - console.warn(`Skipping ${name} data source: ${e}`); - } + const registry = new DataSourceRegistry(options.kvStoreContext); + registry.register(new LocalDataSourceProvider()); + for (const provider of providers) { + registry.register(provider); + } + for (const provider of kvStoreBasedProviders) { + registry.registerKvStoreBasedProvider(provider); } - return provider; + options.kvStoreContext.kvStoreContext.autoDetectRegistry.copyTo( + registry.autoDetectRegistry, + ); + dataSourceAutoDetectRegistry.copyTo(registry.autoDetectRegistry); + return registry; } diff --git a/src/datasource/dvid/api.ts b/src/datasource/dvid/api.ts index 32d556cb3d..c9f0ed1ed2 100644 --- a/src/datasource/dvid/api.ts +++ b/src/datasource/dvid/api.ts @@ -18,7 +18,7 @@ * limitations under the License. */ -import { fetchWithCredentials } from "#src/credentials_provider/http_request.js"; +import { fetchOkWithCredentials } from "#src/credentials_provider/http_request.js"; import type { CredentialsProvider } from "#src/credentials_provider/index.js"; export interface DVIDToken { @@ -76,7 +76,7 @@ export function fetchWithDVIDCredentials( input: string, init: RequestInit, ): Promise { - return fetchWithCredentials( + return fetchOkWithCredentials( credentialsProvider, input, init, diff --git a/src/datasource/dvid/backend.ts b/src/datasource/dvid/backend.ts index 7edc1602f1..c136a10dad 100644 --- a/src/datasource/dvid/backend.ts +++ b/src/datasource/dvid/backend.ts @@ -64,7 +64,7 @@ export class DVIDSkeletonSource extends DVIDSource( SkeletonSource, SkeletonSourceParameters, ) { - download(chunk: SkeletonChunk, abortSignal: AbortSignal) { + download(chunk: SkeletonChunk, signal: AbortSignal) { const { parameters } = this; const bodyid = `${chunk.objectId}`; const url = @@ -76,7 +76,7 @@ export class DVIDSkeletonSource extends DVIDSource( this.credentialsProvider, appendQueryStringForDvid(url, parameters.user), { - signal: abortSignal, + signal: signal, }, ) .then((response) => response.arrayBuffer()) @@ -117,7 +117,7 @@ export class DVIDMeshSource extends DVIDSource( return Promise.resolve(undefined); } - downloadFragment(chunk: FragmentChunk, abortSignal: AbortSignal) { + downloadFragment(chunk: FragmentChunk, signal: AbortSignal) { const { parameters } = this; const dvidInstance = new DVIDInstance( parameters.baseUrl, @@ -132,7 +132,7 @@ export class DVIDMeshSource extends DVIDSource( this.credentialsProvider, appendQueryStringForDvid(meshUrl, parameters.user), { - signal: abortSignal, + signal: signal, }, ) .then((response) => response.arrayBuffer()) @@ -145,7 +145,7 @@ export class DVIDVolumeChunkSource extends DVIDSource( VolumeChunkSource, VolumeChunkSourceParameters, ) { - async download(chunk: VolumeChunk, abortSignal: AbortSignal) { + async download(chunk: VolumeChunk, signal: AbortSignal) { const params = this.parameters; let path: string; { @@ -161,11 +161,11 @@ export class DVIDVolumeChunkSource extends DVIDSource( const response = await fetchWithDVIDCredentials( this.credentialsProvider, appendQueryStringForDvid(`${params.baseUrl}${path}`, params.user), - { signal: abortSignal }, + { signal: signal }, ).then((response) => response.arrayBuffer()); await decoder( chunk, - abortSignal, + signal, params.encoding === VolumeChunkEncoding.JPEG ? response.slice(16) : response, diff --git a/src/datasource/dvid/credentials_provider.ts b/src/datasource/dvid/credentials_provider.ts index 1bdf2e9f62..74dd40f515 100644 --- a/src/datasource/dvid/credentials_provider.ts +++ b/src/datasource/dvid/credentials_provider.ts @@ -26,15 +26,16 @@ import { import { getCredentialsWithStatus } from "#src/credentials_provider/interactive_credentials_provider.js"; import type { DVIDToken } from "#src/datasource/dvid/api.js"; import { fetchOk } from "#src/util/http_request.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; async function getAuthToken( authServer: string, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise { const response = await fetchOk(authServer, { method: "GET", credentials: "include", - signal: abortSignal, + signal: signal, }); const token = await response.text(); return { token }; @@ -45,16 +46,19 @@ class BaseDVIDCredentialsProvider extends CredentialsProvider { super(); } - get = makeCredentialsGetter(async (abortSignal) => { + get = makeCredentialsGetter(async (options) => { const { authServer } = this; if (!authServer) return { token: "" }; + using _span = new ProgressSpan(options.progressListener, { + message: `Requesting DVID access token from ${authServer}`, + }); return await getCredentialsWithStatus( { description: `DVID server ${this.authServer}`, supportsImmediate: true, - get: async (abortSignal, immediate) => { + get: async (signal, immediate) => { if (immediate) { - return await getAuthToken(authServer, abortSignal); + return await getAuthToken(authServer, signal); } // In the current DVID setup, https://flyemlogin. is expected for the login server const match = authServer.match(/^[^/]+\/\/[^/.]+\.([^/]+)/); @@ -70,7 +74,7 @@ class BaseDVIDCredentialsProvider extends CredentialsProvider { } }, }, - abortSignal, + options.signal, ); }); } diff --git a/src/datasource/dvid/frontend.ts b/src/datasource/dvid/frontend.ts index 82b5ec2bc7..d4db7ffa4e 100644 --- a/src/datasource/dvid/frontend.ts +++ b/src/datasource/dvid/frontend.ts @@ -29,10 +29,7 @@ import { makeIdentityTransformedBoundingBox, } from "#src/coordinate_transform.js"; import { WithCredentialsProvider } from "#src/credentials_provider/chunk_source_frontend.js"; -import type { - CredentialsManager, - CredentialsProvider, -} from "#src/credentials_provider/index.js"; +import type { CredentialsProvider } from "#src/credentials_provider/index.js"; import type { DVIDToken } from "#src/datasource/dvid/api.js"; import { credentialsKey, @@ -50,8 +47,8 @@ import type { CompletionResult, DataSource, GetDataSourceOptions, + DataSourceProvider, } from "#src/datasource/index.js"; -import { DataSourceProvider } from "#src/datasource/index.js"; import { MeshSource } from "#src/mesh/frontend.js"; import { SkeletonSource } from "#src/skeleton/frontend.js"; import type { SliceViewSingleResolutionSource } from "#src/sliceview/frontend.js"; @@ -65,7 +62,6 @@ import { MultiscaleVolumeChunkSource, VolumeChunkSource, } from "#src/sliceview/volume/frontend.js"; -import { StatusMessage } from "#src/status.js"; import { transposeNestedArrays } from "#src/util/array.js"; import { applyCompletionOffset, @@ -85,6 +81,8 @@ import { verifyPositiveInt, verifyString, } from "#src/util/json.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; const serverDataTypes = new Map(); serverDataTypes.set("uint8", DataType.UINT8); @@ -453,26 +451,21 @@ export function getServerInfo( chunkManager: ChunkManager, baseUrl: string, credentialsProvider: CredentialsProvider, + options: Partial, ) { - return chunkManager.memoize.getUncounted( + return chunkManager.memoize.getAsync( { type: "dvid:getServerInfo", baseUrl }, - () => { - const result = fetchWithDVIDCredentials( + options, + async (progressOptions) => { + using _span = new ProgressSpan(progressOptions.progressListener, { + message: `Retrieving repository info for DVID server ${baseUrl}`, + }); + const response = await fetchWithDVIDCredentials( credentialsProvider, `${baseUrl}/api/repos/info`, - { - method: "GET", - }, - ) - .then((response) => response.json()) - .then((response) => new ServerInfo(response)); - const description = `repository info for DVID server ${baseUrl}`; - StatusMessage.forPromise(result, { - initialMessage: `Retrieving ${description}.`, - delay: true, - errorPrefix: `Error retrieving ${description}: `, - }); - return result; + progressOptions, + ); + return new ServerInfo(await response.json()); }, ); } @@ -572,7 +565,7 @@ function getVolumeSource( }); const volume = new DvidMultiscaleVolumeChunkSource( - options.chunkManager, + options.registry.chunkManager, baseUrl, nodeKey, dataInstanceKey, @@ -599,7 +592,7 @@ function getVolumeSource( id: "meshes", default: true, subsource: { - mesh: options.chunkManager.getChunkSource(DVIDMeshSource, { + mesh: options.registry.chunkManager.getChunkSource(DVIDMeshSource, { parameters: { ...sourceParameters, dataInstanceKey: info.meshSrc, @@ -615,7 +608,7 @@ function getVolumeSource( id: "skeletons", default: true, subsource: { - mesh: options.chunkManager.getChunkSource(DVIDSkeletonSource, { + mesh: options.registry.chunkManager.getChunkSource(DVIDSkeletonSource, { parameters: { ...sourceParameters, dataInstanceKey: info.skeletonSrc, @@ -642,16 +635,17 @@ export function getDataSource( const sourceParameters = parseSourceUrl(options.providerUrl); const { baseUrl, nodeKey, dataInstanceKey } = sourceParameters; - return options.chunkManager.memoize.getUncounted( + return options.registry.chunkManager.memoize.getAsync( { type: "dvid:MultiscaleVolumeChunkSource", baseUrl, nodeKey: nodeKey, dataInstanceKey, }, - async () => { + options, + async (progressOptions) => { const credentailsProvider = - options.credentialsManager.getCredentialsProvider( + options.registry.credentialsManager.getCredentialsProvider( credentialsKey, { dvidServer: sourceParameters.baseUrl, @@ -659,9 +653,10 @@ export function getDataSource( }, ); const serverInfo = await getServerInfo( - options.chunkManager, + options.registry.chunkManager, baseUrl, credentailsProvider, + progressOptions, ); const repositoryInfo = serverInfo.getNode(nodeKey); if (repositoryInfo === undefined) { @@ -744,12 +739,13 @@ export async function completeUrl( const authServer = getDefaultAuthServer(baseUrl); const serverInfo = await getServerInfo( - options.chunkManager, + options.registry.chunkManager, baseUrl, - options.credentialsManager.getCredentialsProvider( + options.registry.credentialsManager.getCredentialsProvider( credentialsKey, { dvidServer: baseUrl, authServer }, ), + options, ); return applyCompletionOffset( baseUrl.length + 1, @@ -757,11 +753,10 @@ export async function completeUrl( ); } -export class DVIDDataSource extends DataSourceProvider { - constructor(public credentialsManager: CredentialsManager) { - super(); +export class DVIDDataSource implements DataSourceProvider { + get scheme() { + return "dvid"; } - get description() { return "DVID"; } diff --git a/src/datasource/dvid/register_credentials_provider.ts b/src/datasource/dvid/register_credentials_provider.ts index d14e3bb2e9..42891dc49a 100644 --- a/src/datasource/dvid/register_credentials_provider.ts +++ b/src/datasource/dvid/register_credentials_provider.ts @@ -18,11 +18,11 @@ * limitations under the License. */ -import { defaultCredentialsManager } from "#src/credentials_provider/default_manager.js"; +import { registerDefaultCredentialsProvider } from "#src/credentials_provider/default_manager.js"; import { credentialsKey } from "#src/datasource/dvid/api.js"; import { DVIDCredentialsProvider } from "#src/datasource/dvid/credentials_provider.js"; -defaultCredentialsManager.register( +registerDefaultCredentialsProvider( credentialsKey, (params: { dvidServer: string; authServer: string | undefined }) => new DVIDCredentialsProvider(params.dvidServer, params.authServer), diff --git a/src/datasource/dvid/register_default.ts b/src/datasource/dvid/register_default.ts index ac96fb56f7..810a8dbb79 100644 --- a/src/datasource/dvid/register_default.ts +++ b/src/datasource/dvid/register_default.ts @@ -17,7 +17,4 @@ import { registerProvider } from "#src/datasource/default_provider.js"; import { DVIDDataSource } from "#src/datasource/dvid/frontend.js"; -registerProvider( - "dvid", - (options) => new DVIDDataSource(options.credentialsManager), -); +registerProvider(new DVIDDataSource()); diff --git a/src/datasource/enabled_frontend_modules.ts b/src/datasource/enabled_frontend_modules.ts index 96b934b7b7..264c63800b 100644 --- a/src/datasource/enabled_frontend_modules.ts +++ b/src/datasource/enabled_frontend_modules.ts @@ -7,13 +7,12 @@ import "#datasource/deepzoom/register_default"; import "#datasource/dvid/register_default"; import "#datasource/dvid/register_credentials_provider"; import "#datasource/graphene/register_default"; -import "#datasource/middleauth/register_credentials_provider"; import "#datasource/n5/register_default"; -import "#datasource/ngauth/register_credentials_provider"; import "#datasource/nggraph/register_default"; import "#datasource/nifti/register_default"; import "#datasource/obj/register_default"; import "#datasource/precomputed/register_default"; +import "#datasource/python/register_default"; import "#datasource/render/register_default"; import "#datasource/vtk/register_default"; import "#datasource/zarr/register_default"; diff --git a/src/datasource/graphene/backend.ts b/src/datasource/graphene/backend.ts index 174fd2d0b6..987c48d831 100644 --- a/src/datasource/graphene/backend.ts +++ b/src/datasource/graphene/backend.ts @@ -22,7 +22,6 @@ import { ChunkSource, } from "#src/chunk_manager/backend.js"; import { ChunkPriorityTier, ChunkState } from "#src/chunk_manager/base.js"; -import { WithSharedCredentialsProviderCounterpart } from "#src/credentials_provider/shared_counterpart.js"; import type { ChunkedGraphChunkSpecification } from "#src/datasource/graphene/base.js"; import { getGrapheneFragmentKey, @@ -33,8 +32,12 @@ import { CHUNKED_GRAPH_RENDER_LAYER_UPDATE_SOURCES_RPC_ID, RENDER_RATIO_LIMIT, isBaseSegmentId, + parseGrapheneError, } from "#src/datasource/graphene/base.js"; import { decodeManifestChunk } from "#src/datasource/precomputed/backend.js"; +import { WithSharedKvStoreContextCounterpart } from "#src/kvstore/backend.js"; +import type { KvStoreWithPath, ReadResponse } from "#src/kvstore/index.js"; +import { readKvStore } from "#src/kvstore/index.js"; import type { FragmentChunk, ManifestChunk } from "#src/mesh/backend.js"; import { assignMeshFragmentData, MeshSource } from "#src/mesh/backend.js"; import { decodeDraco } from "#src/mesh/draco/index.js"; @@ -59,13 +62,8 @@ import { } from "#src/sliceview/base.js"; import { computeChunkBounds } from "#src/sliceview/volume/backend.js"; import { Uint64Set } from "#src/uint64_set.js"; -import { fetchSpecialHttpByteRange } from "#src/util/byte_range_http_requests.js"; import { vec3, vec3Key } from "#src/util/geom.js"; -import type { - SpecialProtocolCredentials, - SpecialProtocolCredentialsProvider, -} from "#src/util/special_protocol_request.js"; -import { fetchSpecialOk } from "#src/util/special_protocol_request.js"; +import { HttpError } from "#src/util/http_request.js"; import { Uint64 } from "#src/util/uint64.js"; import { getBasePriority, @@ -75,73 +73,67 @@ import { import type { RPC } from "#src/worker_rpc.js"; import { registerSharedObject, registerRPC } from "#src/worker_rpc.js"; -function getVerifiedFragmentPromise( - credentialsProvider: SpecialProtocolCredentialsProvider, - chunk: FragmentChunk, - parameters: MeshSourceParameters, - abortSignal: AbortSignal, -) { - if (chunk.fragmentId && chunk.fragmentId.charAt(0) === "~") { - const parts = chunk.fragmentId.substr(1).split(":"); - const startOffset = Number(parts[1]); - const endOffset = startOffset + Number(parts[2]); - return fetchSpecialHttpByteRange( - credentialsProvider, - `${parameters.fragmentUrl}/initial/${parts[0]}`, - startOffset, - endOffset, - abortSignal, +function downloadFragmentWithSharding( + fragmentKvStore: KvStoreWithPath, + fragmentId: string, + signal: AbortSignal, +): Promise { + if (fragmentId && fragmentId.charAt(0) === "~") { + const parts = fragmentId.substring(1).split(":"); + const byteRange = { offset: Number(parts[1]), length: Number(parts[2]) }; + return readKvStore( + fragmentKvStore.store, + `${fragmentKvStore.path}initial/${parts[0]}`, + { signal, byteRange, throwIfMissing: true }, ); } - return fetchSpecialOk( - credentialsProvider, - `${parameters.fragmentUrl}/dynamic/${chunk.fragmentId}`, - { signal: abortSignal }, - ).then((response) => response.arrayBuffer()); + return readKvStore( + fragmentKvStore.store, + `${fragmentKvStore.path}dynamic/${fragmentId}`, + { signal, throwIfMissing: true }, + ); } -function getFragmentDownloadPromise( - credentialsProvider: SpecialProtocolCredentialsProvider, - chunk: FragmentChunk, +function downloadFragment( + fragmentKvStore: KvStoreWithPath, + fragmentId: string, parameters: MeshSourceParameters, - abortSignal: AbortSignal, -) { - let fragmentDownloadPromise; + signal: AbortSignal, +): Promise { if (parameters.sharding) { - fragmentDownloadPromise = getVerifiedFragmentPromise( - credentialsProvider, - chunk, - parameters, - abortSignal, - ); + return downloadFragmentWithSharding(fragmentKvStore, fragmentId, signal); } else { - fragmentDownloadPromise = fetchSpecialOk( - credentialsProvider, - `${parameters.fragmentUrl}/${chunk.fragmentId}`, - { signal: abortSignal }, - ).then((response) => response.arrayBuffer()); + return readKvStore( + fragmentKvStore.store, + `${fragmentKvStore.path}/${fragmentId}`, + { signal, throwIfMissing: true }, + ); } - return fragmentDownloadPromise; } async function decodeDracoFragmentChunk( chunk: FragmentChunk, - response: ArrayBuffer, + response: Uint8Array, ) { - const rawMesh = await decodeDraco(new Uint8Array(response)); + const rawMesh = await decodeDraco(response); assignMeshFragmentData(chunk, rawMesh); } @registerSharedObject() export class GrapheneMeshSource extends WithParameters( - WithSharedCredentialsProviderCounterpart()( - MeshSource, - ), + WithSharedKvStoreContextCounterpart(MeshSource), MeshSourceParameters, ) { manifestRequestCount = new Map(); newSegments = new Uint64Set(); + manifestKvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore( + this.parameters.manifestUrl, + ); + fragmentKvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore( + this.parameters.fragmentUrl, + ); + addNewSegment(segment: Uint64) { const { newSegments } = this; newSegments.add(segment); @@ -151,48 +143,49 @@ export class GrapheneMeshSource extends WithParameters( }, TEN_MINUTES); } - async download(chunk: ManifestChunk, abortSignal: AbortSignal) { + async download(chunk: ManifestChunk, signal: AbortSignal) { const { parameters, newSegments, manifestRequestCount } = this; if (isBaseSegmentId(chunk.objectId, parameters.nBitsForLayerId)) { return decodeManifestChunk(chunk, { fragments: [] }); } - const url = `${parameters.manifestUrl}/manifest`; - const manifestUrl = `${url}/${chunk.objectId}:${parameters.lod}?verify=1&prepend_seg_ids=1`; - await fetchSpecialOk(this.credentialsProvider, manifestUrl, { - signal: abortSignal, - }) - .then((response) => response.json()) - .then((response) => { - const chunkIdentifier = manifestUrl; - if (newSegments.has(chunk.objectId)) { - const requestCount = - (manifestRequestCount.get(chunkIdentifier) || 0) + 1; - manifestRequestCount.set(chunkIdentifier, requestCount); - setTimeout( - () => { - this.chunkManager.queueManager.updateChunkState( - chunk, - ChunkState.QUEUED, - ); - }, - 2 ** requestCount * 1000, + const { manifestKvStore } = this; + const manifestPath = `${manifestKvStore.path}/manifest/${chunk.objectId}:${parameters.lod}?verify=1&prepend_seg_ids=1`; + const response = await ( + await readKvStore(manifestKvStore.store, manifestPath, { + throwIfMissing: true, + signal, + }) + ).response.json(); + const chunkIdentifier = manifestPath; + if (newSegments.has(chunk.objectId)) { + const requestCount = (manifestRequestCount.get(chunkIdentifier) ?? 0) + 1; + manifestRequestCount.set(chunkIdentifier, requestCount); + setTimeout( + () => { + this.chunkManager.queueManager.updateChunkState( + chunk, + ChunkState.QUEUED, ); - } else { - manifestRequestCount.delete(chunkIdentifier); - } - return decodeManifestChunk(chunk, response); - }); + }, + 2 ** requestCount * 1000, + ); + } else { + manifestRequestCount.delete(chunkIdentifier); + } + return decodeManifestChunk(chunk, response); } - async downloadFragment(chunk: FragmentChunk, abortSignal: AbortSignal) { - const { parameters } = this; - const response = await getFragmentDownloadPromise( - undefined, + async downloadFragment(chunk: FragmentChunk, signal: AbortSignal) { + const { response } = await downloadFragment( + this.fragmentKvStore, + chunk.fragmentId!, + this.parameters, + signal, + ); + await decodeDracoFragmentChunk( chunk, - parameters, - abortSignal, + new Uint8Array(await response.arrayBuffer()), ); - await decodeDracoFragmentChunk(chunk, response); } getFragmentKey(objectKey: string | null, fragmentId: string) { @@ -250,9 +243,7 @@ function decodeChunkedGraphChunk(leaves: string[]) { @registerSharedObject() export class GrapheneChunkedGraphChunkSource extends WithParameters( - WithSharedCredentialsProviderCounterpart()( - ChunkSource, - ), + WithSharedKvStoreContextCounterpart(ChunkSource), ChunkedGraphSourceParameters, ) { spec: ChunkedGraphChunkSpecification; @@ -260,6 +251,10 @@ export class GrapheneChunkedGraphChunkSource extends WithParameters( tempChunkDataSize: Uint32Array; tempChunkPosition: Float32Array; + kvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore( + this.parameters.url, + ); + constructor(rpc: RPC, options: any) { super(rpc, options); this.spec = options.spec; @@ -268,11 +263,7 @@ export class GrapheneChunkedGraphChunkSource extends WithParameters( this.tempChunkPosition = new Float32Array(rank); } - async download( - chunk: ChunkedGraphChunk, - abortSignal: AbortSignal, - ): Promise { - const { parameters } = this; + async download(chunk: ChunkedGraphChunk, signal: AbortSignal): Promise { const chunkPosition = this.computeChunkBounds(chunk); const chunkDataSize = chunk.chunkDataSize!; const bounds = @@ -280,20 +271,25 @@ export class GrapheneChunkedGraphChunkSource extends WithParameters( `${chunkPosition[1]}-${chunkPosition[1] + chunkDataSize[1]}_` + `${chunkPosition[2]}-${chunkPosition[2] + chunkDataSize[2]}`; - const request = fetchSpecialOk( - this.credentialsProvider, - `${parameters.url}/${chunk.segment}/leaves?int64_as_str=1&bounds=${bounds}`, - { signal: abortSignal }, + const { kvStore } = this; + + const request = readKvStore( + kvStore.store, + `${kvStore.path}/${chunk.segment}/leaves?int64_as_str=1&bounds=${bounds}`, + { signal, throwIfMissing: true }, ); await this.withErrorMessage( request, `Fetching leaves of segment ${chunk.segment} in region ${bounds}: `, ) - .then((res) => res.json()) + .then((res) => res.response.json()) .then((res) => { chunk.leaves = decodeChunkedGraphChunk(res.leaf_ids); }) - .catch((err) => console.error(err)); + .catch((err) => { + if (err instanceof Error && err.name === "AbortError") return; + console.error(err); + }); } getChunk(chunkGridPosition: Float32Array, segment: Uint64) { @@ -312,21 +308,17 @@ export class GrapheneChunkedGraphChunkSource extends WithParameters( return computeChunkBounds(this, chunk); } - async withErrorMessage( - promise: Promise, + async withErrorMessage( + promise: Promise, errorPrefix: string, - ): Promise { - const response = await promise; - if (response.ok) { - return response; - } - let msg: string; - try { - msg = (await response.json()).message; - } catch { - msg = await response.text(); - } - throw new Error(`[${response.status}] ${errorPrefix}${msg}`); + ): Promise { + return promise.catch(async (e) => { + if (e instanceof HttpError && e.response) { + const msg = await parseGrapheneError(e); + throw new Error(`[${e.response.status}] ${errorPrefix}${msg ?? ""}`); + } + throw e; + }); } } diff --git a/src/datasource/graphene/base.ts b/src/datasource/graphene/base.ts index 28a6403af3..aedf2a291b 100644 --- a/src/datasource/graphene/base.ts +++ b/src/datasource/graphene/base.ts @@ -25,6 +25,7 @@ import type { } from "#src/sliceview/base.js"; import { makeSliceViewChunkSpecification } from "#src/sliceview/base.js"; import type { mat4 } from "#src/util/geom.js"; +import type { HttpError } from "#src/util/http_request.js"; import { Uint64 } from "#src/util/uint64.js"; @@ -137,3 +138,16 @@ export function makeChunkedGraphChunkSpecification( export interface ChunkedGraphChunkSource extends SliceViewChunkSource { spec: ChunkedGraphChunkSpecification; } + +export async function parseGrapheneError(e: HttpError) { + if (e.response) { + let msg: string; + if (e.response.headers.get("content-type") === "application/json") { + msg = (await e.response.json()).message; + } else { + msg = await e.response.text(); + } + return msg; + } + return undefined; +} diff --git a/src/datasource/graphene/frontend.ts b/src/datasource/graphene/frontend.ts index 71790be466..b0966e59ae 100644 --- a/src/datasource/graphene/frontend.ts +++ b/src/datasource/graphene/frontend.ts @@ -34,14 +34,13 @@ import { LayerChunkProgressInfo } from "#src/chunk_manager/base.js"; import type { ChunkManager } from "#src/chunk_manager/frontend.js"; import { WithParameters } from "#src/chunk_manager/frontend.js"; import { makeIdentityTransform } from "#src/coordinate_transform.js"; -import { WithCredentialsProvider } from "#src/credentials_provider/chunk_source_frontend.js"; -import type { CredentialsManager } from "#src/credentials_provider/index.js"; import type { ChunkedGraphChunkSource as ChunkedGraphChunkSourceInterface, ChunkedGraphChunkSpecification, MultiscaleMeshMetadata, } from "#src/datasource/graphene/base.js"; import { + parseGrapheneError, CHUNKED_GRAPH_LAYER_RPC_ID, CHUNKED_GRAPH_RENDER_LAYER_UPDATE_SOURCES_RPC_ID, ChunkedGraphSourceParameters, @@ -54,10 +53,11 @@ import { } from "#src/datasource/graphene/base.js"; import type { DataSource, + DataSourceLookupResult, DataSubsourceEntry, - GetDataSourceOptions, + GetKvStoreBasedDataSourceOptions, + KvStoreBasedDataSourceProvider, } from "#src/datasource/index.js"; -import { RedirectError } from "#src/datasource/index.js"; import type { ShardingParameters } from "#src/datasource/precomputed/base.js"; import { DataEncoding, @@ -67,11 +67,16 @@ import type { MultiscaleVolumeInfo } from "#src/datasource/precomputed/frontend. import { getSegmentPropertyMap, parseMultiscaleVolumeInfo, - parseProviderUrl, - PrecomputedDataSource, PrecomputedMultiscaleVolumeChunkSource, - resolvePath, } from "#src/datasource/precomputed/frontend.js"; +import { WithSharedKvStoreContext } from "#src/kvstore/chunk_source_frontend.js"; +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; +import { HttpKvStore } from "#src/kvstore/http/index.js"; +import { + ensureEmptyUrlSuffix, + kvstoreEnsureDirectoryPipelineUrl, + pipelineUrlJoin, +} from "#src/kvstore/url.js"; import type { LayerView, MouseSelectionState, @@ -160,6 +165,7 @@ import type { ValueOrError } from "#src/util/error.js"; import { makeValueOrError, valueOrThrow } from "#src/util/error.js"; import { EventActionMap } from "#src/util/event_action_map.js"; import { mat4, vec3, vec4 } from "#src/util/geom.js"; +import type { FetchOk } from "#src/util/http_request.js"; import { HttpError, isNotFoundError } from "#src/util/http_request.js"; import { parseArray, @@ -178,16 +184,9 @@ import { verifyPositiveInt, verifyString, } from "#src/util/json.js"; -import { getObjectId } from "#src/util/object_id.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; import { NullarySignal } from "#src/util/signal.js"; -import type { - SpecialProtocolCredentials, - SpecialProtocolCredentialsProvider, -} from "#src/util/special_protocol_request.js"; -import { - fetchSpecialOk, - parseSpecialUrl, -} from "#src/util/special_protocol_request.js"; import type { Trackable } from "#src/util/trackable.js"; import { Uint64 } from "#src/util/uint64.js"; import { makeDeleteButton } from "#src/widget/delete_button.js"; @@ -213,7 +212,7 @@ const TRANSPARENT_COLOR_PACKED = new Uint64(packColor(TRANSPARENT_COLOR)); const MULTICUT_OFF_COLOR = vec4.fromValues(0, 0, 0, 0.5); class GrapheneMeshSource extends WithParameters( - WithCredentialsProvider()(MeshSource), + WithSharedKvStoreContext(MeshSource), MeshSourceParameters, ) { getFragmentKey(objectKey: string | null, fragmentId: string) { @@ -230,7 +229,7 @@ class AppInfo { // .../1.0/... is the legacy link style // .../table/... is the current, version agnostic link style (for retrieving the info file) const linkStyle = - /^(https?:\/\/[.\w:\-/]+)\/segmentation\/(?:1\.0|table)\/([^/]+)\/?$/; + /^((?:middleauth\+)?https?:\/\/[.\w:\-/]+)\/segmentation\/(?:1\.0|table)\/([^/]+)\/?$/; const match = infoUrl.match(linkStyle); if (match === null) { throw Error(`Graph URL invalid: ${infoUrl}`); @@ -249,7 +248,7 @@ class AppInfo { // Dealing with a prehistoric graph server with no version information this.supported_api_versions = [0]; } - if (PYCG_APP_VERSION in this.supported_api_versions === false) { + if (this.supported_api_versions.includes(PYCG_APP_VERSION) === false) { const redirectMsg = `This Neuroglancer branch requires Graph Server version ${PYCG_APP_VERSION}, but the server only supports version(s) ${this.supported_api_versions}.`; throw new Error(redirectMsg); } @@ -284,12 +283,9 @@ interface GrapheneMultiscaleVolumeInfo extends MultiscaleVolumeInfo { function parseGrapheneMultiscaleVolumeInfo( obj: unknown, url: string, - credentialsManager: CredentialsManager, ): GrapheneMultiscaleVolumeInfo { const volumeInfo = parseMultiscaleVolumeInfo(obj); - const dataUrl = verifyObjectProperty(obj, "data_dir", (x) => - parseSpecialUrl(x, credentialsManager), - ).url; + const dataUrl = verifyObjectProperty(obj, "data_dir", verifyString); const app = verifyObjectProperty(obj, "app", (x) => new AppInfo(url, x)); const graph = verifyObjectProperty(obj, "graph", (x) => new GraphInfo(x)); return { @@ -302,11 +298,10 @@ function parseGrapheneMultiscaleVolumeInfo( class GrapheneMultiscaleVolumeChunkSource extends PrecomputedMultiscaleVolumeChunkSource { constructor( - chunkManager: ChunkManager, - public chunkedGraphCredentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, public info: GrapheneMultiscaleVolumeInfo, ) { - super(chunkManager, undefined, info.dataUrl, info); + super(sharedKvStoreContext, info.dataUrl, info); } getChunkedGraphSource() { @@ -341,7 +336,7 @@ class GrapheneMultiscaleVolumeChunkSource extends PrecomputedMultiscaleVolumeChu GrapheneChunkedGraphChunkSource, { spec, - credentialsProvider: this.chunkedGraphCredentialsProvider, + sharedKvStoreContext: this.sharedKvStoreContext, parameters: { url: `${this.info.app!.segmentationUrl}/node` }, }, ), @@ -430,13 +425,18 @@ function parseMeshMetadata(data: any): ParsedMeshMetadata { } async function getMeshMetadata( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, + options: Partial, ): Promise { let metadata: any; try { - metadata = await getJsonMetadata(chunkManager, credentialsProvider, url); + metadata = await getJsonMetadata( + sharedKvStoreContext, + url, + /*required=*/ false, + options, + ); } catch (e) { if (isNotFoundError(e)) { // If we fail to fetch the info file, assume it is the legacy @@ -512,27 +512,26 @@ function parseGrapheneShardingParameters( } function getShardedMeshSource( - chunkManager: ChunkManager, + sharedKvStoreContext: SharedKvStoreContext, parameters: MeshSourceParameters, - credentialsProvider: SpecialProtocolCredentialsProvider, ) { - return chunkManager.getChunkSource(GrapheneMeshSource, { + return sharedKvStoreContext.chunkManager.getChunkSource(GrapheneMeshSource, { + sharedKvStoreContext, parameters, - credentialsProvider, }); } async function getMeshSource( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, fragmentUrl: string, nBitsForLayerId: number, + options: ProgressOptions, ) { const { metadata, segmentPropertyMap } = await getMeshMetadata( - chunkManager, - undefined, + sharedKvStoreContext, fragmentUrl, + options, ); const parameters: MeshSourceParameters = { manifestUrl: url, @@ -543,27 +542,35 @@ async function getMeshSource( }; const transform = metadata?.transform || mat4.create(); return { - source: getShardedMeshSource(chunkManager, parameters, credentialsProvider), + source: getShardedMeshSource(sharedKvStoreContext, parameters), transform, segmentPropertyMap, }; } -function getJsonMetadata( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, +export function getJsonMetadata( + sharedKvStoreContext: SharedKvStoreContext, url: string, + required: boolean, + options: Partial, ): Promise { - return chunkManager.memoize.getUncounted( + return sharedKvStoreContext.chunkManager.memoize.getAsync( { - type: "graphene:metadata", + type: "precomputed:metadata", url, - credentialsProvider: getObjectId(credentialsProvider), }, - async () => { - return await fetchSpecialOk(credentialsProvider, `${url}/info`, {}).then( - (response) => response.json(), - ); + options, + async (options) => { + const infoUrl = pipelineUrlJoin(url, "info"); + using _span = new ProgressSpan(options.progressListener, { + message: `Reading graphene metadata from ${infoUrl}`, + }); + const response = await sharedKvStoreContext.kvStoreContext.read(infoUrl, { + ...options, + throwIfMissing: required, + }); + if (response === undefined) return undefined; + return await response.response.json(); }, ); } @@ -578,31 +585,22 @@ function getSubsourceToModelSubspaceTransform(info: MultiscaleVolumeInfo) { } async function getVolumeDataSource( - options: GetDataSourceOptions, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, metadata: any, + options: ProgressOptions, + stateJson: any, ): Promise { - const info = parseGrapheneMultiscaleVolumeInfo( - metadata, - url, - options.credentialsManager, - ); + const info = parseGrapheneMultiscaleVolumeInfo(metadata, url); const volume = new GrapheneMultiscaleVolumeChunkSource( - options.chunkManager, - credentialsProvider, + sharedKvStoreContext, info, ); const state = new GrapheneState(); - if (options.state) { - state.restoreState(options.state); + if (stateJson) { + state.restoreState(stateJson); } - const segmentationGraph = new GrapheneGraphSource( - info, - credentialsProvider, - volume, - state, - ); + const segmentationGraph = new GrapheneGraphSource(info, volume, state); const { modelSpace } = info; const subsources: DataSubsourceEntry[] = [ { @@ -626,18 +624,19 @@ async function getVolumeDataSource( }, ]; if (info.segmentPropertyMap !== undefined) { - const mapUrl = resolvePath(url, info.segmentPropertyMap); - const metadata = await getJsonMetadata( - options.chunkManager, - credentialsProvider, - mapUrl, + const mapUrl = kvstoreEnsureDirectoryPipelineUrl( + sharedKvStoreContext.kvStoreContext.resolveRelativePath( + url, + info.segmentPropertyMap, + ), ); - const segmentPropertyMap = getSegmentPropertyMap( - options.chunkManager, - credentialsProvider, - metadata, + const metadata = await getJsonMetadata( + sharedKvStoreContext, mapUrl, + /*required=*/ true, + options, ); + const segmentPropertyMap = getSegmentPropertyMap(metadata); subsources.push({ id: "properties", default: true, @@ -646,11 +645,16 @@ async function getVolumeDataSource( } if (info.mesh !== undefined) { const { source: meshSource, transform } = await getMeshSource( - options.chunkManager, - credentialsProvider, + sharedKvStoreContext, info.app!.meshingUrl, - resolvePath(info.dataUrl, info.mesh), + kvstoreEnsureDirectoryPipelineUrl( + sharedKvStoreContext.kvStoreContext.resolveRelativePath( + info.dataUrl, + info.mesh, + ), + ), info.graph.nBitsForLayerId, + options, ); const subsourceToModelSubspaceTransform = getSubsourceToModelSubspaceTransform(info); @@ -673,56 +677,56 @@ async function getVolumeDataSource( }; } -export class GrapheneDataSource extends PrecomputedDataSource { - get description() { - return "Graphene file-backed data source"; +// Note: Graphene is not really a kvstore-based data source, since it relies on +// making arbitrary HTTP requests rather than just kvstore. It fails if the +// provided kvstore does not inherit from HttpKvStore. +export class GrapheneDataSource implements KvStoreBasedDataSourceProvider { + get scheme() { + return "graphene"; } - - get(options: GetDataSourceOptions): Promise { - const { url: providerUrl, parameters } = parseProviderUrl( - options.providerUrl, - ); - return options.chunkManager.memoize.getUncounted( - { type: "graphene:get", providerUrl, parameters }, - async (): Promise => { - const { url, credentialsProvider } = parseSpecialUrl( - providerUrl, - options.credentialsManager, + get description() { + return "Graphene data source"; + } + + get( + options: GetKvStoreBasedDataSourceOptions, + ): Promise { + ensureEmptyUrlSuffix(options.url); + const url = kvstoreEnsureDirectoryPipelineUrl(options.kvStoreUrl); + return options.registry.chunkManager.memoize.getAsync( + { type: "graphene:get", url }, + options, + async (progressOptions) => { + const metadata = await getJsonMetadata( + options.registry.sharedKvStoreContext, + url, + /*required=*/ true, + progressOptions, ); - let metadata: any; - try { - metadata = await getJsonMetadata( - options.chunkManager, - credentialsProvider, - url, - ); - } catch (e) { - if (isNotFoundError(e)) { - if (parameters.type === "mesh") { - console.log("does this happen?"); - } - } - throw e; - } verifyObject(metadata); const redirect = verifyOptionalObjectProperty( metadata, "redirect", verifyString, ); + const canonicalUrl = `${options.url.scheme}://${url}`; if (redirect !== undefined) { - throw new RedirectError(redirect); + return { canonicalUrl, targetUrl: redirect }; } const t = verifyOptionalObjectProperty(metadata, "@type", verifyString); switch (t) { case "neuroglancer_multiscale_volume": - case undefined: - return await getVolumeDataSource( - options, - credentialsProvider, + case undefined: { + const dataSource = await getVolumeDataSource( + options.registry.sharedKvStoreContext, url, metadata, + progressOptions, + options.state, ); + dataSource.canonicalUrl = canonicalUrl; + return dataSource; + } default: throw new Error(`Invalid type: ${JSON.stringify(t)}`); } @@ -1536,26 +1540,13 @@ class GraphConnection extends SegmentationGraphSourceConnection { } } -async function parseGrapheneError(e: HttpError) { - if (e.response) { - let msg: string; - if (e.response.headers.get("content-type") === "application/json") { - msg = (await e.response.json()).message; - } else { - msg = await e.response.text(); - } - return msg; - } - return undefined; -} - -async function withErrorMessageHTTP( - promise: Promise, +async function withErrorMessageHTTP( + promise: Promise, options: { initialMessage?: string; errorPrefix: string; }, -): Promise { +): Promise { let status: StatusMessage | undefined = undefined; let dispose = () => {}; if (options.initialMessage) { @@ -1587,10 +1578,17 @@ async function withErrorMessageHTTP( export const GRAPH_SERVER_NOT_SPECIFIED = Symbol("Graph Server Not Specified."); class GrapheneGraphServerInterface { - constructor( - private url: string, - private credentialsProvider: SpecialProtocolCredentialsProvider, - ) {} + private fetchOkImpl: FetchOk; + private url: string; + constructor(sharedKvStoreContext: SharedKvStoreContext, url: string) { + const { store, path } = sharedKvStoreContext.kvStoreContext.getKvStore(url); + if (store instanceof HttpKvStore) { + this.fetchOkImpl = store.fetchOkImpl; + this.url = store.baseUrl + path; + } else { + throw new Error(`Non-HTTP protocol not supported: ${url}`); + } + } async getRoot(segment: Uint64, timestamp = "") { const timestampEpoch = new Date(timestamp).valueOf() / 1000; @@ -1599,13 +1597,13 @@ class GrapheneGraphServerInterface { Number.isNaN(timestampEpoch) ? "" : `×tamp=${timestampEpoch}` }`; - const promise = fetchSpecialOk(this.credentialsProvider, url, {}); - - const response = await withErrorMessageHTTP(promise, { - initialMessage: `Retrieving root for segment ${segment}`, - errorPrefix: "Could not fetch root: ", - }); - const jsonResp = await response.json(); + const jsonResp = await withErrorMessageHTTP( + this.fetchOkImpl(url).then((response) => response.json()), + { + initialMessage: `Retrieving root for segment ${segment}`, + errorPrefix: "Could not fetch root: ", + }, + ); return Uint64.parseString(jsonResp.root_id); } @@ -1619,23 +1617,19 @@ class GrapheneGraphServerInterface { return Promise.reject(GRAPH_SERVER_NOT_SPECIFIED); } - const promise = fetchSpecialOk( - this.credentialsProvider, - `${url}/merge?int64_as_str=1`, - { - method: "POST", - body: JSON.stringify([ - [ - String(first.segmentId), - ...first.position.map((val, i) => val * annotationToNanometers[i]), - ], - [ - String(second.segmentId), - ...second.position.map((val, i) => val * annotationToNanometers[i]), - ], - ]), - }, - ); + const promise = this.fetchOkImpl(`${url}/merge?int64_as_str=1`, { + method: "POST", + body: JSON.stringify([ + [ + String(first.segmentId), + ...first.position.map((val, i) => val * annotationToNanometers[i]), + ], + [ + String(second.segmentId), + ...second.position.map((val, i) => val * annotationToNanometers[i]), + ], + ]), + }); try { const response = await promise; @@ -1660,23 +1654,19 @@ class GrapheneGraphServerInterface { return Promise.reject(GRAPH_SERVER_NOT_SPECIFIED); } - const promise = fetchSpecialOk( - this.credentialsProvider, - `${url}/split?int64_as_str=1`, - { - method: "POST", - body: JSON.stringify({ - sources: first.map((x) => [ - String(x.segmentId), - ...x.position.map((val, i) => val * annotationToNanometers[i]), - ]), - sinks: second.map((x) => [ - String(x.segmentId), - ...x.position.map((val, i) => val * annotationToNanometers[i]), - ]), - }), - }, - ); + const promise = this.fetchOkImpl(`${url}/split?int64_as_str=1`, { + method: "POST", + body: JSON.stringify({ + sources: first.map((x) => [ + String(x.segmentId), + ...x.position.map((val, i) => val * annotationToNanometers[i]), + ]), + sinks: second.map((x) => [ + String(x.segmentId), + ...x.position.map((val, i) => val * annotationToNanometers[i]), + ]), + }), + }); const response = await withErrorMessageHTTP(promise, { initialMessage: `Splitting ${first.length} sources from ${second.length} sinks`, @@ -1693,17 +1683,19 @@ class GrapheneGraphServerInterface { async filterLatestRoots(segments: Uint64[]): Promise { const url = `${this.url}/is_latest_roots`; - const promise = fetchSpecialOk(this.credentialsProvider, url, { + const promise = this.fetchOkImpl(url, { method: "POST", body: JSON.stringify({ node_ids: segments.map((x) => x.toJSON()), }), }); - const response = await withErrorMessageHTTP(promise, { - errorPrefix: "Could not check latest: ", - }); - const jsonResp = await response.json(); + const jsonResp = await withErrorMessageHTTP( + promise.then((response) => response.json()), + { + errorPrefix: "Could not check latest: ", + }, + ); const res: Uint64[] = []; for (const [i, isLatest] of jsonResp.is_latest.entries()) { @@ -1721,14 +1713,13 @@ class GrapheneGraphSource extends SegmentationGraphSource { constructor( public info: GrapheneMultiscaleVolumeInfo, - credentialsProvider: SpecialProtocolCredentialsProvider, private chunkSource: GrapheneMultiscaleVolumeChunkSource, public state: GrapheneState, ) { super(); this.graphServer = new GrapheneGraphServerInterface( + chunkSource.sharedKvStoreContext, info.app!.segmentationUrl, - credentialsProvider, ); } @@ -1827,9 +1818,7 @@ class ChunkedGraphChunkSource } class GrapheneChunkedGraphChunkSource extends WithParameters( - WithCredentialsProvider()( - ChunkedGraphChunkSource, - ), + WithSharedKvStoreContext(ChunkedGraphChunkSource), ChunkedGraphSourceParameters, ) {} diff --git a/src/datasource/graphene/register_default.ts b/src/datasource/graphene/register_default.ts index e256a146fb..a0532a80b6 100644 --- a/src/datasource/graphene/register_default.ts +++ b/src/datasource/graphene/register_default.ts @@ -16,5 +16,8 @@ import { registerProvider } from "#src/datasource/default_provider.js"; import { GrapheneDataSource } from "#src/datasource/graphene/frontend.js"; +import { KvStoreBasedDataSourceLegacyUrlAdapter } from "#src/datasource/index.js"; -registerProvider("graphene", () => new GrapheneDataSource()); +registerProvider( + new KvStoreBasedDataSourceLegacyUrlAdapter(new GrapheneDataSource()), +); diff --git a/src/datasource/index.ts b/src/datasource/index.ts index 398f887f08..746dc22b36 100644 --- a/src/datasource/index.ts +++ b/src/datasource/index.ts @@ -22,12 +22,21 @@ import type { CoordinateSpaceTransform, CoordinateTransformSpecification, } from "#src/coordinate_transform.js"; +import type { SharedCredentialsManager } from "#src/credentials_provider/shared.js"; +import { getKvStoreCompletions } from "#src/datasource/kvstore_completions.js"; +import type { LocalDataSource } from "#src/datasource/local.js"; import { - emptyValidCoordinateSpace, - makeCoordinateSpace, - makeIdentityTransform, -} from "#src/coordinate_transform.js"; -import type { CredentialsManager } from "#src/credentials_provider/index.js"; + AutoDetectRegistry, + autoDetectFormat, +} from "#src/kvstore/auto_detect.js"; +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; +import type { UrlWithParsedScheme } from "#src/kvstore/url.js"; +import { + extractQueryAndFragment, + finalPipelineUrlComponent, + parsePipelineUrlComponent, + splitPipelineUrl, +} from "#src/kvstore/url.js"; import type { MeshSource, MultiscaleMeshSource } from "#src/mesh/frontend.js"; import type { SegmentPropertyMap } from "#src/segmentation_display_state/property_map.js"; import type { SegmentationGraphSource } from "#src/segmentation_graph/source.js"; @@ -43,19 +52,12 @@ import { applyCompletionOffset, getPrefixMatchesWithDescriptions, } from "#src/util/completion.js"; -import type { Owned } from "#src/util/disposable.js"; import { RefCounted } from "#src/util/disposable.js"; -import { createIdentity } from "#src/util/matrix.js"; +import { type ProgressOptions } from "#src/util/progress_listener.js"; import type { Trackable } from "#src/util/trackable.js"; export type CompletionResult = BasicCompletionResult; -export class RedirectError extends Error { - constructor(public redirectTarget: string) { - super(`Redirected to: ${redirectTarget}`); - } -} - /** * Returns the length of the prefix of path that corresponds to the "group", according to the * specified separator. @@ -93,9 +95,7 @@ export function suggestLayerNameBasedOnSeparator( return path.substring(groupIndex); } -export interface GetDataSourceOptionsBase { - chunkManager: ChunkManager; - abortSignal?: AbortSignal; +export interface GetDataSourceOptionsBase extends Partial { url: string; transform: CoordinateTransformSpecification | undefined; globalCoordinateSpace: WatchableValueInterface; @@ -103,11 +103,10 @@ export interface GetDataSourceOptionsBase { } export interface GetDataSourceOptions extends GetDataSourceOptionsBase { - registry: DataSourceProviderRegistry; + registry: DataSourceRegistry; providerUrl: string; - abortSignal: AbortSignal; - providerProtocol: string; - credentialsManager: CredentialsManager; + signal: AbortSignal; + providerScheme: string; } export interface ConvertLegacyUrlOptionsBase { @@ -116,24 +115,9 @@ export interface ConvertLegacyUrlOptionsBase { } export interface ConvertLegacyUrlOptions extends ConvertLegacyUrlOptionsBase { - registry: DataSourceProviderRegistry; - providerUrl: string; - providerProtocol: string; -} - -export interface NormalizeUrlOptionsBase { - url: string; -} - -export interface NormalizeUrlOptions extends NormalizeUrlOptionsBase { - registry: DataSourceProviderRegistry; + registry: DataSourceRegistry; providerUrl: string; - providerProtocol: string; -} - -export enum LocalDataSource { - annotations = 0, - equivalences = 1, + providerScheme: string; } export interface DataSubsource { @@ -147,17 +131,14 @@ export interface DataSubsource { segmentationGraph?: SegmentationGraphSource; } -export interface CompleteUrlOptionsBase { +export interface CompleteUrlOptionsBase extends Partial { url: string; - abortSignal?: AbortSignal; - chunkManager: ChunkManager; } export interface CompleteUrlOptions extends CompleteUrlOptionsBase { - registry: DataSourceProviderRegistry; + registry: DataSourceRegistry; providerUrl: string; - abortSignal: AbortSignal; - credentialsManager: CredentialsManager; + signal: AbortSignal; } export interface DataSubsourceEntry { @@ -196,20 +177,23 @@ export interface DataSource { modelTransform: CoordinateSpaceTransform; canChangeModelSpaceRank?: boolean; state?: Trackable; + canonicalUrl?: string; } -// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging -export interface DataSourceProvider { - /** - * Returns a suggested layer name for the given volume source. - */ - suggestLayerName?(path: string): string; +export interface DataSourceRedirect { + // Canonical URL of the source. + canonicalUrl?: string; - /** - * Returns the length of the prefix of path that is its 'group'. This is used for suggesting a - * default URL for adding a new layer. - */ - findSourceGroup?(path: string): number; + // Target URL. + targetUrl: string; +} + +export type DataSourceLookupResult = DataSource | DataSourceRedirect; + +export interface DataSourceWithRedirectInfo extends DataSource { + // Canonical URL prior to any redirects. + originalCanonicalUrl?: string; + redirectLog: Set; } export interface DataSubsourceSpecification { @@ -222,6 +206,9 @@ export interface DataSourceSpecification { enableDefaultSubsources: boolean; subsources: Map; state?: any; + + // Indicates that the spec was set manually in the UI. + setManually?: boolean; } export function makeEmptyDataSourceSpecification(): DataSourceSpecification { @@ -233,220 +220,233 @@ export function makeEmptyDataSourceSpecification(): DataSourceSpecification { }; } -// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging -export abstract class DataSourceProvider extends RefCounted { - abstract description?: string; +export interface DataSourceProvider { + scheme: string; + description?: string; + // Exclude from completion list. + hidden?: boolean; - abstract get(options: GetDataSourceOptions): Promise; + get(options: GetDataSourceOptions): Promise; - normalizeUrl(options: NormalizeUrlOptions): string { - return options.url; - } + convertLegacyUrl?: (options: ConvertLegacyUrlOptions) => string; - convertLegacyUrl(options: ConvertLegacyUrlOptions): string { - return options.url; - } + completeUrl?: (options: CompleteUrlOptions) => Promise; +} - async completeUrl(options: CompleteUrlOptions): Promise { - options; - throw null; - } +export interface KvStoreBasedDataSourceProvider { + scheme: string; + description?: string; + singleFile?: boolean; + expectsDirectory?: boolean; + get( + options: GetKvStoreBasedDataSourceOptions, + ): Promise; + completeUrl?: ( + options: GetKvStoreBasedDataSourceOptions, + ) => Promise; } -export const localAnnotationsUrl = "local://annotations"; -export const localEquivalencesUrl = "local://equivalences"; +export interface GetKvStoreBasedDataSourceOptions + extends Partial { + registry: DataSourceRegistry; + kvStoreUrl: string; + url: UrlWithParsedScheme; + state?: any; +} -class LocalDataSourceProvider extends DataSourceProvider { - get description() { - return "Local in-memory"; - } +const schemePattern = /^(?:([a-zA-Z][a-zA-Z0-9-+_]*):\/\/)?(.*)$/; - async get(options: GetDataSourceOptions): Promise { - switch (options.url) { - case localAnnotationsUrl: { - const { transform } = options; - let modelTransform: CoordinateSpaceTransform; - if (transform === undefined) { - const baseSpace = options.globalCoordinateSpace.value; - const { rank, names, scales, units } = baseSpace; - const inputSpace = makeCoordinateSpace({ - rank, - scales, - units, - names: names.map((_, i) => `${i}`), - }); - const outputSpace = makeCoordinateSpace({ - rank, - scales, - units, - names, - }); - modelTransform = { - rank, - sourceRank: rank, - inputSpace, - outputSpace, - transform: createIdentity(Float64Array, rank + 1), - }; - } else { - modelTransform = makeIdentityTransform(emptyValidCoordinateSpace); - } - return { - modelTransform, - canChangeModelSpaceRank: true, - subsources: [ - { - id: "default", - default: true, - subsource: { - local: LocalDataSource.annotations, - }, - }, - ], - }; - } - case localEquivalencesUrl: { - return { - modelTransform: makeIdentityTransform(emptyValidCoordinateSpace), - canChangeModelSpaceRank: false, - subsources: [ - { - id: "default", - default: true, - subsource: { - local: LocalDataSource.equivalences, - }, - }, - ], - }; - } - } - throw new Error("Invalid local data source URL"); +export class DataSourceRegistry extends RefCounted { + get credentialsManager(): SharedCredentialsManager { + return this.sharedKvStoreContext.credentialsManager; } - async completeUrl(options: CompleteUrlOptions) { - return { - offset: 0, - completions: getPrefixMatchesWithDescriptions( - options.providerUrl, - [ - { - value: "annotations", - description: "Annotations stored in the JSON state", - }, - { - value: "equivalences", - description: - "Segmentation equivalence graph stored in the JSON state", - }, - ], - (x) => x.value, - (x) => x.description, - ), - }; + get chunkManager(): ChunkManager { + return this.sharedKvStoreContext.chunkManager; } -} - -const protocolPattern = /^(?:([a-zA-Z][a-zA-Z0-9-+_]*):\/\/)?(.*)$/; -export class DataSourceProviderRegistry extends RefCounted { - constructor(public credentialsManager: CredentialsManager) { + constructor(public sharedKvStoreContext: SharedKvStoreContext) { super(); } - dataSources = new Map>([ - ["local", new LocalDataSourceProvider()], - ]); + dataSources = new Map(); + kvStoreBasedDataSources = new Map(); + autoDetectRegistry = new AutoDetectRegistry(); - register(name: string, dataSource: Owned) { - this.dataSources.set(name, this.registerDisposer(dataSource)); + register(provider: DataSourceProvider) { + this.dataSources.set(provider.scheme, provider); + } + registerKvStoreBasedProvider(provider: KvStoreBasedDataSourceProvider) { + this.kvStoreBasedDataSources.set(provider.scheme, provider); } getProvider(url: string): [DataSourceProvider, string, string] { - const m = url.match(protocolPattern); + const m = url.match(schemePattern); if (m === null || m[1] === undefined) { throw new Error( - `Data source URL must have the form "://".`, + `Data source URL must have the form "://".`, ); } - const [, providerProtocol, providerUrl] = m; - const factory = this.dataSources.get(providerProtocol); + const [, providerScheme, providerUrl] = m; + const factory = this.dataSources.get(providerScheme); if (factory === undefined) { throw new Error( - `Unsupported data source: ${JSON.stringify(providerProtocol)}.`, + `Unsupported data source: ${JSON.stringify(providerScheme)}.`, ); } - return [factory, providerUrl, providerProtocol]; + return [factory, providerUrl, providerScheme]; } - async get(options: GetDataSourceOptionsBase): Promise { - const redirectLog = new Set(); - const { abortSignal } = options; - let url: string = options.url; + private async autoDetectFormat(options: GetDataSourceOptionsBase) { + const { matches, url } = await autoDetectFormat({ + url: options.url, + kvStoreContext: this.sharedKvStoreContext.kvStoreContext, + signal: options.signal, + progressListener: options.progressListener, + autoDetectDirectory: () => this.autoDetectRegistry.directorySpec, + autoDetectFile: () => this.autoDetectRegistry.fileSpec, + }); + if (matches.length !== 1) { + let message: string; + if (matches.length === 0) { + message = "no format detected"; + } else { + message = `multiple formats detected: ${JSON.stringify(matches)}`; + } + throw new Error( + `Failed to auto-detect data source for ${JSON.stringify(options.url)}: ${message}`, + ); + } + return `${url}|${matches[0].suffix}`; + } + + private async resolveKvStoreBasedDataSource( + options: GetDataSourceOptionsBase, + ): Promise { while (true) { - const [provider, providerUrl, providerProtocol] = this.getProvider( - options.url, + const finalPart = parsePipelineUrlComponent( + finalPipelineUrlComponent(options.url), ); - redirectLog.add(options.url); - try { - return provider.get({ + const dataSourceProvider = this.kvStoreBasedDataSources.get( + finalPart.scheme, + ); + if (dataSourceProvider === undefined) { + // Attempt to auto-detect format. + const newUrl = await this.autoDetectFormat(options); + options = { ...options, url: newUrl }; + continue; + } + return await dataSourceProvider.get({ + registry: this, + url: finalPart, + kvStoreUrl: options.url.substring( + 0, + options.url.length - finalPart.url.length - 1, + ), + signal: options.signal, + progressListener: options.progressListener, + state: options.state, + }); + } + } + + private async resolvePipeline( + options: GetDataSourceOptionsBase, + ): Promise { + const pipelineParts = splitPipelineUrl(options.url); + + const basePart = pipelineParts[0]; + const baseScheme = basePart.scheme; + + // Check kvstore providers + { + const provider = + this.sharedKvStoreContext.kvStoreContext.baseKvStoreProviders.get( + baseScheme, + ); + if (provider !== undefined) { + return await this.resolveKvStoreBasedDataSource(options); + } + } + + if (pipelineParts.length !== 1) { + throw new Error(`${baseScheme}: scheme does not support | URL pipelines`); + } + + const suffix = basePart.suffix ?? ""; + if (!suffix.startsWith("//")) { + throw new Error(`${baseScheme}: URLs must start with "${baseScheme}://"`); + } + + const providerUrl = suffix.substring(2); + + // Check non-kvstore-based providers + { + const provider = this.dataSources.get(baseScheme); + if (provider !== undefined) { + return await provider.get({ ...options, - url, - providerProtocol, + url: pipelineParts[0].url, + providerScheme: baseScheme, providerUrl, registry: this, - abortSignal: abortSignal ?? new AbortController().signal, - credentialsManager: this.credentialsManager, + signal: options.signal ?? new AbortController().signal, }); - } catch (e) { - if (e instanceof RedirectError) { - const redirect = e.redirectTarget; - if (redirectLog.has(redirect)) { - throw Error( - `Layer source redirection contains loop: ${JSON.stringify( - Array.from(redirectLog), - )}`, - ); - } - if (redirectLog.size >= 10) { - throw Error( - `Too many layer source redirections: ${JSON.stringify( - Array.from(redirectLog), - )}`, - ); - } - url = redirect; - continue; - } - throw e; } } + + throw new Error(`Unsupported scheme: ${baseScheme}:`); } - convertLegacyUrl(options: ConvertLegacyUrlOptionsBase): string { - try { - const [provider, providerUrl, providerProtocol] = this.getProvider( - options.url, - ); - return provider.convertLegacyUrl({ - ...options, - providerUrl, - providerProtocol, - registry: this, - }); - } catch { - return options.url; + async get( + options: GetDataSourceOptionsBase, + ): Promise { + const redirectLog = new Set(); + let url: string = options.url; + // Trim any trailing "|" characters. + url = url.replace(/\|+$/, ""); + let originalCanonicalUrl: string | undefined; + while (true) { + redirectLog.add(url); + const dataSource = await this.resolvePipeline({ ...options, url }); + if (originalCanonicalUrl === undefined) { + originalCanonicalUrl = dataSource.canonicalUrl; + } + if ("targetUrl" in dataSource) { + const { targetUrl } = dataSource; + if (redirectLog.has(targetUrl)) { + throw Error( + `Layer source redirection contains loop: ${JSON.stringify([ + ...redirectLog, + targetUrl, + ])}`, + ); + } + if (redirectLog.size >= 10) { + throw Error( + `Too many layer source redirections: ${JSON.stringify([ + ...redirectLog, + targetUrl, + ])}`, + ); + } + url = targetUrl; + continue; + } + return { ...dataSource, redirectLog, originalCanonicalUrl }; } } - normalizeUrl(options: NormalizeUrlOptionsBase): string { + // Converts legacy precomputed mesh and skeleton datasource URLs. + convertLegacyUrl(options: ConvertLegacyUrlOptionsBase): string { try { - const [provider, providerUrl, providerProtocol] = this.getProvider( + const [provider, providerUrl, providerScheme] = this.getProvider( options.url, ); - return provider.normalizeUrl({ + if (provider.convertLegacyUrl === undefined) return options.url; + return provider.convertLegacyUrl({ ...options, providerUrl, - providerProtocol, + providerScheme: providerScheme, registry: this, }); } catch { @@ -457,52 +457,209 @@ export class DataSourceProviderRegistry extends RefCounted { async completeUrl( options: CompleteUrlOptionsBase, ): Promise { - // Check if url matches a protocol. Note that protocolPattern always matches. - const { url, abortSignal } = options; - const protocolMatch = url.match(protocolPattern)!; - const protocol = protocolMatch[1]; - if (protocol === undefined) { - return Promise.resolve({ - offset: 0, + // Check if url matches a scheme. Note that schemePattern always matches. + const { signal } = options; + + const { url } = options; + + const finalComponent = finalPipelineUrlComponent(url); + const parsedFinalComponent = parsePipelineUrlComponent(finalComponent); + const { scheme } = parsedFinalComponent; + + // Check if we need to complete a scheme. + if ( + finalComponent === url && + !(parsedFinalComponent.suffix ?? "").startsWith("//") + ) { + const providers: { + scheme: string; + description?: string; + }[] = []; + const add = ( + iterable: Iterable, + predicate?: (provider: Provider) => boolean, + ) => { + for (const provider of iterable) { + if (predicate?.(provider) === false) continue; + providers.push(provider); + } + }; + + if (finalComponent === url) { + add( + this.sharedKvStoreContext.kvStoreContext.baseKvStoreProviders.values(), + ); + add(this.dataSources.values(), (provider) => provider.hidden !== true); + } else { + add(this.kvStoreBasedDataSources.values()); + add( + this.sharedKvStoreContext.kvStoreContext.kvStoreAdapterProviders.values(), + ); + } + + const schemeSuffix = finalComponent === url ? "//" : ""; + return { + offset: url.length - finalComponent.length, completions: getPrefixMatchesWithDescriptions( - url, - this.dataSources, - ([name]) => `${name}://`, - ([, factory]) => factory.description, + scheme, + providers, + ({ scheme }) => `${scheme}:${schemeSuffix}`, + ({ description }) => description, ), - }); + }; } - const factory = this.dataSources.get(protocol); - if (factory !== undefined) { - const completions = await factory.completeUrl({ - registry: this, - url, - providerUrl: protocolMatch[2], - chunkManager: options.chunkManager, - abortSignal: abortSignal ?? new AbortController().signal, - credentialsManager: this.credentialsManager, + + if (parsedFinalComponent.suffix === undefined) { + const prevPipelineUrl = options.url.substring( + 0, + options.url.length - finalComponent.length - 1, + ); + const { matches } = await autoDetectFormat({ + url: prevPipelineUrl, + kvStoreContext: this.sharedKvStoreContext.kvStoreContext, + signal: options.signal, + progressListener: options.progressListener, + autoDetectDirectory: () => this.autoDetectRegistry.directorySpec, + autoDetectFile: () => this.autoDetectRegistry.fileSpec, }); - return applyCompletionOffset(protocol.length + 3, completions); + if (matches.length === 0) { + throw new Error( + `Failed to auto-detect data source for ${JSON.stringify(prevPipelineUrl)}`, + ); + } + return { + offset: url.length - finalComponent.length, + completions: getPrefixMatchesWithDescriptions( + parsedFinalComponent.scheme, + matches, + ({ suffix }) => suffix, + ({ description }) => description, + ), + }; + } + + if (finalComponent === url) { + // Single component pipeline. + const provider = this.dataSources.get(scheme); + if (provider !== undefined) { + // Non-kvstore-based protocol. + if (provider.completeUrl === undefined) throw null; + const completions = await provider.completeUrl({ + registry: this, + url: options.url, + providerUrl: parsedFinalComponent.suffix!.substring(2), + signal: signal ?? new AbortController().signal, + progressListener: options.progressListener, + }); + return applyCompletionOffset(scheme.length + 3, completions); + } + } + + { + const provider = this.kvStoreBasedDataSources.get(scheme); + if (provider !== undefined) { + if (provider.completeUrl === undefined) throw null; + const completions = await provider.completeUrl({ + registry: this, + signal: signal ?? new AbortController().signal, + progressListener: options.progressListener, + kvStoreUrl: url.substring(0, url.length - finalComponent.length - 1), + url: parsedFinalComponent, + }); + return applyCompletionOffset( + url.length - + finalComponent.length + + parsedFinalComponent.scheme.length + + 1, + completions, + ); + } } - throw null; + + return await getKvStoreCompletions(this.sharedKvStoreContext, { + url, + signal, + progressListener: options.progressListener, + autoDetectDirectory: () => this.autoDetectRegistry.directorySpec, + }); } - suggestLayerName(url: string) { - let [dataSource, path] = this.getProvider(url); - if (path.endsWith("/")) { - path = path.substring(0, path.length - 1); + suggestLayerName(url: string): string | undefined { + const parts = splitPipelineUrl(url); + for (let i = parts.length - 1; i >= 0; --i) { + let { suffix } = parts[i]; + if (!suffix) continue; + suffix = suffix.replace(/^\/+/, "").replace(/\/+$/, ""); + if (!suffix) continue; + return suggestLayerNameBasedOnSeparator(suffix); } - const suggestor = dataSource.suggestLayerName; - if (suggestor !== undefined) { - return suggestor(path); + return undefined; + } +} + +export class KvStoreBasedDataSourceLegacyUrlAdapter + implements DataSourceProvider +{ + constructor( + public base: KvStoreBasedDataSourceProvider, + public scheme = base.scheme, + ) {} + + get hidden() { + return true; + } + + get description() { + return this.base.description; + } + + private parseProviderUrl(url: string) { + if (url.includes("|")) { + throw new Error("Only a single pipeline component supported"); } - return suggestLayerNameBasedOnSeparator(path); + const { base, queryAndFragment } = extractQueryAndFragment(url); + return { + kvStoreUrl: base, + queryAndFragment, + url: parsePipelineUrlComponent(`${this.base.scheme}:${queryAndFragment}`), + }; + } + + get(options: GetDataSourceOptions): Promise { + const { kvStoreUrl, url } = this.parseProviderUrl(options.providerUrl); + return this.base.get({ + registry: options.registry, + url, + kvStoreUrl, + state: options.state, + signal: options.signal, + progressListener: options.progressListener, + }); } - findSourceGroup(url: string) { - const [dataSource, path, dataSourceName] = this.getProvider(url); - const helper = - dataSource.findSourceGroup || findSourceGroupBasedOnSeparator; - return helper(path) + dataSourceName.length + 3; + async completeUrl(options: CompleteUrlOptions): Promise { + const { kvStoreUrl, url, queryAndFragment } = this.parseProviderUrl( + options.providerUrl, + ); + if (queryAndFragment === "") { + return await getKvStoreCompletions( + options.registry.sharedKvStoreContext, + { + url: options.providerUrl, + signal: options.signal, + progressListener: options.progressListener, + singlePipelineComponent: true, + directoryOnly: this.base.expectsDirectory, + }, + ); + } + if (!this.base.completeUrl) throw null; + return this.base.completeUrl({ + registry: options.registry, + signal: options.signal, + progressListener: options.progressListener, + kvStoreUrl, + url, + }); } } diff --git a/src/datasource/kvstore_completions.ts b/src/datasource/kvstore_completions.ts new file mode 100644 index 0000000000..05a8f80ab2 --- /dev/null +++ b/src/datasource/kvstore_completions.ts @@ -0,0 +1,141 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { CompletionResult } from "#src/datasource/index.js"; +import type { AutoDetectDirectorySpec } from "#src/kvstore/auto_detect.js"; +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; +import { listKvStore } from "#src/kvstore/index.js"; +import { + encodePathForUrl, + finalPipelineUrlComponent, +} from "#src/kvstore/url.js"; +import type { CompletionWithDescription } from "#src/util/completion.js"; +import type { ProgressListener } from "#src/util/progress_listener.js"; + +export async function getKvStoreCompletions( + sharedKvStoreContext: SharedKvStoreContext, + options: { + url: string; + signal?: AbortSignal; + progressListener?: ProgressListener; + directoryOnly?: boolean; + autoDetectDirectory?: () => AutoDetectDirectorySpec; + singlePipelineComponent?: boolean; + }, +): Promise { + const { url, autoDetectDirectory } = options; + const kvStore = sharedKvStoreContext.kvStoreContext.getKvStore(url); + if (kvStore.store.getUrl(kvStore.path) !== url) { + // URL is valid but lacks final "/" terminator, skip completion. + // + // This avoids attempting to access e.g. `gs://bucke` as the user is typing + // `gs://bucket/`. + throw null; + } + const results = await listKvStore(kvStore.store, kvStore.path, { + signal: options.signal, + progressListener: options.progressListener, + responseKeys: "url", + }); + + const finalComponent = finalPipelineUrlComponent(url); + + // Infallible pattern + const [, directoryPath, namePrefix] = finalComponent.match( + /^((?:[a-zA-Z][a-zA-Z0-9-+.]*:)(?:.*\/)?)([^/]*)$/, + )!; + const offset = url.length - namePrefix.length; + const matches: CompletionWithDescription[] = []; + const directoryOffset = + url.length - finalComponent.length + directoryPath.length; + for (const entry of results.directories) { + matches.push({ value: entry.substring(directoryOffset) + "/" }); + } + if (!options.directoryOnly) { + const matchSuffix = options.singlePipelineComponent === true ? "" : "|"; + for (const entry of results.entries) { + matches.push({ + value: entry.key.substring(directoryOffset) + matchSuffix, + }); + } + } + + let defaultCompletion: string | undefined; + + if (autoDetectDirectory !== undefined && namePrefix === "") { + const names = new Set(); + for (const entry of results.entries) { + names.add(entry.key.substring(directoryOffset)); + } + const pipelineMatches = await autoDetectDirectory().match({ + url, + fileNames: names, + signal: options.signal, + }); + for (const match of pipelineMatches) { + matches.push({ + value: `|${match.suffix}`, + description: match.description, + }); + } + if (pipelineMatches.length === 1) { + defaultCompletion = `|${pipelineMatches[0].suffix}`; + } + } + return { offset, completions: matches, defaultCompletion }; +} + +export async function getKvStorePathCompletions( + sharedKvStoreContext: SharedKvStoreContext, + options: { + baseUrl: string; + path: string; + signal?: AbortSignal; + progressListener?: ProgressListener; + directoryOnly?: boolean; + }, +): Promise { + const { baseUrl, path } = options; + const { store, path: basePath } = + sharedKvStoreContext.kvStoreContext.getKvStore(baseUrl); + if (!store.list) { + throw new Error("Listing not supported"); + } + + const fullPath = basePath + path; + + const fullOffset = Math.max(basePath.length, fullPath.lastIndexOf("/") + 1); + + const results = await store.list(fullPath, { + signal: options.signal, + progressListener: options.progressListener, + }); + + const matches: CompletionWithDescription[] = []; + for (const entry of results.directories) { + matches.push({ + value: encodePathForUrl(entry.substring(fullOffset) + "/"), + }); + } + if (!options.directoryOnly) { + for (const entry of results.entries) { + matches.push({ + value: encodePathForUrl(entry.key.substring(fullOffset)), + }); + } + } + return { offset: fullOffset - basePath.length, completions: matches }; +} diff --git a/src/datasource/local.ts b/src/datasource/local.ts new file mode 100644 index 0000000000..4df6dc0443 --- /dev/null +++ b/src/datasource/local.ts @@ -0,0 +1,132 @@ +/** + * @license + * Copyright 2016 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + emptyValidCoordinateSpace, + makeCoordinateSpace, + makeIdentityTransform, + type CoordinateSpaceTransform, +} from "#src/coordinate_transform.js"; +import type { + CompleteUrlOptions, + DataSource, + GetDataSourceOptions, + DataSourceProvider, +} from "#src/datasource/index.js"; +import { getPrefixMatchesWithDescriptions } from "#src/util/completion.js"; +import { createIdentity } from "#src/util/matrix.js"; + +export const localAnnotationsUrl = "local://annotations"; +export const localEquivalencesUrl = "local://equivalences"; + +export enum LocalDataSource { + annotations = 0, + equivalences = 1, +} + +export class LocalDataSourceProvider implements DataSourceProvider { + get scheme() { + return "local"; + } + get description() { + return "Local in-memory"; + } + + async get(options: GetDataSourceOptions): Promise { + switch (options.url) { + case localAnnotationsUrl: { + const { transform } = options; + let modelTransform: CoordinateSpaceTransform; + if (transform === undefined) { + const baseSpace = options.globalCoordinateSpace.value; + const { rank, names, scales, units } = baseSpace; + const inputSpace = makeCoordinateSpace({ + rank, + scales, + units, + names: names.map((_, i) => `${i}`), + }); + const outputSpace = makeCoordinateSpace({ + rank, + scales, + units, + names, + }); + modelTransform = { + rank, + sourceRank: rank, + inputSpace, + outputSpace, + transform: createIdentity(Float64Array, rank + 1), + }; + } else { + modelTransform = makeIdentityTransform(emptyValidCoordinateSpace); + } + return { + modelTransform, + canChangeModelSpaceRank: true, + subsources: [ + { + id: "default", + default: true, + subsource: { + local: LocalDataSource.annotations, + }, + }, + ], + }; + } + case localEquivalencesUrl: { + return { + modelTransform: makeIdentityTransform(emptyValidCoordinateSpace), + canChangeModelSpaceRank: false, + subsources: [ + { + id: "default", + default: true, + subsource: { + local: LocalDataSource.equivalences, + }, + }, + ], + }; + } + } + throw new Error("Invalid local data source URL"); + } + + async completeUrl(options: CompleteUrlOptions) { + return { + offset: 0, + completions: getPrefixMatchesWithDescriptions( + options.providerUrl, + [ + { + value: "annotations", + description: "Annotations stored in the JSON state", + }, + { + value: "equivalences", + description: + "Segmentation equivalence graph stored in the JSON state", + }, + ], + (x) => x.value, + (x) => x.description, + ), + }; + } +} diff --git a/src/datasource/n5/backend.ts b/src/datasource/n5/backend.ts index 0a20bd745d..6e1a7fcb9d 100644 --- a/src/datasource/n5/backend.ts +++ b/src/datasource/n5/backend.ts @@ -18,24 +18,21 @@ import { decodeBlosc } from "#src/async_computation/decode_blosc_request.js"; import { decodeZstd } from "#src/async_computation/decode_zstd_request.js"; import { requestAsyncComputation } from "#src/async_computation/request.js"; import { WithParameters } from "#src/chunk_manager/backend.js"; -import { WithSharedCredentialsProviderCounterpart } from "#src/credentials_provider/shared_counterpart.js"; import { VolumeChunkEncoding, VolumeChunkSourceParameters, } from "#src/datasource/n5/base.js"; +import { WithSharedKvStoreContextCounterpart } from "#src/kvstore/backend.js"; import { decodeRawChunk } from "#src/sliceview/backend_chunk_decoders/raw.js"; import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; import { VolumeChunkSource } from "#src/sliceview/volume/backend.js"; import { Endianness } from "#src/util/endian.js"; import { decodeGzip } from "#src/util/gzip.js"; -import { isNotFoundError } from "#src/util/http_request.js"; -import type { SpecialProtocolCredentials } from "#src/util/special_protocol_request.js"; -import { fetchSpecialOk } from "#src/util/special_protocol_request.js"; import { registerSharedObject } from "#src/worker_rpc.js"; async function decodeChunk( chunk: VolumeChunk, - abortSignal: AbortSignal, + signal: AbortSignal, response: ArrayBuffer, encoding: VolumeChunkEncoding, ) { @@ -66,7 +63,7 @@ async function decodeChunk( case VolumeChunkEncoding.BLOSC: buffer = await requestAsyncComputation( decodeBlosc, - abortSignal, + signal, [buffer.buffer], buffer, ); @@ -74,7 +71,7 @@ async function decodeChunk( case VolumeChunkEncoding.ZSTD: buffer = await requestAsyncComputation( decodeZstd, - abortSignal, + signal, [buffer.buffer], buffer, ); @@ -82,7 +79,7 @@ async function decodeChunk( } await decodeRawChunk( chunk, - abortSignal, + signal, buffer.buffer, Endianness.BIG, buffer.byteOffset, @@ -92,31 +89,32 @@ async function decodeChunk( @registerSharedObject() export class PrecomputedVolumeChunkSource extends WithParameters( - WithSharedCredentialsProviderCounterpart()( - VolumeChunkSource, - ), + WithSharedKvStoreContextCounterpart(VolumeChunkSource), VolumeChunkSourceParameters, ) { - async download(chunk: VolumeChunk, abortSignal: AbortSignal) { - const { parameters } = this; + private chunkKvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore( + this.parameters.url, + ); + async download(chunk: VolumeChunk, signal: AbortSignal) { + const { parameters, chunkKvStore } = this; const { chunkGridPosition } = chunk; - let url = parameters.url; + let path = chunkKvStore.path; const rank = this.spec.rank; for (let i = 0; i < rank; ++i) { - url += `/${chunkGridPosition[i]}`; - } - try { - const response = await fetchSpecialOk(this.credentialsProvider, url, { - signal: abortSignal, - }); - await decodeChunk( - chunk, - abortSignal, - await response.arrayBuffer(), - parameters.encoding, - ); - } catch (e) { - if (!isNotFoundError(e)) throw e; + if (i !== 0) { + path += "/"; + } + path += `${chunkGridPosition[i]}`; } + const response = await chunkKvStore.store.read(path, { + signal, + }); + if (response === undefined) return; + await decodeChunk( + chunk, + signal, + await response.response.arrayBuffer(), + parameters.encoding, + ); } } diff --git a/src/datasource/n5/frontend.ts b/src/datasource/n5/frontend.ts index e76695b2ad..3c9f5856c4 100644 --- a/src/datasource/n5/frontend.ts +++ b/src/datasource/n5/frontend.ts @@ -21,10 +21,11 @@ * * https://github.com/saalfeldlab/n5-viewer * https://github.com/bigdataviewer/bigdataviewer-core/blob/master/BDV%20N5%20format.md + * + * https://github.com/janelia-cellmap/schemas/blob/master/multiscale.md */ import { makeDataBoundsBoundingBoxAnnotationSet } from "#src/annotation/index.js"; -import type { ChunkManager } from "#src/chunk_manager/frontend.js"; import { WithParameters } from "#src/chunk_manager/frontend.js"; import type { CoordinateArray, @@ -34,17 +35,28 @@ import { makeCoordinateSpace, makeIdentityTransform, } from "#src/coordinate_transform.js"; -import { WithCredentialsProvider } from "#src/credentials_provider/chunk_source_frontend.js"; import type { - CompleteUrlOptions, DataSource, - GetDataSourceOptions, + GetKvStoreBasedDataSourceOptions, + KvStoreBasedDataSourceProvider, } from "#src/datasource/index.js"; -import { DataSourceProvider } from "#src/datasource/index.js"; +import { getKvStorePathCompletions } from "#src/datasource/kvstore_completions.js"; import { VolumeChunkEncoding, VolumeChunkSourceParameters, } from "#src/datasource/n5/base.js"; +import type { AutoDetectRegistry } from "#src/kvstore/auto_detect.js"; +import { simpleFilePresenceAutoDetectDirectorySpec } from "#src/kvstore/auto_detect.js"; +import { WithSharedKvStoreContext } from "#src/kvstore/chunk_source_frontend.js"; +import type { CompletionResult, KvStoreContext } from "#src/kvstore/context.js"; +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; +import { + ensureNoQueryOrFragmentParameters, + ensurePathIsDirectory, + joinPath, + kvstoreEnsureDirectoryPipelineUrl, + pipelineUrlJoin, +} from "#src/kvstore/url.js"; import type { SliceViewSingleResolutionSource } from "#src/sliceview/frontend.js"; import type { VolumeSourceOptions } from "#src/sliceview/volume/base.js"; import { @@ -57,9 +69,6 @@ import { VolumeChunkSource, } from "#src/sliceview/volume/frontend.js"; import { transposeNestedArrays } from "#src/util/array.js"; -import type { Borrowed } from "#src/util/disposable.js"; -import { completeHttpPath } from "#src/util/http_path_completion.js"; -import { isNotFoundError, parseUrl } from "#src/util/http_request.js"; import { expectArray, parseArray, @@ -75,19 +84,12 @@ import { verifyStringArray, } from "#src/util/json.js"; import { createHomogeneousScaleMatrix } from "#src/util/matrix.js"; -import { getObjectId } from "#src/util/object_id.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; import { scaleByExp10, unitFromJson } from "#src/util/si_units.js"; -import type { - SpecialProtocolCredentials, - SpecialProtocolCredentialsProvider, -} from "#src/util/special_protocol_request.js"; -import { - fetchSpecialOk, - parseSpecialUrl, -} from "#src/util/special_protocol_request.js"; class N5VolumeChunkSource extends WithParameters( - WithCredentialsProvider()(VolumeChunkSource), + WithSharedKvStoreContext(VolumeChunkSource), VolumeChunkSourceParameters, ) {} @@ -103,12 +105,11 @@ export class MultiscaleVolumeChunkSource extends GenericMultiscaleVolumeChunkSou } constructor( - chunkManager: Borrowed, - public credentialsProvider: SpecialProtocolCredentialsProvider, + public sharedKvStoreContext: SharedKvStoreContext, public multiscaleMetadata: MultiscaleMetadata, public scales: (ScaleMetadata | undefined)[], ) { - super(chunkManager); + super(sharedKvStoreContext.chunkManager); let dataType: DataType | undefined; let baseScaleIndex: number | undefined; scales.forEach((scale, i) => { @@ -128,7 +129,7 @@ export class MultiscaleVolumeChunkSource extends GenericMultiscaleVolumeChunkSou if (dataType === undefined) { throw new Error("At least one scale must be specified."); } - const baseDownsamplingInfo = multiscaleMetadata.scales[baseScaleIndex!]!; + const baseDownsamplingInfo = scales[baseScaleIndex!]!; const baseScale = scales[baseScaleIndex!]!; this.dataType = dataType; this.volumeType = VolumeType.IMAGE; @@ -143,7 +144,7 @@ export class MultiscaleVolumeChunkSource extends GenericMultiscaleVolumeChunkSou { transform: createHomogeneousScaleMatrix( Float64Array, - baseDownsamplingInfo.downsamplingFactor, + baseDownsamplingInfo.downsamplingFactors, /*square=*/ false, ), box: { @@ -158,14 +159,12 @@ export class MultiscaleVolumeChunkSource extends GenericMultiscaleVolumeChunkSou getSources(volumeSourceOptions: VolumeSourceOptions) { const { scales, rank } = this; - const scalesDownsamplingInfo = this.multiscaleMetadata.scales; return transposeNestedArrays( (scales.filter((scale) => scale !== undefined) as ScaleMetadata[]).map( - (scale, i) => { - const scaleDownsamplingInfo = scalesDownsamplingInfo[i]; + (scale) => { const transform = createHomogeneousScaleMatrix( Float32Array, - scaleDownsamplingInfo.downsamplingFactor, + scale.downsamplingFactors, ); return makeDefaultVolumeChunkSpecifications({ rank, @@ -180,10 +179,10 @@ export class MultiscaleVolumeChunkSource extends GenericMultiscaleVolumeChunkSou chunkSource: this.chunkManager.getChunkSource( N5VolumeChunkSource, { - credentialsProvider: this.credentialsProvider, + sharedKvStoreContext: this.sharedKvStoreContext, spec, parameters: { - url: scaleDownsamplingInfo.url, + url: scale.url, encoding: scale.encoding, }, }, @@ -201,146 +200,209 @@ interface MultiscaleMetadata { url: string; attributes: any; modelSpace: CoordinateSpace; - scales: { readonly url: string; readonly downsamplingFactor: Float64Array }[]; + scales: ( + | { + readonly url: string; + readonly downsamplingFactors?: Float64Array; + } + | undefined + )[]; } -class ScaleMetadata { +interface ScaleMetadata { + url: string; dataType: DataType; encoding: VolumeChunkEncoding; - size: Float32Array; - chunkSize: Uint32Array; + size: Float32Array; + chunkSize: Uint32Array; + downsamplingFactors: Float64Array; +} - constructor(obj: any) { - verifyObject(obj); - this.dataType = verifyObjectProperty(obj, "dataType", (x) => - verifyEnumString(x, DataType), - ); - this.size = Float32Array.from( - verifyObjectProperty(obj, "dimensions", (x) => - parseArray(x, verifyPositiveInt), - ), +function parseScaleMetadata( + url: string, + obj: any, + scaleIndex: number, + downsamplingFactors?: Float64Array, +): ScaleMetadata { + verifyObject(obj); + const dataType = verifyObjectProperty(obj, "dataType", (x) => + verifyEnumString(x, DataType), + ); + const size = Float32Array.from( + verifyObjectProperty(obj, "dimensions", (x) => + parseArray(x, verifyPositiveInt), + ), + ); + const chunkSize = verifyObjectProperty(obj, "blockSize", (x) => + parseFixedLengthArray(new Uint32Array(size.length), x, verifyPositiveInt), + ); + + let encoding: VolumeChunkEncoding | undefined; + verifyOptionalObjectProperty(obj, "compression", (compression) => { + encoding = verifyObjectProperty(compression, "type", (x) => + verifyEnumString(x, VolumeChunkEncoding), ); - this.chunkSize = verifyObjectProperty(obj, "blockSize", (x) => - parseFixedLengthArray( - new Uint32Array(this.size.length), - x, - verifyPositiveInt, - ), + if ( + encoding === VolumeChunkEncoding.GZIP && + verifyOptionalObjectProperty( + compression, + "useZlib", + verifyBoolean, + false, + ) === true + ) { + encoding = VolumeChunkEncoding.ZLIB; + } + }); + if (encoding === undefined) { + encoding = verifyObjectProperty(obj, "compressionType", (x) => + verifyEnumString(x, VolumeChunkEncoding), ); + } - let encoding: VolumeChunkEncoding | undefined; - verifyOptionalObjectProperty(obj, "compression", (compression) => { - encoding = verifyObjectProperty(compression, "type", (x) => - verifyEnumString(x, VolumeChunkEncoding), - ); - if ( - encoding === VolumeChunkEncoding.GZIP && - verifyOptionalObjectProperty( - compression, - "useZlib", - verifyBoolean, - false, - ) === true - ) { - encoding = VolumeChunkEncoding.ZLIB; + if (downsamplingFactors === undefined) { + downsamplingFactors = verifyOptionalObjectProperty( + obj, + "downsamplingFactors", + (x) => + parseFixedLengthArray( + new Float64Array(size.length), + x, + verifyFinitePositiveFloat, + ), + ); + if (downsamplingFactors === undefined) { + if (scaleIndex === 0) { + downsamplingFactors = new Float64Array(size.length); + downsamplingFactors.fill(1); + } else { + throw new Error("Expected downsamplingFactors attribute"); } - }); - if (encoding === undefined) { - encoding = verifyObjectProperty(obj, "compressionType", (x) => - verifyEnumString(x, VolumeChunkEncoding), - ); } - this.encoding = encoding; } + + return { url, dataType, encoding, size, chunkSize, downsamplingFactors }; } function getAllScales( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, multiscaleMetadata: MultiscaleMetadata, + options: Partial, ): Promise<(ScaleMetadata | undefined)[]> { return Promise.all( - multiscaleMetadata.scales.map(async (scale) => { - const attributes = await getAttributes( - chunkManager, - credentialsProvider, + multiscaleMetadata.scales.map(async (scale, scaleIndex) => { + if (scale === undefined) return undefined; + const { attributes } = (await getAttributes( + sharedKvStoreContext, scale.url, true, - ); + options, + ))!; if (attributes === undefined) return undefined; - return new ScaleMetadata(attributes); + try { + return parseScaleMetadata( + scale.url, + attributes, + scaleIndex, + scale.downsamplingFactors, + ); + } catch (e) { + throw new Error(`Error parsing array metadata at ${scale.url}`, { + cause: e, + }); + } }), ); } -function getAttributesJsonUrls(url: string): string[] { - let { protocol, host, path } = parseUrl(url); - if (path.endsWith("/")) { - path = path.substring(0, path.length - 1); - } - const urls: string[] = []; +function getAttributesJsonUrls( + kvStoreContext: KvStoreContext, + url: string, +): { attributesJsonUrl: string; directoryUrl: string; relativePath: string }[] { + const kvStore = kvStoreContext.getKvStore(url); + const urls: { + attributesJsonUrl: string; + directoryUrl: string; + relativePath: string; + }[] = []; + let path = kvStore.path.substring(0, kvStore.path.length - 1); while (true) { - urls.push(`${protocol}://${host}${path}/attributes.json`); + const directoryPath = ensurePathIsDirectory(path); + urls.push({ + attributesJsonUrl: kvStore.store.getUrl( + joinPath(path, "attributes.json"), + ), + directoryUrl: kvStore.store.getUrl(directoryPath), + relativePath: kvStore.path.substring(directoryPath.length), + }); + if (path === "") break; const index = path.lastIndexOf("/"); - if (index === -1) break; path = path.substring(0, index); } return urls; } function getIndividualAttributesJson( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, required: boolean, + options: Partial, ): Promise { - return chunkManager.memoize.getUncounted( + return sharedKvStoreContext.chunkManager.memoize.getAsync( { type: "n5:attributes.json", url, - credentialsProvider: getObjectId(credentialsProvider), }, - () => - fetchSpecialOk(credentialsProvider, url, {}) - .then((response) => response.json()) - .then((j) => { - try { - return verifyObject(j); - } catch (e) { - throw new Error( - `Error reading attributes from ${url}: ${e.message}`, - ); - } - }) - .catch((e) => { - if (isNotFoundError(e)) { - if (required) return undefined; - return {}; - } - throw e; - }), + options, + async (progressOptions) => { + using _span = new ProgressSpan(progressOptions.progressListener, { + message: `Reading n5 metadata from ${url}`, + }); + const response = await sharedKvStoreContext.kvStoreContext.read(url, { + ...progressOptions, + throwIfMissing: required, + }); + if (response === undefined) return undefined; + const json = await response.response.json(); + try { + return verifyObject(json); + } catch (e) { + throw new Error(`Error reading attributes from ${url}`, { cause: e }); + } + }, ); } async function getAttributes( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, required: boolean, -): Promise { - const attributesJsonUrls = getAttributesJsonUrls(url); + options: Partial, +): Promise< + { attributes: unknown; rootUrl: string; pathFromRoot: string } | undefined +> { + const attributesJsonUrls = getAttributesJsonUrls( + sharedKvStoreContext.kvStoreContext, + url, + ); const metadata = await Promise.all( attributesJsonUrls.map((u, i) => getIndividualAttributesJson( - chunkManager, - credentialsProvider, - u, + sharedKvStoreContext, + u.attributesJsonUrl, required && i === attributesJsonUrls.length - 1, + options, ), ), ); - if (metadata.indexOf(undefined) !== -1) return undefined; + const rootIndex = metadata.findLastIndex((x) => x !== undefined); + if (rootIndex === -1) return undefined; metadata.reverse(); - return Object.assign({}, ...metadata); + const rootInfo = attributesJsonUrls[rootIndex]; + return { + attributes: Object.assign({}, ...metadata.filter((x) => x !== undefined)), + rootUrl: rootInfo.directoryUrl, + pathFromRoot: rootInfo.relativePath, + }; } function verifyRank(existing: number, n: number) { @@ -386,10 +448,12 @@ function getDefaultAxes(rank: number) { return axes; } -function getMultiscaleMetadata( +async function getMultiscaleMetadata( + sharedKvStoreContext: SharedKvStoreContext, url: string, attributes: any, -): MultiscaleMetadata { + progressOptions: ProgressOptions, +): Promise { verifyObject(attributes); let rank = -1; @@ -409,8 +473,8 @@ function getMultiscaleMetadata( return units; }); let defaultUnit = { unit: "m", exponent: -9 }; - let singleDownsamplingFactors: Float64Array | undefined; - let allDownsamplingFactors: Float64Array[] | undefined; + let singleDownsamplingFactors: Float64Array | undefined; + let allDownsamplingFactors: Float64Array[] | undefined; verifyOptionalObjectProperty(attributes, "downsamplingFactors", (dObj) => { const { single, all, rank: curRank } = parseDownsamplingFactors(dObj); rank = verifyRank(rank, curRank); @@ -499,17 +563,33 @@ function getMultiscaleMetadata( }); if (dimensions === undefined) { if (allDownsamplingFactors === undefined) { - throw new Error( - "Not valid single-resolution or multi-resolution dataset", + const scaleDirectories = await findScaleDirectories( + sharedKvStoreContext, + url, + progressOptions, ); + if (scaleDirectories.length === 0) { + throw new Error( + "Not valid single-resolution or multi-resolution dataset", + ); + } + return { + modelSpace, + url, + attributes, + scales: scaleDirectories.map((name) => ({ + url: `${url}${name}/`, + downsamplingFactors: undefined, + })), + }; } return { modelSpace, url, attributes, scales: allDownsamplingFactors.map((f, i) => ({ - url: `${url}/s${i}`, - downsamplingFactor: f, + url: `${url}s${i}/`, + downsamplingFactors: f, })), }; } @@ -521,45 +601,80 @@ function getMultiscaleMetadata( modelSpace, url, attributes, - scales: [{ url, downsamplingFactor: singleDownsamplingFactors }], + scales: [{ url, downsamplingFactors: singleDownsamplingFactors }], }; } -export class N5DataSource extends DataSourceProvider { +async function findScaleDirectories( + sharedKvStoreContext: SharedKvStoreContext, + url: string, + progressOptions: ProgressOptions, +): Promise { + const result = await sharedKvStoreContext.kvStoreContext.list(url, { + responseKeys: "suffix", + ...progressOptions, + }); + const scaleDirectories: string[] = []; + for (const directory of result.directories) { + if (directory.match(/^s(?:0|[1-9][0-9]*)$/)) { + const scale = Number(directory.substring(1)); + scaleDirectories[scale] = directory; + } + } + return scaleDirectories; +} + +export class N5DataSource implements KvStoreBasedDataSourceProvider { + get scheme() { + return "n5"; + } + get expectsDirectory() { + return true; + } get description() { return "N5 data source"; } - get(options: GetDataSourceOptions): Promise { - let { providerUrl } = options; - if (providerUrl.endsWith("/")) { - providerUrl = providerUrl.substring(0, providerUrl.length - 1); - } - return options.chunkManager.memoize.getUncounted( - { type: "n5:MultiscaleVolumeChunkSource", providerUrl }, - async () => { - const { url, credentialsProvider } = parseSpecialUrl( - providerUrl, - options.credentialsManager, - ); - const attributes = await getAttributes( - options.chunkManager, - credentialsProvider, + get(options: GetKvStoreBasedDataSourceOptions): Promise { + ensureNoQueryOrFragmentParameters(options.url); + const url = kvstoreEnsureDirectoryPipelineUrl( + pipelineUrlJoin( + kvstoreEnsureDirectoryPipelineUrl(options.kvStoreUrl), + options.url.suffix ?? "", + ), + ); + const { sharedKvStoreContext } = options.registry; + return options.registry.chunkManager.memoize.getAsync( + { type: "n5:MultiscaleVolumeChunkSource", url }, + options, + async (progressOptions) => { + const attributeResult = await getAttributes( + sharedKvStoreContext, url, false, + progressOptions, + ); + if (attributeResult === undefined) { + throw new Error("N5 metadata not found"); + } + const { attributes, rootUrl, pathFromRoot } = attributeResult; + const multiscaleMetadata = await getMultiscaleMetadata( + sharedKvStoreContext, + url, + attributes, + progressOptions, ); - const multiscaleMetadata = getMultiscaleMetadata(url, attributes); const scales = await getAllScales( - options.chunkManager, - credentialsProvider, + sharedKvStoreContext, multiscaleMetadata, + progressOptions, ); const volume = new MultiscaleVolumeChunkSource( - options.chunkManager, - credentialsProvider, + sharedKvStoreContext, multiscaleMetadata, scales, ); return { + canonicalUrl: `${rootUrl}|${options.url.scheme}:${pathFromRoot}`, modelTransform: makeIdentityTransform(volume.modelSpace), subsources: [ { @@ -584,11 +699,25 @@ export class N5DataSource extends DataSourceProvider { ); } - completeUrl(options: CompleteUrlOptions) { - return completeHttpPath( - options.credentialsManager, - options.providerUrl, - options.abortSignal, - ); + async completeUrl( + options: GetKvStoreBasedDataSourceOptions, + ): Promise { + ensureNoQueryOrFragmentParameters(options.url); + return getKvStorePathCompletions(options.registry.sharedKvStoreContext, { + baseUrl: kvstoreEnsureDirectoryPipelineUrl(options.kvStoreUrl), + path: options.url.suffix ?? "", + directoryOnly: true, + signal: options.signal, + progressListener: options.progressListener, + }); } } + +export function registerAutoDetect(registry: AutoDetectRegistry) { + registry.registerDirectoryFormat( + simpleFilePresenceAutoDetectDirectorySpec(new Set(["attributes.json"]), { + suffix: "n5:", + description: "N5", + }), + ); +} diff --git a/src/datasource/n5/register_default.ts b/src/datasource/n5/register_default.ts index dddddbb211..7732860e74 100644 --- a/src/datasource/n5/register_default.ts +++ b/src/datasource/n5/register_default.ts @@ -14,7 +14,18 @@ * limitations under the License. */ -import { registerProvider } from "#src/datasource/default_provider.js"; -import { N5DataSource } from "#src/datasource/n5/frontend.js"; +import { + registerKvStoreBasedDataProvider, + dataSourceAutoDetectRegistry, + registerProvider, +} from "#src/datasource/default_provider.js"; +import { KvStoreBasedDataSourceLegacyUrlAdapter } from "#src/datasource/index.js"; +import { + N5DataSource, + registerAutoDetect, +} from "#src/datasource/n5/frontend.js"; -registerProvider("n5", () => new N5DataSource()); +const provider = new N5DataSource(); +registerKvStoreBasedDataProvider(provider); +registerProvider(new KvStoreBasedDataSourceLegacyUrlAdapter(provider)); +registerAutoDetect(dataSourceAutoDetectRegistry); diff --git a/src/datasource/nggraph/frontend.ts b/src/datasource/nggraph/frontend.ts index bd3e53c461..8578c9cf6d 100644 --- a/src/datasource/nggraph/frontend.ts +++ b/src/datasource/nggraph/frontend.ts @@ -16,7 +16,7 @@ import { debounce } from "lodash-es"; import type { ChunkManager } from "#src/chunk_manager/frontend.js"; -import { fetchWithCredentials } from "#src/credentials_provider/http_request.js"; +import { fetchOkWithCredentials } from "#src/credentials_provider/http_request.js"; import { CredentialsProvider, makeCredentialsGetter, @@ -27,8 +27,8 @@ import type { DataSource, DataSubsourceEntry, GetDataSourceOptions, + DataSourceProvider, } from "#src/datasource/index.js"; -import { DataSourceProvider } from "#src/datasource/index.js"; import type { Credentials } from "#src/datasource/nggraph/credentials_provider.js"; import { NggraphCredentialsProvider } from "#src/datasource/nggraph/credentials_provider.js"; import type { SegmentationUserLayer } from "#src/layer/segmentation/index.js"; @@ -47,6 +47,7 @@ import { StatusMessage } from "#src/status.js"; import type { Uint64Set } from "#src/uint64_set.js"; import { getPrefixMatchesWithDescriptions } from "#src/util/completion.js"; import { DisjointUint64Sets } from "#src/util/disjoint_sets.js"; +import type { RequestInitWithProgress } from "#src/util/http_request.js"; import { parseArray, verifyFiniteFloat, @@ -609,7 +610,7 @@ function fetchWithNggraphCredentials( path: string, init: RequestInit, ): Promise { - return fetchWithCredentials( + return fetchOkWithCredentials( credentialsProvider, `${serverUrl}${path}`, init, @@ -701,7 +702,7 @@ function nggraphGraphFetch( serverUrl: string, entityName: string, path: string, - init: RequestInit, + init: RequestInitWithProgress, ): Promise { return fetchWithNggraphCredentials( getEntityCredentialsProvider(chunkManager, serverUrl, entityName), @@ -732,41 +733,46 @@ function parseListResponse(response: any) { ); } -export class NggraphDataSource extends DataSourceProvider { +export class NggraphDataSource implements DataSourceProvider { + get scheme() { + return "nggraph"; + } get description() { return "nggraph data source"; } get(options: GetDataSourceOptions): Promise { const { serverUrl, id } = parseNggraphUrl(options.providerUrl); - return options.chunkManager.memoize.getUncounted( + return options.registry.chunkManager.memoize.getAsync( { type: "nggraph:get", serverUrl, id }, - async (): Promise => { + options, + async (progressOptions): Promise => { const entityCredentialsProvider = getEntityCredentialsProvider( - options.chunkManager, + options.registry.chunkManager, serverUrl, id, ); - const { entityType } = (await entityCredentialsProvider.get()) - .credentials; + const { entityType } = ( + await entityCredentialsProvider.get(undefined, progressOptions) + ).credentials; if (entityType !== "graph") { throw new Error( `Unsupported entity type: ${JSON.stringify(entityType)}`, ); } const { datasource_url: baseSegmentation } = await nggraphGraphFetch( - options.chunkManager, + options.registry.chunkManager, serverUrl, id, "/graph/config", - { method: "POST" }, + { method: "POST", ...progressOptions }, ); const baseSegmentationDataSource = await options.registry.get({ ...options, url: baseSegmentation, }); const segmentationGraph = new NggraphSegmentationGraphSource( - options.chunkManager, + options.registry.chunkManager, serverUrl, id, ); @@ -789,13 +795,20 @@ export class NggraphDataSource extends DataSourceProvider { async completeUrl(options: CompleteUrlOptions): Promise { const { serverUrl, id } = parseNggraphUrl(options.providerUrl); - const list = await options.chunkManager.memoize.getUncounted( + const list = await options.registry.chunkManager.memoize.getAsync( { type: "nggraph:list", serverUrl }, - async () => { + options, + async (progressOptions) => { return parseListResponse( - await nggraphServerFetch(options.chunkManager, serverUrl, "/list", { - method: "POST", - }), + await nggraphServerFetch( + options.registry.chunkManager, + serverUrl, + "/list", + { + method: "POST", + ...progressOptions, + }, + ), ); }, ); diff --git a/src/datasource/nggraph/register_default.ts b/src/datasource/nggraph/register_default.ts index 82c0754698..99e3c0dede 100644 --- a/src/datasource/nggraph/register_default.ts +++ b/src/datasource/nggraph/register_default.ts @@ -17,4 +17,4 @@ import { registerProvider } from "#src/datasource/default_provider.js"; import { NggraphDataSource } from "#src/datasource/nggraph/frontend.js"; -registerProvider("nggraph", () => new NggraphDataSource()); +registerProvider(new NggraphDataSource()); diff --git a/src/datasource/nifti/backend.ts b/src/datasource/nifti/backend.ts index 1235cd6981..34c62465e7 100644 --- a/src/datasource/nifti/backend.ts +++ b/src/datasource/nifti/backend.ts @@ -16,23 +16,20 @@ import type { NIFTI2 } from "nifti-reader-js"; import { isCompressed, NIFTI1, readHeader, readImage } from "nifti-reader-js"; -import type { ChunkManager } from "#src/chunk_manager/backend.js"; import { WithParameters } from "#src/chunk_manager/backend.js"; -import { ChunkPriorityTier } from "#src/chunk_manager/base.js"; -import type { PriorityGetter } from "#src/chunk_manager/generic_file_source.js"; -import { GenericSharedDataSource } from "#src/chunk_manager/generic_file_source.js"; -import type { SharedCredentialsProviderCounterpart } from "#src/credentials_provider/shared_counterpart.js"; -import { WithSharedCredentialsProviderCounterpart } from "#src/credentials_provider/shared_counterpart.js"; +import { getCachedDecodedUrl } from "#src/chunk_manager/generic_file_source.js"; import type { NiftiVolumeInfo } from "#src/datasource/nifti/base.js"; import { GET_NIFTI_VOLUME_INFO_RPC_ID, VolumeSourceParameters, } from "#src/datasource/nifti/base.js"; +import type { SharedKvStoreContextCounterpart } from "#src/kvstore/backend.js"; +import { WithSharedKvStoreContextCounterpart } from "#src/kvstore/backend.js"; +import type { ReadResponse } from "#src/kvstore/index.js"; import { decodeRawChunk } from "#src/sliceview/backend_chunk_decoders/raw.js"; import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; import { VolumeChunkSource } from "#src/sliceview/volume/backend.js"; import { DataType } from "#src/sliceview/volume/base.js"; -import type { Borrowed } from "#src/util/disposable.js"; import { Endianness } from "#src/util/endian.js"; import { kOneVec, @@ -43,10 +40,7 @@ import { } from "#src/util/geom.js"; import { decodeGzip } from "#src/util/gzip.js"; import * as matrix from "#src/util/matrix.js"; -import type { - SpecialProtocolCredentials, - SpecialProtocolCredentialsProvider, -} from "#src/util/special_protocol_request.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; import type { RPCPromise } from "#src/worker_rpc.js"; import { registerPromiseRPC, registerSharedObject } from "#src/worker_rpc.js"; @@ -56,11 +50,15 @@ export class NiftiFileData { } async function decodeNiftiFile( - buffer: ArrayBuffer, - _cancellationToken: AbortSignal, + readResponse: ReadResponse | undefined, + options: ProgressOptions, ) { + if (readResponse === undefined) { + throw new Error("Not found"); + } + let buffer = await readResponse.response.arrayBuffer(); if (isCompressed(buffer)) { - buffer = await decodeGzip(buffer, "gzip"); + buffer = await decodeGzip(buffer, "gzip", options.signal); } const data = new NiftiFileData(); data.uncompressedData = buffer; @@ -73,40 +71,24 @@ async function decodeNiftiFile( } function getNiftiFileData( - chunkManager: Borrowed, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContextCounterpart: SharedKvStoreContextCounterpart, url: string, - getPriority: PriorityGetter, - abortSignal: AbortSignal, + options: Partial, ) { - return GenericSharedDataSource.getUrl( - chunkManager, - credentialsProvider, - decodeNiftiFile, + return getCachedDecodedUrl( + sharedKvStoreContextCounterpart, url, - getPriority, - abortSignal, + decodeNiftiFile, + options, ); } -const NIFTI_HEADER_INFO_PRIORITY = 1000; - async function getNiftiHeaderInfo( - chunkManager: Borrowed, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContextCounterpart, url: string, - abortSignal: AbortSignal, + options: Partial, ) { - const data = await getNiftiFileData( - chunkManager, - credentialsProvider, - url, - () => ({ - priorityTier: ChunkPriorityTier.VISIBLE, - priority: NIFTI_HEADER_INFO_PRIORITY, - }), - abortSignal, - ); + const data = await getNiftiFileData(sharedKvStoreContext, url, options); return data.header; } @@ -165,174 +147,161 @@ const DATA_TYPE_CONVERSIONS = new Map([ registerPromiseRPC( GET_NIFTI_VOLUME_INFO_RPC_ID, - async function (x, abortSignal): RPCPromise { - const chunkManager = this.getRef(x.chunkManager); - const credentialsProvider = this.getOptionalRef< - SharedCredentialsProviderCounterpart< - Exclude - > - >(x.credentialsProvider); - try { - const header = await getNiftiHeaderInfo( - chunkManager, - credentialsProvider, - x.url, - abortSignal, + async function (x, progressOptions): RPCPromise { + const sharedKvStoreContext = this.get( + x.sharedKvStoreContext, + ) as SharedKvStoreContextCounterpart; + const header = await getNiftiHeaderInfo( + sharedKvStoreContext, + x.url, + progressOptions, + ); + const dataTypeInfo = DATA_TYPE_CONVERSIONS.get(header.datatypeCode); + if (dataTypeInfo === undefined) { + throw new Error( + "Unsupported data type: " + + `${NiftiDataType[header.datatypeCode] || header.datatypeCode}.`, ); - const dataTypeInfo = DATA_TYPE_CONVERSIONS.get(header.datatypeCode); - if (dataTypeInfo === undefined) { - throw new Error( - "Unsupported data type: " + - `${NiftiDataType[header.datatypeCode] || header.datatypeCode}.`, - ); - } - let spatialInvScale = 1; - let spatialUnit = ""; - switch (header.xyzt_units & NIFTI1.SPATIAL_UNITS_MASK) { - case NIFTI1.UNITS_METER: - spatialInvScale = 1; - spatialUnit = "m"; - break; - case NIFTI1.UNITS_MM: - spatialInvScale = 1e3; - spatialUnit = "m"; - break; - case NIFTI1.UNITS_MICRON: - spatialInvScale = 1e6; - spatialUnit = "m"; - break; - } + } + let spatialInvScale = 1; + let spatialUnit = ""; + switch (header.xyzt_units & NIFTI1.SPATIAL_UNITS_MASK) { + case NIFTI1.UNITS_METER: + spatialInvScale = 1; + spatialUnit = "m"; + break; + case NIFTI1.UNITS_MM: + spatialInvScale = 1e3; + spatialUnit = "m"; + break; + case NIFTI1.UNITS_MICRON: + spatialInvScale = 1e6; + spatialUnit = "m"; + break; + } - let timeUnit = ""; - let timeInvScale = 1; - switch (header.xyzt_units & NIFTI1.TEMPORAL_UNITS_MASK) { - case NIFTI1.UNITS_SEC: - timeUnit = "s"; - timeInvScale = 1; - break; - case NIFTI1.UNITS_MSEC: - timeUnit = "s"; - timeInvScale = 1e3; - break; - case NIFTI1.UNITS_USEC: - timeUnit = "s"; - timeInvScale = 1e6; - break; - case NIFTI1.UNITS_HZ: - timeUnit = "Hz"; - timeInvScale = 1; - break; - case NIFTI1.UNITS_RADS: - timeUnit = "rad/s"; - timeInvScale = 1; - break; - } - let units: string[] = [ - spatialUnit, - spatialUnit, - spatialUnit, - timeUnit, - "", - "", - "", - ]; - let sourceScales = Float64Array.of( - header.pixDims[1] / spatialInvScale, - header.pixDims[2] / spatialInvScale, - header.pixDims[3] / spatialInvScale, - header.pixDims[4] / timeInvScale, - header.pixDims[5], - header.pixDims[6], - header.pixDims[7], - ); - let viewScales = Float64Array.of( - 1 / spatialInvScale, - 1 / spatialInvScale, - 1 / spatialInvScale, - 1 / timeInvScale, - 1, - 1, - 1, - ); - let sourceNames = ["i", "j", "k", "m", "c^", "c1^", "c2^"]; - let viewNames = ["x", "y", "z", "t", "c^", "c1^", "c2^"]; - const rank = header.dims[0]; - sourceNames = sourceNames.slice(0, rank); - viewNames = viewNames.slice(0, rank); - units = units.slice(0, rank); - sourceScales = sourceScales.slice(0, rank); - viewScales = viewScales.slice(0, rank); - const { quatern_b, quatern_c, quatern_d } = header; - const quatern_a = Math.sqrt( - 1.0 - - quatern_b * quatern_b - - quatern_c * quatern_c - - quatern_d * quatern_d, - ); - const qfac = header.pixDims[0] === -1 ? -1 : 1; - const qoffset = vec3.fromValues( - header.qoffset_x, - header.qoffset_y, - header.qoffset_z, - ); - // https://nifti.nimh.nih.gov/nifti-1/documentation/nifti1fields/nifti1fields_pages/qsform.html - const method3Transform = convertAffine(header.affine); - method3Transform; - const method2Transform = translationRotationScaleZReflectionToMat4( - mat4.create(), - qoffset, - quat.fromValues(quatern_b, quatern_c, quatern_d, quatern_a), - kOneVec, - qfac, - ); - const transform = matrix.createIdentity(Float64Array, rank + 1); - const copyRank = Math.min(3, rank); - for (let row = 0; row < copyRank; ++row) { - for (let col = 0; col < copyRank; ++col) { - transform[col * (rank + 1) + row] = method2Transform[col * 4 + row]; - } - transform[rank * (rank + 1) + row] = method2Transform[12 + row]; + let timeUnit = ""; + let timeInvScale = 1; + switch (header.xyzt_units & NIFTI1.TEMPORAL_UNITS_MASK) { + case NIFTI1.UNITS_SEC: + timeUnit = "s"; + timeInvScale = 1; + break; + case NIFTI1.UNITS_MSEC: + timeUnit = "s"; + timeInvScale = 1e3; + break; + case NIFTI1.UNITS_USEC: + timeUnit = "s"; + timeInvScale = 1e6; + break; + case NIFTI1.UNITS_HZ: + timeUnit = "Hz"; + timeInvScale = 1; + break; + case NIFTI1.UNITS_RADS: + timeUnit = "rad/s"; + timeInvScale = 1; + break; + } + let units: string[] = [ + spatialUnit, + spatialUnit, + spatialUnit, + timeUnit, + "", + "", + "", + ]; + let sourceScales = Float64Array.of( + header.pixDims[1] / spatialInvScale, + header.pixDims[2] / spatialInvScale, + header.pixDims[3] / spatialInvScale, + header.pixDims[4] / timeInvScale, + header.pixDims[5], + header.pixDims[6], + header.pixDims[7], + ); + let viewScales = Float64Array.of( + 1 / spatialInvScale, + 1 / spatialInvScale, + 1 / spatialInvScale, + 1 / timeInvScale, + 1, + 1, + 1, + ); + let sourceNames = ["i", "j", "k", "m", "c^", "c1^", "c2^"]; + let viewNames = ["x", "y", "z", "t", "c^", "c1^", "c2^"]; + const rank = header.dims[0]; + sourceNames = sourceNames.slice(0, rank); + viewNames = viewNames.slice(0, rank); + units = units.slice(0, rank); + sourceScales = sourceScales.slice(0, rank); + viewScales = viewScales.slice(0, rank); + const { quatern_b, quatern_c, quatern_d } = header; + const quatern_a = Math.sqrt( + 1.0 - + quatern_b * quatern_b - + quatern_c * quatern_c - + quatern_d * quatern_d, + ); + const qfac = header.pixDims[0] === -1 ? -1 : 1; + const qoffset = vec3.fromValues( + header.qoffset_x, + header.qoffset_y, + header.qoffset_z, + ); + // https://nifti.nimh.nih.gov/nifti-1/documentation/nifti1fields/nifti1fields_pages/qsform.html + const method3Transform = convertAffine(header.affine); + method3Transform; + const method2Transform = translationRotationScaleZReflectionToMat4( + mat4.create(), + qoffset, + quat.fromValues(quatern_b, quatern_c, quatern_d, quatern_a), + kOneVec, + qfac, + ); + const transform = matrix.createIdentity(Float64Array, rank + 1); + const copyRank = Math.min(3, rank); + for (let row = 0; row < copyRank; ++row) { + for (let col = 0; col < copyRank; ++col) { + transform[col * (rank + 1) + row] = method2Transform[col * 4 + row]; } - const info: NiftiVolumeInfo = { - rank, - sourceNames, - viewNames, - units, - sourceScales, - viewScales, - description: header.description, - transform, - dataType: dataTypeInfo.dataType, - volumeSize: Uint32Array.from(header.dims.slice(1, 1 + rank)), - }; - return { value: info }; - } finally { - chunkManager.dispose(); - credentialsProvider?.dispose(); + transform[rank * (rank + 1) + row] = method2Transform[12 + row]; } + const info: NiftiVolumeInfo = { + rank, + sourceNames, + viewNames, + units, + sourceScales, + viewScales, + description: header.description, + transform, + dataType: dataTypeInfo.dataType, + volumeSize: Uint32Array.from(header.dims.slice(1, 1 + rank)), + }; + return { value: info }; }, ); @registerSharedObject() export class NiftiVolumeChunkSource extends WithParameters( - WithSharedCredentialsProviderCounterpart()( - VolumeChunkSource, - ), + WithSharedKvStoreContextCounterpart(VolumeChunkSource), VolumeSourceParameters, ) { - async download(chunk: VolumeChunk, abortSignal: AbortSignal) { + async download(chunk: VolumeChunk, signal: AbortSignal) { chunk.chunkDataSize = this.spec.chunkDataSize; const data = await getNiftiFileData( - this.chunkManager, - this.credentialsProvider, + this.sharedKvStoreContext, this.parameters.url, - () => ({ priorityTier: chunk.priorityTier, priority: chunk.priority }), - abortSignal, + { signal }, ); const imageBuffer = readImage(data.header, data.uncompressedData); await decodeRawChunk( chunk, - abortSignal, + signal, imageBuffer, data.header.littleEndian ? Endianness.LITTLE : Endianness.BIG, ); diff --git a/src/datasource/nifti/frontend.ts b/src/datasource/nifti/frontend.ts index 4243fc20f5..35bdceeeb3 100644 --- a/src/datasource/nifti/frontend.ts +++ b/src/datasource/nifti/frontend.ts @@ -20,28 +20,29 @@ */ import { makeDataBoundsBoundingBoxAnnotationSet } from "#src/annotation/index.js"; -import type { ChunkManager } from "#src/chunk_manager/frontend.js"; import { WithParameters } from "#src/chunk_manager/frontend.js"; import { makeCoordinateSpace, makeIdentityTransformedBoundingBox, } from "#src/coordinate_transform.js"; import { - getCredentialsProviderCounterpart, - WithCredentialsProvider, -} from "#src/credentials_provider/chunk_source_frontend.js"; -import type { CredentialsManager } from "#src/credentials_provider/index.js"; -import type { - CompleteUrlOptions, - DataSource, - GetDataSourceOptions, + type DataSource, + type GetKvStoreBasedDataSourceOptions, + type KvStoreBasedDataSourceProvider, } from "#src/datasource/index.js"; -import { DataSourceProvider } from "#src/datasource/index.js"; import type { NiftiVolumeInfo } from "#src/datasource/nifti/base.js"; import { GET_NIFTI_VOLUME_INFO_RPC_ID, VolumeSourceParameters, } from "#src/datasource/nifti/base.js"; +import type { + AutoDetectFileOptions, + AutoDetectFileSpec, + AutoDetectRegistry, +} from "#src/kvstore/auto_detect.js"; +import { WithSharedKvStoreContext } from "#src/kvstore/chunk_source_frontend.js"; +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; +import { ensureEmptyUrlSuffix } from "#src/kvstore/url.js"; import type { VolumeSourceOptions } from "#src/sliceview/volume/base.js"; import { makeVolumeChunkSpecificationWithDefaultCompression, @@ -51,27 +52,22 @@ import { MultiscaleVolumeChunkSource, VolumeChunkSource, } from "#src/sliceview/volume/frontend.js"; -import { completeHttpPath } from "#src/util/http_path_completion.js"; +import { Endianness } from "#src/util/endian.js"; import * as matrix from "#src/util/matrix.js"; -import type { - SpecialProtocolCredentials, - SpecialProtocolCredentialsProvider, -} from "#src/util/special_protocol_request.js"; -import { parseSpecialUrl } from "#src/util/special_protocol_request.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; class NiftiVolumeChunkSource extends WithParameters( - WithCredentialsProvider()(VolumeChunkSource), + WithSharedKvStoreContext(VolumeChunkSource), VolumeSourceParameters, ) {} export class NiftiMultiscaleVolumeChunkSource extends MultiscaleVolumeChunkSource { constructor( - chunkManager: ChunkManager, - public credentialsProvider: SpecialProtocolCredentialsProvider, + public sharedKvStoreContext: SharedKvStoreContext, public url: string, public info: NiftiVolumeInfo, ) { - super(chunkManager); + super(sharedKvStoreContext.chunkManager); } get dataType() { return this.info.dataType; @@ -103,7 +99,7 @@ export class NiftiMultiscaleVolumeChunkSource extends MultiscaleVolumeChunkSourc chunkSource: this.chunkManager.getChunkSource( NiftiVolumeChunkSource, { - credentialsProvider: this.credentialsProvider, + sharedKvStoreContext: this.sharedKvStoreContext, spec, parameters: { url: this.url }, }, @@ -116,47 +112,37 @@ export class NiftiMultiscaleVolumeChunkSource extends MultiscaleVolumeChunkSourc } function getNiftiVolumeInfo( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, - abortSignal?: AbortSignal, + options: Partial, ) { - return chunkManager.rpc!.promiseInvoke( + return sharedKvStoreContext.chunkManager.rpc!.promiseInvoke( GET_NIFTI_VOLUME_INFO_RPC_ID, { - chunkManager: chunkManager.addCounterpartRef(), - credentialsProvider: - getCredentialsProviderCounterpart( - chunkManager, - credentialsProvider, - ), + sharedKvStoreContext: sharedKvStoreContext.rpcId, url: url, }, - abortSignal, + { signal: options.signal, progressListener: options.progressListener }, ); } function getDataSource( - chunkManager: ChunkManager, - credentialsManager: CredentialsManager, + sharedKvStoreContext: SharedKvStoreContext, url: string, + options: Partial, ) { - return chunkManager.memoize.getUncounted( + return sharedKvStoreContext.chunkManager.memoize.getAsync( { type: "nifti/getVolume", url }, - async () => { - const { url: parsedUrl, credentialsProvider } = parseSpecialUrl( - url, - credentialsManager, - ); + options, + async (progressOptions) => { const info = await getNiftiVolumeInfo( - chunkManager, - credentialsProvider, - parsedUrl, + sharedKvStoreContext, + url, + progressOptions, ); const volume = new NiftiMultiscaleVolumeChunkSource( - chunkManager, - credentialsProvider, - parsedUrl, + sharedKvStoreContext, + url, info, ); const box = { @@ -177,6 +163,7 @@ function getDataSource( units: info.units, }); const dataSource: DataSource = { + canonicalUrl: `${url}|nifti:`, subsources: [ { id: "default", @@ -204,23 +191,70 @@ function getDataSource( ); } -export class NiftiDataSource extends DataSourceProvider { +export class NiftiDataSource implements KvStoreBasedDataSourceProvider { + get scheme() { + return "nifti"; + } get description() { - return "Single NIfTI file"; + return "NIfTI"; } - get(options: GetDataSourceOptions): Promise { + get singleFile() { + return true; + } + get(options: GetKvStoreBasedDataSourceOptions): Promise { + ensureEmptyUrlSuffix(options.url); return getDataSource( - options.chunkManager, - options.credentialsManager, - options.providerUrl, + options.registry.sharedKvStoreContext, + options.kvStoreUrl, + options, ); } +} - completeUrl(options: CompleteUrlOptions) { - return completeHttpPath( - options.credentialsManager, - options.providerUrl, - options.abortSignal, +function getAutoDetectSpec( + headerSize: number, + magicStringOffset: number, + magicString: string, + version: string, +): AutoDetectFileSpec { + async function match(options: AutoDetectFileOptions) { + const { prefix } = options; + if (prefix.length < magicStringOffset + magicString.length) return []; + const dv = new DataView( + prefix.buffer, + prefix.byteOffset, + prefix.byteLength, ); + let endianness: Endianness; + if (dv.getInt32(0, /*littleEndian=*/ true) === headerSize) { + endianness = Endianness.LITTLE; + } else if (dv.getInt32(0, /*littleEndian=*/ false) === headerSize) { + endianness = Endianness.BIG; + } else { + return []; + } + for (let i = 0; i < magicString.length; ++i) { + if (magicString.charCodeAt(i) !== prefix[i + magicStringOffset]) + return []; + } + + return [ + { + suffix: "nifti:", + description: `NIfTI ${version} (${Endianness[endianness].toLowerCase()}-endian)`, + }, + ]; } + return { + prefixLength: magicStringOffset + magicString.length, + suffixLength: 0, + match, + }; +} + +export function registerAutoDetect(registry: AutoDetectRegistry) { + registry.registerFileFormat(getAutoDetectSpec(348, 344, "n+1\0", "v1")); + registry.registerFileFormat( + getAutoDetectSpec(540, 4, "n+2\0\r\n\x1a\n", "v2"), + ); } diff --git a/src/datasource/nifti/register_default.ts b/src/datasource/nifti/register_default.ts index cdf9a6c4ee..41ed086d2f 100644 --- a/src/datasource/nifti/register_default.ts +++ b/src/datasource/nifti/register_default.ts @@ -14,7 +14,18 @@ * limitations under the License. */ -import { registerProvider } from "#src/datasource/default_provider.js"; -import { NiftiDataSource } from "#src/datasource/nifti/frontend.js"; +import { + dataSourceAutoDetectRegistry, + registerKvStoreBasedDataProvider, + registerProvider, +} from "#src/datasource/default_provider.js"; +import { KvStoreBasedDataSourceLegacyUrlAdapter } from "#src/datasource/index.js"; +import { + NiftiDataSource, + registerAutoDetect, +} from "#src/datasource/nifti/frontend.js"; -registerProvider("nifti", () => new NiftiDataSource()); +const provider = new NiftiDataSource(); +registerKvStoreBasedDataProvider(provider); +registerProvider(new KvStoreBasedDataSourceLegacyUrlAdapter(provider)); +registerAutoDetect(dataSourceAutoDetectRegistry); diff --git a/src/datasource/obj/backend.ts b/src/datasource/obj/backend.ts index bc5f42c798..27554ccc35 100644 --- a/src/datasource/obj/backend.ts +++ b/src/datasource/obj/backend.ts @@ -16,17 +16,26 @@ import { parseOBJFromArrayBuffer } from "#src/async_computation/obj_mesh_request.js"; import { requestAsyncComputation } from "#src/async_computation/request.js"; -import { GenericSharedDataSource } from "#src/chunk_manager/generic_file_source.js"; +import { getCachedDecodedUrl } from "#src/chunk_manager/generic_file_source.js"; +import type { ReadResponse } from "#src/kvstore/index.js"; import { registerSingleMeshFactory } from "#src/single_mesh/backend.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; /** - * This needs to be a global function, because it identifies the instance of GenericSharedDataSource + * This needs to be a global function, because it identifies the instance of SimpleAsyncCache * to use. */ -function parse(buffer: ArrayBuffer, abortSignal: AbortSignal) { +async function parse( + readResponse: ReadResponse | undefined, + progressOptions: Partial, +) { + if (readResponse === undefined) { + throw new Error("Not found"); + } + const buffer = await readResponse.response.arrayBuffer(); return requestAsyncComputation( parseOBJFromArrayBuffer, - abortSignal, + progressOptions.signal, [buffer], buffer, ); @@ -34,13 +43,6 @@ function parse(buffer: ArrayBuffer, abortSignal: AbortSignal) { registerSingleMeshFactory("obj", { description: "OBJ", - getMesh: (chunkManager, credentialsProvider, url, getPriority, abortSignal) => - GenericSharedDataSource.getUrl( - chunkManager, - credentialsProvider, - parse, - url, - getPriority, - abortSignal, - ), + getMesh: (sharedKvStoreContext, url, options) => + getCachedDecodedUrl(sharedKvStoreContext, url, parse, options), }); diff --git a/src/datasource/obj/frontend.ts b/src/datasource/obj/frontend.ts index e2d9ab15a6..19d2c6a8ce 100644 --- a/src/datasource/obj/frontend.ts +++ b/src/datasource/obj/frontend.ts @@ -19,24 +19,31 @@ import { makeIdentityTransform, } from "#src/coordinate_transform.js"; import type { - CompleteUrlOptions, DataSource, - GetDataSourceOptions, + GetKvStoreBasedDataSourceOptions, + KvStoreBasedDataSourceProvider, } from "#src/datasource/index.js"; -import { DataSourceProvider } from "#src/datasource/index.js"; +import { ensureEmptyUrlSuffix } from "#src/kvstore/url.js"; import { getSingleMeshSource } from "#src/single_mesh/frontend.js"; -import { completeHttpPath } from "#src/util/http_path_completion.js"; -export class ObjDataSource extends DataSourceProvider { +export class ObjDataSource implements KvStoreBasedDataSourceProvider { + get scheme() { + return "obj"; + } get description() { - return "Wavefront OBJ mesh file"; + return "Wavefront OBJ mesh"; + } + + get singleFile() { + return true; } - async get(options: GetDataSourceOptions): Promise { + async get(options: GetKvStoreBasedDataSourceOptions): Promise { + ensureEmptyUrlSuffix(options.url); const meshSource = await getSingleMeshSource( - options.chunkManager, - options.credentialsManager, - options.url, + options.registry.sharedKvStoreContext, + options.kvStoreUrl, + options, ); const modelSpace = makeCoordinateSpace({ rank: 3, @@ -56,11 +63,4 @@ export class ObjDataSource extends DataSourceProvider { }; return dataSource; } - completeUrl(options: CompleteUrlOptions) { - return completeHttpPath( - options.credentialsManager, - options.providerUrl, - options.abortSignal, - ); - } } diff --git a/src/datasource/obj/register_default.ts b/src/datasource/obj/register_default.ts index 7f846e4743..4e4e2832bd 100644 --- a/src/datasource/obj/register_default.ts +++ b/src/datasource/obj/register_default.ts @@ -14,7 +14,13 @@ * limitations under the License. */ -import { registerProvider } from "#src/datasource/default_provider.js"; +import { + registerKvStoreBasedDataProvider, + registerProvider, +} from "#src/datasource/default_provider.js"; +import { KvStoreBasedDataSourceLegacyUrlAdapter } from "#src/datasource/index.js"; import { ObjDataSource } from "#src/datasource/obj/frontend.js"; -registerProvider("obj", () => new ObjDataSource()); +const provider = new ObjDataSource(); +registerKvStoreBasedDataProvider(provider); +registerProvider(new KvStoreBasedDataSourceLegacyUrlAdapter(provider)); diff --git a/src/datasource/precomputed/backend.ts b/src/datasource/precomputed/backend.ts index c7a1eac117..1189847503 100644 --- a/src/datasource/precomputed/backend.ts +++ b/src/datasource/precomputed/backend.ts @@ -30,23 +30,24 @@ import { annotationTypeHandlers, annotationTypes, } from "#src/annotation/index.js"; -import type { Chunk, ChunkManager } from "#src/chunk_manager/backend.js"; import { WithParameters } from "#src/chunk_manager/backend.js"; -import { GenericSharedDataSource } from "#src/chunk_manager/generic_file_source.js"; -import { WithSharedCredentialsProviderCounterpart } from "#src/credentials_provider/shared_counterpart.js"; -import type { ShardingParameters } from "#src/datasource/precomputed/base.js"; import { AnnotationSourceParameters, AnnotationSpatialIndexSourceParameters, - DataEncoding, - IndexedSegmentPropertySourceParameters, MeshSourceParameters, MultiscaleMeshSourceParameters, - ShardingHashFunction, SkeletonSourceParameters, VolumeChunkEncoding, VolumeChunkSourceParameters, } from "#src/datasource/precomputed/base.js"; +import type { + ShardedKvStore, + ShardInfo, +} from "#src/datasource/precomputed/sharded.js"; +import { getShardedKvStoreIfApplicable } from "#src/datasource/precomputed/sharded.js"; +import { WithSharedKvStoreContextCounterpart } from "#src/kvstore/backend.js"; +import type { KvStoreWithPath, ReadResponse } from "#src/kvstore/index.js"; +import { readKvStore } from "#src/kvstore/index.js"; import type { FragmentChunk, ManifestChunk, @@ -64,7 +65,6 @@ import { MultiscaleMeshSource, } from "#src/mesh/backend.js"; import { decodeDracoPartitioned } from "#src/mesh/draco/index.js"; -import { IndexedSegmentPropertySourceBackend } from "#src/segmentation_display_state/backend.js"; import type { SkeletonChunk } from "#src/skeleton/backend.js"; import { SkeletonSource } from "#src/skeleton/backend.js"; import { decodeSkeletonChunk } from "#src/skeleton/decode_precomputed_skeleton.js"; @@ -77,20 +77,8 @@ import { decodePngChunk } from "#src/sliceview/backend_chunk_decoders/png.js"; import { decodeRawChunk } from "#src/sliceview/backend_chunk_decoders/raw.js"; import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; import { VolumeChunkSource } from "#src/sliceview/volume/backend.js"; -import { fetchSpecialHttpByteRange } from "#src/util/byte_range_http_requests.js"; -import type { Borrowed } from "#src/util/disposable.js"; import { convertEndian32, Endianness } from "#src/util/endian.js"; import { vec3 } from "#src/util/geom.js"; -import { decodeGzip } from "#src/util/gzip.js"; -import { murmurHash3_x86_128Hash64Bits } from "#src/util/hash.js"; -import { isNotFoundError } from "#src/util/http_request.js"; -import { stableStringify } from "#src/util/json.js"; -import { getObjectId } from "#src/util/object_id.js"; -import type { - SpecialProtocolCredentials, - SpecialProtocolCredentialsProvider, -} from "#src/util/special_protocol_request.js"; -import { fetchSpecialOk } from "#src/util/special_protocol_request.js"; import { Uint64 } from "#src/util/uint64.js"; import { encodeZIndexCompressed, @@ -102,251 +90,6 @@ import { registerSharedObject } from "#src/worker_rpc.js"; // Set to true to validate the multiscale index. const DEBUG_MULTISCALE_INDEX = false; -const shardingHashFunctions: Map void> = - new Map([ - [ - ShardingHashFunction.MURMURHASH3_X86_128, - (out) => { - murmurHash3_x86_128Hash64Bits(out, 0, out.low, out.high); - }, - ], - [ShardingHashFunction.IDENTITY, (_out) => {}], - ]); - -interface ShardInfo { - shardUrl: string; - offset: Uint64; -} - -interface DecodedMinishardIndex { - data: Uint32Array; - shardUrl: string; -} - -interface MinishardIndexSource - extends GenericSharedDataSource { - sharding: ShardingParameters; - credentialsProvider: SpecialProtocolCredentialsProvider; -} - -function getMinishardIndexDataSource( - chunkManager: Borrowed, - credentialsProvider: SpecialProtocolCredentialsProvider, - parameters: { url: string; sharding: ShardingParameters | undefined }, -): MinishardIndexSource | undefined { - const { url, sharding } = parameters; - if (sharding === undefined) return undefined; - const source = GenericSharedDataSource.get< - Uint64, - DecodedMinishardIndex | undefined - >( - chunkManager, - stableStringify({ - type: "precomputed:shardedDataSource", - url, - sharding, - credentialsProvider: getObjectId(credentialsProvider), - }), - { - download: async (shardAndMinishard: Uint64, abortSignal: AbortSignal) => { - const minishard = Uint64.lowMask(new Uint64(), sharding.minishardBits); - Uint64.and(minishard, minishard, shardAndMinishard); - const shard = Uint64.lowMask(new Uint64(), sharding.shardBits); - const temp = new Uint64(); - Uint64.rshift(temp, shardAndMinishard, sharding.minishardBits); - Uint64.and(shard, shard, temp); - const shardUrl = `${url}/${shard - .toString(16) - .padStart(Math.ceil(sharding.shardBits / 4), "0")}.shard`; - // Retrive minishard index start/end offsets. - const shardIndexSize = new Uint64(16); - Uint64.lshift(shardIndexSize, shardIndexSize, sharding.minishardBits); - - // Multiply minishard by 16. - const shardIndexStart = Uint64.lshift(new Uint64(), minishard, 4); - const shardIndexEnd = Uint64.addUint32( - new Uint64(), - shardIndexStart, - 16, - ); - let shardIndexResponse: ArrayBuffer; - try { - shardIndexResponse = await fetchSpecialHttpByteRange( - credentialsProvider, - shardUrl, - shardIndexStart, - shardIndexEnd, - abortSignal, - ); - } catch (e) { - if (isNotFoundError(e)) return { data: undefined, size: 0 }; - throw e; - } - if (shardIndexResponse.byteLength !== 16) { - throw new Error("Failed to retrieve minishard offset"); - } - const shardIndexDv = new DataView(shardIndexResponse); - const minishardStartOffset = new Uint64( - shardIndexDv.getUint32(0, /*littleEndian=*/ true), - shardIndexDv.getUint32(4, /*littleEndian=*/ true), - ); - const minishardEndOffset = new Uint64( - shardIndexDv.getUint32(8, /*littleEndian=*/ true), - shardIndexDv.getUint32(12, /*littleEndian=*/ true), - ); - if (Uint64.equal(minishardStartOffset, minishardEndOffset)) { - return { data: undefined, size: 0 }; - } - // The start/end offsets in the shard index are relative to the end of the shard - // index. - Uint64.add(minishardStartOffset, minishardStartOffset, shardIndexSize); - Uint64.add(minishardEndOffset, minishardEndOffset, shardIndexSize); - - let minishardIndexResponse = await fetchSpecialHttpByteRange( - credentialsProvider, - shardUrl, - minishardStartOffset, - minishardEndOffset, - abortSignal, - ); - if (sharding.minishardIndexEncoding === DataEncoding.GZIP) { - minishardIndexResponse = await decodeGzip( - minishardIndexResponse, - "gzip", - ); - } - if (minishardIndexResponse.byteLength % 24 !== 0) { - throw new Error( - `Invalid minishard index length: ${minishardIndexResponse.byteLength}`, - ); - } - const minishardIndex = new Uint32Array(minishardIndexResponse); - convertEndian32(minishardIndex, Endianness.LITTLE); - - const minishardIndexSize = minishardIndex.byteLength / 24; - let prevEntryKeyLow = 0; - let prevEntryKeyHigh = 0; - // Offsets in the minishard index are relative to the end of the shard index. - let prevStartLow = shardIndexSize.low; - let prevStartHigh = shardIndexSize.high; - for (let i = 0; i < minishardIndexSize; ++i) { - let entryKeyLow = prevEntryKeyLow + minishardIndex[i * 2]; - let entryKeyHigh = prevEntryKeyHigh + minishardIndex[i * 2 + 1]; - if (entryKeyLow >= 4294967296) { - entryKeyLow -= 4294967296; - entryKeyHigh += 1; - } - prevEntryKeyLow = minishardIndex[i * 2] = entryKeyLow; - prevEntryKeyHigh = minishardIndex[i * 2 + 1] = entryKeyHigh; - let startLow = - prevStartLow + minishardIndex[(minishardIndexSize + i) * 2]; - let startHigh = - prevStartHigh + minishardIndex[(minishardIndexSize + i) * 2 + 1]; - if (startLow >= 4294967296) { - startLow -= 4294967296; - startHigh += 1; - } - minishardIndex[(minishardIndexSize + i) * 2] = startLow; - minishardIndex[(minishardIndexSize + i) * 2 + 1] = startHigh; - const sizeLow = minishardIndex[(2 * minishardIndexSize + i) * 2]; - const sizeHigh = minishardIndex[(2 * minishardIndexSize + i) * 2 + 1]; - let endLow = startLow + sizeLow; - let endHigh = startHigh + sizeHigh; - if (endLow >= 4294967296) { - endLow -= 4294967296; - endHigh += 1; - } - prevStartLow = endLow; - prevStartHigh = endHigh; - minishardIndex[(2 * minishardIndexSize + i) * 2] = endLow; - minishardIndex[(2 * minishardIndexSize + i) * 2 + 1] = endHigh; - } - return { - data: { data: minishardIndex, shardUrl }, - size: minishardIndex.byteLength, - }; - }, - encodeKey: (key: Uint64) => key.toString(), - sourceQueueLevel: 1, - }, - ) as MinishardIndexSource; - source.sharding = sharding; - source.credentialsProvider = credentialsProvider; - return source; -} - -function findMinishardEntry( - minishardIndex: DecodedMinishardIndex, - key: Uint64, -): { startOffset: Uint64; endOffset: Uint64 } | undefined { - const minishardIndexData = minishardIndex.data; - const minishardIndexSize = minishardIndexData.length / 6; - const keyLow = key.low; - const keyHigh = key.high; - for (let i = 0; i < minishardIndexSize; ++i) { - if ( - minishardIndexData[i * 2] !== keyLow || - minishardIndexData[i * 2 + 1] !== keyHigh - ) { - continue; - } - const startOffset = new Uint64( - minishardIndexData[(minishardIndexSize + i) * 2], - minishardIndexData[(minishardIndexSize + i) * 2 + 1], - ); - const endOffset = new Uint64( - minishardIndexData[(2 * minishardIndexSize + i) * 2], - minishardIndexData[(2 * minishardIndexSize + i) * 2 + 1], - ); - return { startOffset, endOffset }; - } - return undefined; -} - -async function getShardedData( - minishardIndexSource: MinishardIndexSource, - chunk: Chunk, - key: Uint64, - abortSignal: AbortSignal, -): Promise<{ shardInfo: ShardInfo; data: ArrayBuffer } | undefined> { - const { sharding } = minishardIndexSource; - const hashFunction = shardingHashFunctions.get(sharding.hash)!; - const hashCode = Uint64.rshift(new Uint64(), key, sharding.preshiftBits); - hashFunction(hashCode); - const shardAndMinishard = Uint64.lowMask( - new Uint64(), - sharding.minishardBits + sharding.shardBits, - ); - Uint64.and(shardAndMinishard, shardAndMinishard, hashCode); - const getPriority = () => ({ - priorityTier: chunk.priorityTier, - priority: chunk.priority, - }); - const minishardIndex = await minishardIndexSource.getData( - shardAndMinishard, - getPriority, - abortSignal, - ); - if (minishardIndex === undefined) return undefined; - const minishardEntry = findMinishardEntry(minishardIndex, key); - if (minishardEntry === undefined) return undefined; - const { startOffset, endOffset } = minishardEntry; - let data = await fetchSpecialHttpByteRange( - minishardIndexSource.credentialsProvider, - minishardIndex.shardUrl, - startOffset, - endOffset, - abortSignal, - ); - if (minishardIndexSource.sharding.dataEncoding === DataEncoding.GZIP) { - data = await decodeGzip(data, "gzip"); - } - return { - data, - shardInfo: { shardUrl: minishardIndex.shardUrl, offset: startOffset }, - }; -} - function getOrNotFoundError(v: T | undefined) { if (v === undefined) throw new Error("not found"); return v; @@ -365,16 +108,17 @@ chunkDecoders.set(VolumeChunkEncoding.JXL, decodeJxlChunk); @registerSharedObject() export class PrecomputedVolumeChunkSource extends WithParameters( - WithSharedCredentialsProviderCounterpart()( - VolumeChunkSource, - ), + WithSharedKvStoreContextCounterpart(VolumeChunkSource), VolumeChunkSourceParameters, ) { chunkDecoder = chunkDecoders.get(this.parameters.encoding)!; - private minishardIndexSource = getMinishardIndexDataSource( - this.chunkManager, - this.credentialsProvider, - this.parameters, + kvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore( + this.parameters.url, + ); + shardedKvStore = getShardedKvStoreIfApplicable( + this, + this.kvStore, + this.parameters.sharding, ); gridShape = (() => { @@ -386,36 +130,25 @@ export class PrecomputedVolumeChunkSource extends WithParameters( return gridShape; })(); - async download(chunk: VolumeChunk, abortSignal: AbortSignal): Promise { - const { parameters } = this; - - const { minishardIndexSource } = this; - let response: ArrayBuffer | undefined; - if (minishardIndexSource === undefined) { - let url: string; + async download(chunk: VolumeChunk, signal: AbortSignal): Promise { + const { shardedKvStore } = this; + let readResponse: ReadResponse | undefined; + if (shardedKvStore === undefined) { + const { kvStore } = this; + let path: string; { // chunkPosition must not be captured, since it will be invalidated by the next call to // computeChunkBounds. const chunkPosition = this.computeChunkBounds(chunk); const chunkDataSize = chunk.chunkDataSize!; - url = - `${parameters.url}/${chunkPosition[0]}-${ + path = + `${kvStore.path}${chunkPosition[0]}-${ chunkPosition[0] + chunkDataSize[0] }_` + `${chunkPosition[1]}-${chunkPosition[1] + chunkDataSize[1]}_` + `${chunkPosition[2]}-${chunkPosition[2] + chunkDataSize[2]}`; } - try { - response = await fetchSpecialOk(this.credentialsProvider, url, { - signal: abortSignal, - }).then((response) => response.arrayBuffer()); - } catch (e) { - if (isNotFoundError(e)) { - response = undefined; - } else { - throw e; - } - } + readResponse = await kvStore.store.read(path, { signal }); } else { this.computeChunkBounds(chunk); const { gridShape } = this; @@ -424,7 +157,6 @@ export class PrecomputedVolumeChunkSource extends WithParameters( const yBits = Math.ceil(Math.log2(gridShape[1])); const zBits = Math.ceil(Math.log2(gridShape[2])); const chunkIndex = encodeZIndexCompressed3d( - new Uint64(), xBits, yBits, zBits, @@ -432,17 +164,14 @@ export class PrecomputedVolumeChunkSource extends WithParameters( chunkGridPosition[1], chunkGridPosition[2], ); - response = ( - await getShardedData( - minishardIndexSource, - chunk, - chunkIndex, - abortSignal, - ) - )?.data; + readResponse = await shardedKvStore.read(chunkIndex, { signal }); } - if (response !== undefined) { - await this.chunkDecoder(chunk, abortSignal, response); + if (readResponse !== undefined) { + await this.chunkDecoder( + chunk, + signal, + await readResponse.response.arrayBuffer(), + ); } } } @@ -470,29 +199,30 @@ export function decodeFragmentChunk( @registerSharedObject() export class PrecomputedMeshSource extends WithParameters( - WithSharedCredentialsProviderCounterpart()( - MeshSource, - ), + WithSharedKvStoreContextCounterpart(MeshSource), MeshSourceParameters, ) { - async download(chunk: ManifestChunk, abortSignal: AbortSignal) { - const { parameters } = this; - const response = await fetchSpecialOk( - this.credentialsProvider, - `${parameters.url}/${chunk.objectId}:${parameters.lod}`, - { signal: abortSignal }, + kvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore( + this.parameters.url, + ); + async download(chunk: ManifestChunk, signal: AbortSignal) { + const { parameters, kvStore } = this; + const response = await readKvStore( + kvStore.store, + `${kvStore.path}${chunk.objectId}:${parameters.lod}`, + { signal, throwIfMissing: true }, ); - decodeManifestChunk(chunk, await response.json()); + decodeManifestChunk(chunk, await response.response.json()); } - async downloadFragment(chunk: FragmentChunk, abortSignal: AbortSignal) { - const { parameters } = this; - const response = await fetchSpecialOk( - this.credentialsProvider, - `${parameters.url}/${chunk.fragmentId}`, - { signal: abortSignal }, + async downloadFragment(chunk: FragmentChunk, signal: AbortSignal) { + const { kvStore } = this; + const response = await readKvStore( + kvStore.store, + `${kvStore.path}${chunk.fragmentId}`, + { signal, throwIfMissing: true }, ); - decodeFragmentChunk(chunk, await response.arrayBuffer()); + decodeFragmentChunk(chunk, await response.response.arrayBuffer()); } } @@ -740,146 +470,128 @@ async function decodeMultiscaleFragmentChunk( @registerSharedObject() // export class PrecomputedMultiscaleMeshSource extends WithParameters( - WithSharedCredentialsProviderCounterpart()( - MultiscaleMeshSource, - ), + WithSharedKvStoreContextCounterpart(MultiscaleMeshSource), MultiscaleMeshSourceParameters, ) { - private minishardIndexSource = getMinishardIndexDataSource( - this.chunkManager, - this.credentialsProvider, - { url: this.parameters.url, sharding: this.parameters.metadata.sharding }, + kvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore( + this.parameters.url, + ); + shardedKvStore = getShardedKvStoreIfApplicable( + this, + this.kvStore, + this.parameters.metadata.sharding, ); async download( chunk: PrecomputedMultiscaleManifestChunk, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise { - const { parameters, minishardIndexSource } = this; - let data: ArrayBuffer; - if (minishardIndexSource === undefined) { - data = await fetchSpecialOk( - this.credentialsProvider, - `${parameters.url}/${chunk.objectId}.index`, - { signal: abortSignal }, - ).then((response) => response.arrayBuffer()); + const { shardedKvStore } = this; + let readResponse: ReadResponse | undefined; + if (shardedKvStore === undefined) { + const { kvStore } = this; + readResponse = await kvStore.store.read( + `${kvStore.path}${chunk.objectId}.index`, + { signal }, + ); } else { - ({ data, shardInfo: chunk.shardInfo } = getOrNotFoundError( - await getShardedData( - minishardIndexSource, - chunk, - chunk.objectId, - abortSignal, - ), - )); + ({ response: readResponse, shardInfo: chunk.shardInfo } = + getOrNotFoundError( + await shardedKvStore.readWithShardInfo(chunk.objectId.toBigInt(), { + signal, + }), + )); } + + const data = await getOrNotFoundError(readResponse).response.arrayBuffer(); + decodeMultiscaleManifestChunk(chunk, data); } async downloadFragment( chunk: MultiscaleFragmentChunk, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise { - const { parameters } = this; + const { kvStore } = this; const manifestChunk = chunk.manifestChunk! as PrecomputedMultiscaleManifestChunk; const chunkIndex = chunk.chunkIndex; const { shardInfo, offsets } = manifestChunk; const startOffset = offsets[chunkIndex]; const endOffset = offsets[chunkIndex + 1]; - let requestUrl: string; - let adjustedStartOffset: Uint64 | number; - let adjustedEndOffset: Uint64 | number; + let requestPath: string; + let adjustedStartOffset: number; + let adjustedEndOffset: number; if (shardInfo !== undefined) { - requestUrl = shardInfo.shardUrl; + requestPath = shardInfo.shardPath; const fullDataSize = offsets[offsets.length - 1]; - let startLow = shardInfo.offset.low - fullDataSize + startOffset; - let startHigh = shardInfo.offset.high; - let endLow = startLow + endOffset - startOffset; - let endHigh = startHigh; - while (startLow < 0) { - startLow += 4294967296; - startHigh -= 1; - } - while (endLow < 0) { - endLow += 4294967296; - endHigh -= 1; - } - while (endLow > 4294967296) { - endLow -= 4294967296; - endHigh += 1; - } - adjustedStartOffset = new Uint64(startLow, startHigh); - adjustedEndOffset = new Uint64(endLow, endHigh); + const start = shardInfo.offset - fullDataSize + startOffset; + const end = start + endOffset - startOffset; + adjustedStartOffset = start; + adjustedEndOffset = end; } else { - requestUrl = `${parameters.url}/${manifestChunk.objectId}`; + requestPath = `${kvStore.path}${manifestChunk.objectId}`; adjustedStartOffset = startOffset; adjustedEndOffset = endOffset; } - const response = await fetchSpecialHttpByteRange( - this.credentialsProvider, - requestUrl, - adjustedStartOffset, - adjustedEndOffset, - abortSignal, + const readResponse = await readKvStore(kvStore.store, requestPath, { + signal, + byteRange: { + offset: adjustedStartOffset, + length: adjustedEndOffset - adjustedStartOffset, + }, + throwIfMissing: true, + strictByteRange: true, + }); + await decodeMultiscaleFragmentChunk( + chunk, + await readResponse.response.arrayBuffer(), ); - await decodeMultiscaleFragmentChunk(chunk, response); } } async function fetchByUint64( - credentialsProvider: SpecialProtocolCredentialsProvider, - url: string, - chunk: Chunk, - minishardIndexSource: MinishardIndexSource | undefined, + chunkSource: { + kvStore: KvStoreWithPath; + shardedKvStore: ShardedKvStore | undefined; + }, id: Uint64, - abortSignal: AbortSignal, -) { - if (minishardIndexSource === undefined) { - try { - return await fetchSpecialOk(credentialsProvider, `${url}/${id}`, { - signal: abortSignal, - }).then((response) => response.arrayBuffer()); - } catch (e) { - if (isNotFoundError(e)) return undefined; - throw e; - } + signal: AbortSignal, +): Promise { + const { shardedKvStore } = chunkSource; + if (shardedKvStore === undefined) { + const { kvStore } = chunkSource; + return kvStore.store.read(`${kvStore.path}${id}`, { + signal, + }); + } else { + return shardedKvStore.read(id.toBigInt(), { signal }); } - const result = await getShardedData( - minishardIndexSource, - chunk, - id, - abortSignal, - ); - if (result === undefined) return undefined; - return result.data; } @registerSharedObject() // export class PrecomputedSkeletonSource extends WithParameters( - WithSharedCredentialsProviderCounterpart()( - SkeletonSource, - ), + WithSharedKvStoreContextCounterpart(SkeletonSource), SkeletonSourceParameters, ) { - private minishardIndexSource = getMinishardIndexDataSource( - this.chunkManager, - this.credentialsProvider, - { url: this.parameters.url, sharding: this.parameters.metadata.sharding }, + kvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore( + this.parameters.url, + ); + shardedKvStore = getShardedKvStoreIfApplicable( + this, + this.kvStore, + this.parameters.metadata.sharding, ); - async download(chunk: SkeletonChunk, abortSignal: AbortSignal) { + async download(chunk: SkeletonChunk, signal: AbortSignal) { const { parameters } = this; const response = getOrNotFoundError( - await fetchByUint64( - this.credentialsProvider, - parameters.url, - chunk, - this.minishardIndexSource, - chunk.objectId, - abortSignal, - ), + await fetchByUint64(this, chunk.objectId, signal), + ); + decodeSkeletonChunk( + chunk, + await response.response.arrayBuffer(), + parameters.metadata.vertexAttributes, ); - decodeSkeletonChunk(chunk, response, parameters.metadata.vertexAttributes); } } @@ -1022,52 +734,39 @@ function parseSingleAnnotation( @registerSharedObject() // export class PrecomputedAnnotationSpatialIndexSourceBackend extends WithParameters( - WithSharedCredentialsProviderCounterpart()( - AnnotationGeometryChunkSourceBackend, - ), + WithSharedKvStoreContextCounterpart(AnnotationGeometryChunkSourceBackend), AnnotationSpatialIndexSourceParameters, ) { - private minishardIndexSource = getMinishardIndexDataSource( - this.chunkManager, - this.credentialsProvider, - this.parameters, + kvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore( + this.parameters.url, + ); + shardedKvStore = getShardedKvStoreIfApplicable( + this, + this.kvStore, + this.parameters.sharding, ); declare parent: PrecomputedAnnotationSourceBackend; - async download(chunk: AnnotationGeometryChunk, abortSignal: AbortSignal) { - const { parameters } = this; - - const { minishardIndexSource } = this; + async download(chunk: AnnotationGeometryChunk, signal: AbortSignal) { + const { shardedKvStore } = this; const { parent } = this; - let response: ArrayBuffer | undefined; + let response: ReadResponse | undefined; const { chunkGridPosition } = chunk; - if (minishardIndexSource === undefined) { - const url = `${parameters.url}/${chunkGridPosition.join("_")}`; - try { - response = await fetchSpecialOk(this.credentialsProvider, url, { - signal: abortSignal, - }).then((response) => response.arrayBuffer()); - } catch (e) { - if (!isNotFoundError(e)) throw e; - } + if (shardedKvStore === undefined) { + const { kvStore } = this; + const path = `${kvStore.path}/${chunkGridPosition.join("_")}`; + response = await kvStore.store.read(path, { signal }); } else { const { upperChunkBound } = this.spec; const { chunkGridPosition } = chunk; const chunkIndex = encodeZIndexCompressed( - new Uint64(), chunkGridPosition, upperChunkBound, ); - const result = await getShardedData( - minishardIndexSource, - chunk, - chunkIndex, - abortSignal, - ); - if (result !== undefined) response = result.data; + response = await shardedKvStore.read(chunkIndex, { signal }); } if (response !== undefined) { chunk.data = parseAnnotations( - response, + await response.response.arrayBuffer(), parent.parameters, parent.annotationPropertySerializer, ); @@ -1077,19 +776,26 @@ export class PrecomputedAnnotationSpatialIndexSourceBackend extends WithParamete @registerSharedObject() // export class PrecomputedAnnotationSourceBackend extends WithParameters( - WithSharedCredentialsProviderCounterpart()( - AnnotationSource, - ), + WithSharedKvStoreContextCounterpart(AnnotationSource), AnnotationSourceParameters, ) { - private byIdMinishardIndexSource = getMinishardIndexDataSource( - this.chunkManager, - this.credentialsProvider, - this.parameters.byId, + kvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore( + this.parameters.byId.url, ); - private relationshipIndexSource = this.parameters.relationships.map((x) => - getMinishardIndexDataSource(this.chunkManager, this.credentialsProvider, x), + shardedKvStore = getShardedKvStoreIfApplicable( + this, + this.kvStore, + this.parameters.byId.sharding, ); + private relationshipIndexSource = this.parameters.relationships.map((x) => { + const kvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore(x.url); + const shardedKvStore = getShardedKvStoreIfApplicable( + this, + kvStore, + x.sharding, + ); + return { kvStore, shardedKvStore }; + }); annotationPropertySerializer = new AnnotationPropertySerializer( this.parameters.rank, annotationTypeHandlers[this.parameters.type].serializedBytes( @@ -1101,45 +807,30 @@ export class PrecomputedAnnotationSourceBackend extends WithParameters( async downloadSegmentFilteredGeometry( chunk: AnnotationSubsetGeometryChunk, relationshipIndex: number, - abortSignal: AbortSignal, + signal: AbortSignal, ) { - const { parameters } = this; const response = await fetchByUint64( - this.credentialsProvider, - parameters.relationships[relationshipIndex].url, - chunk, this.relationshipIndexSource[relationshipIndex], chunk.objectId, - abortSignal, + signal, ); if (response !== undefined) { chunk.data = parseAnnotations( - response, + await response.response.arrayBuffer(), this.parameters, this.annotationPropertySerializer, ); } } - async downloadMetadata( - chunk: AnnotationMetadataChunk, - abortSignal: AbortSignal, - ) { - const { parameters } = this; + async downloadMetadata(chunk: AnnotationMetadataChunk, signal: AbortSignal) { const id = Uint64.parseString(chunk.key!); - const response = await fetchByUint64( - this.credentialsProvider, - parameters.byId.url, - chunk, - this.byIdMinishardIndexSource, - id, - abortSignal, - ); + const response = await fetchByUint64(this, id, signal); if (response === undefined) { chunk.annotation = null; } else { chunk.annotation = parseSingleAnnotation( - response, + await response.response.arrayBuffer(), this.parameters, this.annotationPropertySerializer, chunk.key!, @@ -1147,17 +838,3 @@ export class PrecomputedAnnotationSourceBackend extends WithParameters( } } } - -@registerSharedObject() -export class PrecomputedIndexedSegmentPropertySourceBackend extends WithParameters( - WithSharedCredentialsProviderCounterpart()( - IndexedSegmentPropertySourceBackend, - ), - IndexedSegmentPropertySourceParameters, -) { - minishardIndexSource = getMinishardIndexDataSource( - this.chunkManager, - this.credentialsProvider, - this.parameters, - ); -} diff --git a/src/datasource/precomputed/frontend.ts b/src/datasource/precomputed/frontend.ts index 52d29ecba4..bf532e0bbb 100644 --- a/src/datasource/precomputed/frontend.ts +++ b/src/datasource/precomputed/frontend.ts @@ -24,7 +24,6 @@ import { makeDataBoundsBoundingBoxAnnotationSet, parseAnnotationPropertySpecs, } from "#src/annotation/index.js"; -import type { ChunkManager } from "#src/chunk_manager/frontend.js"; import { WithParameters } from "#src/chunk_manager/frontend.js"; import type { BoundingBox, @@ -37,16 +36,15 @@ import { makeIdentityTransform, makeIdentityTransformedBoundingBox, } from "#src/coordinate_transform.js"; -import { WithCredentialsProvider } from "#src/credentials_provider/chunk_source_frontend.js"; -import type { - CompleteUrlOptions, - ConvertLegacyUrlOptions, - DataSource, - DataSubsourceEntry, - GetDataSourceOptions, - NormalizeUrlOptions, +import { + KvStoreBasedDataSourceLegacyUrlAdapter, + type ConvertLegacyUrlOptions, + type DataSource, + type DataSourceLookupResult, + type DataSubsourceEntry, + type GetKvStoreBasedDataSourceOptions, + type KvStoreBasedDataSourceProvider, } from "#src/datasource/index.js"; -import { DataSourceProvider, RedirectError } from "#src/datasource/index.js"; import type { MultiscaleMeshMetadata, ShardingParameters, @@ -56,7 +54,6 @@ import { AnnotationSourceParameters, AnnotationSpatialIndexSourceParameters, DataEncoding, - IndexedSegmentPropertySourceParameters, MeshSourceParameters, MultiscaleMeshSourceParameters, ShardingHashFunction, @@ -64,6 +61,16 @@ import { VolumeChunkEncoding, VolumeChunkSourceParameters, } from "#src/datasource/precomputed/base.js"; +import type { AutoDetectRegistry } from "#src/kvstore/auto_detect.js"; +import { simpleFilePresenceAutoDetectDirectorySpec } from "#src/kvstore/auto_detect.js"; +import { WithSharedKvStoreContext } from "#src/kvstore/chunk_source_frontend.js"; +import type { KvStoreContext } from "#src/kvstore/context.js"; +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; +import { + kvstoreEnsureDirectoryPipelineUrl, + parseUrlSuffix, + pipelineUrlJoin, +} from "#src/kvstore/url.js"; import { VertexPositionFormat } from "#src/mesh/base.js"; import { MeshSource, MultiscaleMeshSource } from "#src/mesh/frontend.js"; import type { @@ -71,7 +78,6 @@ import type { InlineSegmentPropertyMap, } from "#src/segmentation_display_state/property_map.js"; import { - IndexedSegmentPropertySource, normalizeInlineSegmentPropertyMap, SegmentPropertyMap, } from "#src/segmentation_display_state/property_map.js"; @@ -90,10 +96,7 @@ import { } from "#src/sliceview/volume/frontend.js"; import { transposeNestedArrays } from "#src/util/array.js"; import { DATA_TYPE_ARRAY_CONSTRUCTOR, DataType } from "#src/util/data_type.js"; -import type { Borrowed } from "#src/util/disposable.js"; import { mat4, vec3 } from "#src/util/geom.js"; -import { completeHttpPath } from "#src/util/http_path_completion.js"; -import { isNotFoundError } from "#src/util/http_request.js"; import { parseArray, parseFixedLengthArray, @@ -113,34 +116,27 @@ import { verifyOptionalBoolean, } from "#src/util/json.js"; import * as matrix from "#src/util/matrix.js"; -import { getObjectId } from "#src/util/object_id.js"; -import type { - SpecialProtocolCredentials, - SpecialProtocolCredentialsProvider, -} from "#src/util/special_protocol_request.js"; -import { - fetchSpecialOk, - parseSpecialUrl, -} from "#src/util/special_protocol_request.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; import { Uint64 } from "#src/util/uint64.js"; export class PrecomputedVolumeChunkSource extends WithParameters( - WithCredentialsProvider()(VolumeChunkSource), + WithSharedKvStoreContext(VolumeChunkSource), VolumeChunkSourceParameters, ) {} class PrecomputedMeshSource extends WithParameters( - WithCredentialsProvider()(MeshSource), + WithSharedKvStoreContext(MeshSource), MeshSourceParameters, ) {} class PrecomputedMultiscaleMeshSource extends WithParameters( - WithCredentialsProvider()(MultiscaleMeshSource), + WithSharedKvStoreContext(MultiscaleMeshSource), MultiscaleMeshSourceParameters, ) {} class PrecomputedSkeletonSource extends WithParameters( - WithCredentialsProvider()(SkeletonSource), + WithSharedKvStoreContext(SkeletonSource), SkeletonSourceParameters, ) { get skeletonVertexCoordinatesInVoxels() { @@ -151,20 +147,6 @@ class PrecomputedSkeletonSource extends WithParameters( } } -export function resolvePath(a: string, b: string) { - const outputParts = a.split("/"); - for (const part of b.split("/")) { - if (part === "..") { - if (outputParts.length !== 0) { - outputParts.length = outputParts.length - 1; - continue; - } - } - outputParts.push(part); - } - return outputParts.join("/"); -} - class ScaleInfo { key: string; encoding: VolumeChunkEncoding; @@ -325,12 +307,11 @@ export class PrecomputedMultiscaleVolumeChunkSource extends MultiscaleVolumeChun } constructor( - chunkManager: ChunkManager, - public credentialsProvider: SpecialProtocolCredentialsProvider, + public sharedKvStoreContext: SharedKvStoreContext, public url: string, public info: MultiscaleVolumeInfo, ) { - super(chunkManager); + super(sharedKvStoreContext.chunkManager); } getSources(volumeSourceOptions: VolumeSourceOptions) { @@ -380,10 +361,15 @@ export class PrecomputedMultiscaleVolumeChunkSource extends MultiscaleVolumeChun chunkSource: this.chunkManager.getChunkSource( PrecomputedVolumeChunkSource, { - credentialsProvider: this.credentialsProvider, + sharedKvStoreContext: this.sharedKvStoreContext, spec, parameters: { - url: resolvePath(this.url, scaleInfo.key), + url: kvstoreEnsureDirectoryPipelineUrl( + this.sharedKvStoreContext.kvStoreContext.resolveRelativePath( + this.url, + scaleInfo.key, + ), + ), encoding: scaleInfo.encoding, sharding: scaleInfo.sharding, }, @@ -400,44 +386,36 @@ export class PrecomputedMultiscaleVolumeChunkSource extends MultiscaleVolumeChun } const MultiscaleAnnotationSourceBase = WithParameters( - WithCredentialsProvider()( - MultiscaleAnnotationSource, - ), + WithSharedKvStoreContext(MultiscaleAnnotationSource), AnnotationSourceParameters, ); class PrecomputedAnnotationSpatialIndexSource extends WithParameters( - WithCredentialsProvider()( - AnnotationGeometryChunkSource, - ), + WithSharedKvStoreContext(AnnotationGeometryChunkSource), AnnotationSpatialIndexSourceParameters, ) {} interface PrecomputedAnnotationSourceOptions { metadata: AnnotationMetadata; parameters: AnnotationSourceParameters; - credentialsProvider: SpecialProtocolCredentialsProvider; + sharedKvStoreContext: SharedKvStoreContext; } export class PrecomputedAnnotationSource extends MultiscaleAnnotationSourceBase { declare key: any; metadata: AnnotationMetadata; - credentialsProvider: SpecialProtocolCredentialsProvider; declare OPTIONS: PrecomputedAnnotationSourceOptions; - constructor( - chunkManager: ChunkManager, - options: PrecomputedAnnotationSourceOptions, - ) { + constructor(options: PrecomputedAnnotationSourceOptions) { const { parameters } = options; - super(chunkManager, { + super(options.sharedKvStoreContext.chunkManager, { rank: parameters.rank, relationships: parameters.relationships.map((x) => x.name), properties: parameters.properties, + sharedKvStoreContext: options.sharedKvStoreContext, parameters, } as any); this.readonly = true; this.metadata = options.metadata; - this.credentialsProvider = options.credentialsProvider; } getSources(): SliceViewSingleResolutionSource[][] { @@ -448,7 +426,7 @@ export class PrecomputedAnnotationSource extends MultiscaleAnnotationSourceBase chunkSource: this.chunkManager.getChunkSource( PrecomputedAnnotationSpatialIndexSource, { - credentialsProvider: this.credentialsProvider, + sharedKvStoreContext: this.sharedKvStoreContext, parent: this, spec, parameters: spatialIndexLevel.parameters, @@ -462,14 +440,16 @@ export class PrecomputedAnnotationSource extends MultiscaleAnnotationSourceBase } function getLegacyMeshSource( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, parameters: MeshSourceParameters, ) { - return chunkManager.getChunkSource(PrecomputedMeshSource, { - parameters, - credentialsProvider, - }); + return sharedKvStoreContext.chunkManager.getChunkSource( + PrecomputedMeshSource, + { + parameters, + sharedKvStoreContext, + }, + ); } function parseTransform(data: any): mat4 { @@ -533,20 +513,20 @@ function parseMeshMetadata(data: any): ParsedMeshMetadata { } async function getMeshMetadata( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, + options: Partial, ): Promise { - let metadata: any; - try { - metadata = await getJsonMetadata(chunkManager, credentialsProvider, url); - } catch (e) { - if (isNotFoundError(e)) { - // If we fail to fetch the info file, assume it is the legacy - // single-resolution mesh format. - return { metadata: undefined }; - } - throw e; + const metadata = await getJsonMetadata( + sharedKvStoreContext, + url, + /*required=*/ false, + options, + ); + if (metadata === undefined) { + // If the info file is missing, assume it is the legacy + // single-resolution mesh format. + return { metadata: undefined }; } return parseMeshMetadata(metadata); } @@ -649,14 +629,15 @@ function parseSkeletonMetadata(data: any): ParsedSkeletonMetadata { } async function getSkeletonMetadata( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, + options: Partial, ): Promise { const metadata = await getJsonMetadata( - chunkManager, - credentialsProvider, + sharedKvStoreContext, url, + /*required=*/ true, + options, ); return parseSkeletonMetadata(metadata); } @@ -670,18 +651,18 @@ function getDefaultCoordinateSpace() { } async function getMeshSource( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, + options: Partial, ) { const { metadata, segmentPropertyMap } = await getMeshMetadata( - chunkManager, - credentialsProvider, + sharedKvStoreContext, url, + options, ); if (metadata === undefined) { return { - source: getLegacyMeshSource(chunkManager, credentialsProvider, { + source: getLegacyMeshSource(sharedKvStoreContext, { url, lod: 0, }), @@ -701,57 +682,71 @@ async function getMeshSource( ); } return { - source: chunkManager.getChunkSource(PrecomputedMultiscaleMeshSource, { - credentialsProvider, - parameters: { url, metadata }, - format: { - fragmentRelativeVertices: true, - vertexPositionFormat, + source: sharedKvStoreContext.chunkManager.getChunkSource( + PrecomputedMultiscaleMeshSource, + { + sharedKvStoreContext, + parameters: { url, metadata }, + format: { + fragmentRelativeVertices: true, + vertexPositionFormat, + }, }, - }), + ), transform: metadata.transform, segmentPropertyMap, }; } async function getSkeletonSource( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, + options: Partial, ) { const { metadata, segmentPropertyMap } = await getSkeletonMetadata( - chunkManager, - credentialsProvider, + sharedKvStoreContext, url, + options, ); return { - source: chunkManager.getChunkSource(PrecomputedSkeletonSource, { - credentialsProvider, - parameters: { - url, - metadata, + source: sharedKvStoreContext.chunkManager.getChunkSource( + PrecomputedSkeletonSource, + { + sharedKvStoreContext, + parameters: { + url, + metadata, + }, }, - }), + ), transform: metadata.transform, segmentPropertyMap, }; } -function getJsonMetadata( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, +export function getJsonMetadata( + sharedKvStoreContext: SharedKvStoreContext, url: string, + required: boolean, + options: Partial, ): Promise { - return chunkManager.memoize.getUncounted( + return sharedKvStoreContext.chunkManager.memoize.getAsync( { type: "precomputed:metadata", url, - credentialsProvider: getObjectId(credentialsProvider), }, - async () => { - return await fetchSpecialOk(credentialsProvider, `${url}/info`, {}).then( - (response) => response.json(), - ); + options, + async (options) => { + const infoUrl = pipelineUrlJoin(url, "info"); + using _span = new ProgressSpan(options.progressListener, { + message: `Reading neuroglancer_precomputed metadata from ${infoUrl}`, + }); + const response = await sharedKvStoreContext.kvStoreContext.read(infoUrl, { + ...options, + throwIfMissing: required, + }); + if (response === undefined) return undefined; + return await response.response.json(); }, ); } @@ -766,15 +761,14 @@ function getSubsourceToModelSubspaceTransform(info: MultiscaleVolumeInfo) { } async function getVolumeDataSource( - options: GetDataSourceOptions, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, metadata: any, + options: Partial, ): Promise { const info = parseMultiscaleVolumeInfo(metadata); const volume = new PrecomputedMultiscaleVolumeChunkSource( - options.chunkManager, - credentialsProvider, + sharedKvStoreContext, url, info, ); @@ -796,18 +790,19 @@ async function getVolumeDataSource( }, ]; if (info.segmentPropertyMap !== undefined) { - const mapUrl = resolvePath(url, info.segmentPropertyMap); - const metadata = await getJsonMetadata( - options.chunkManager, - credentialsProvider, - mapUrl, + const mapUrl = kvstoreEnsureDirectoryPipelineUrl( + sharedKvStoreContext.kvStoreContext.resolveRelativePath( + url, + info.segmentPropertyMap, + ), ); - const segmentPropertyMap = getSegmentPropertyMap( - options.chunkManager, - credentialsProvider, - metadata, + const metadata = await getJsonMetadata( + sharedKvStoreContext, mapUrl, + /*required=*/ true, + options, ); + const segmentPropertyMap = getSegmentPropertyMap(metadata); subsources.push({ id: "properties", default: true, @@ -815,11 +810,13 @@ async function getVolumeDataSource( }); } if (info.mesh !== undefined) { - const meshUrl = resolvePath(url, info.mesh); + const meshUrl = kvstoreEnsureDirectoryPipelineUrl( + sharedKvStoreContext.kvStoreContext.resolveRelativePath(url, info.mesh), + ); const { source: meshSource, transform } = await getMeshSource( - options.chunkManager, - credentialsProvider, + sharedKvStoreContext, meshUrl, + options, ); const subsourceToModelSubspaceTransform = getSubsourceToModelSubspaceTransform(info); @@ -836,11 +833,16 @@ async function getVolumeDataSource( }); } if (info.skeletons !== undefined) { - const skeletonsUrl = resolvePath(url, info.skeletons); + const skeletonsUrl = kvstoreEnsureDirectoryPipelineUrl( + sharedKvStoreContext.kvStoreContext.resolveRelativePath( + url, + info.skeletons, + ), + ); const { source: skeletonSource, transform } = await getSkeletonSource( - options.chunkManager, - credentialsProvider, + sharedKvStoreContext, skeletonsUrl, + options, ); const subsourceToModelSubspaceTransform = getSubsourceToModelSubspaceTransform(info); @@ -860,15 +862,15 @@ async function getVolumeDataSource( } async function getSkeletonsDataSource( - options: GetDataSourceOptions, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, + options: Partial, ): Promise { const { source: skeletons, transform, segmentPropertyMap, - } = await getSkeletonSource(options.chunkManager, credentialsProvider, url); + } = await getSkeletonSource(sharedKvStoreContext, url, options); const subsources: DataSubsourceEntry[] = [ { id: "default", @@ -878,18 +880,19 @@ async function getSkeletonsDataSource( }, ]; if (segmentPropertyMap !== undefined) { - const mapUrl = resolvePath(url, segmentPropertyMap); - const metadata = await getJsonMetadata( - options.chunkManager, - credentialsProvider, - mapUrl, + const mapUrl = kvstoreEnsureDirectoryPipelineUrl( + sharedKvStoreContext.kvStoreContext.resolveRelativePath( + url, + segmentPropertyMap, + ), ); - const segmentPropertyMapData = getSegmentPropertyMap( - options.chunkManager, - credentialsProvider, - metadata, + const metadata = await getJsonMetadata( + sharedKvStoreContext, mapUrl, + /*required=*/ true, + options, ); + const segmentPropertyMapData = getSegmentPropertyMap(metadata); subsources.push({ id: "properties", default: true, @@ -902,10 +905,17 @@ async function getSkeletonsDataSource( }; } -function parseKeyAndShardingSpec(url: string, obj: any) { +function parseKeyAndShardingSpec( + kvStoreContext: KvStoreContext, + url: string, + obj: any, +) { verifyObject(obj); + const relativePath = verifyObjectProperty(obj, "key", verifyString); return { - url: resolvePath(url, verifyObjectProperty(obj, "key", verifyString)), + url: kvstoreEnsureDirectoryPipelineUrl( + kvStoreContext.resolveRelativePath(url, relativePath), + ), sharding: verifyObjectProperty(obj, "sharding", parseShardingParameters), }; } @@ -921,6 +931,7 @@ class AnnotationMetadata { parameters: AnnotationSourceParameters; spatialIndices: AnnotationSpatialIndexLevelMetadata[]; constructor( + kvStoreContext: KvStoreContext, public url: string, metadata: any, ) { @@ -970,7 +981,7 @@ class AnnotationMetadata { "relationships", (relsObj) => parseArray(relsObj, (relObj) => { - const common = parseKeyAndShardingSpec(url, relObj); + const common = parseKeyAndShardingSpec(kvStoreContext, url, relObj); const name = verifyObjectProperty(relObj, "id", verifyString); return { ...common, name }; }), @@ -981,7 +992,7 @@ class AnnotationMetadata { parseAnnotationPropertySpecs, ), byId: verifyObjectProperty(metadata, "by_id", (obj) => - parseKeyAndShardingSpec(url, obj), + parseKeyAndShardingSpec(kvStoreContext, url, obj), ), }; this.spatialIndices = verifyObjectProperty( @@ -990,7 +1001,7 @@ class AnnotationMetadata { (spatialObj) => parseArray(spatialObj, (levelObj) => { const common: AnnotationSpatialIndexSourceParameters = - parseKeyAndShardingSpec(url, levelObj); + parseKeyAndShardingSpec(kvStoreContext, url, levelObj); const gridShape = verifyObjectProperty(levelObj, "grid_shape", (j) => parseFixedLengthArray(new Float32Array(rank), j, verifyPositiveInt), ); @@ -1038,13 +1049,16 @@ class AnnotationMetadata { } } -async function getAnnotationDataSource( - options: GetDataSourceOptions, - credentialsProvider: SpecialProtocolCredentialsProvider, +function getAnnotationDataSource( + sharedKvStoreContext: SharedKvStoreContext, url: string, metadata: any, -): Promise { - const info = new AnnotationMetadata(url, metadata); +): DataSource { + const info = new AnnotationMetadata( + sharedKvStoreContext.kvStoreContext, + url, + metadata, + ); const dataSource: DataSource = { modelTransform: makeIdentityTransform(info.coordinateSpace), subsources: [ @@ -1052,10 +1066,10 @@ async function getAnnotationDataSource( id: "default", default: true, subsource: { - annotation: options.chunkManager.getChunkSource( + annotation: sharedKvStoreContext.chunkManager.getChunkSource( PrecomputedAnnotationSource, { - credentialsProvider, + sharedKvStoreContext, metadata: info, parameters: info.parameters, }, @@ -1068,15 +1082,15 @@ async function getAnnotationDataSource( } async function getMeshDataSource( - options: GetDataSourceOptions, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, + options: Partial, ): Promise { const { source: mesh, transform, segmentPropertyMap, - } = await getMeshSource(options.chunkManager, credentialsProvider, url); + } = await getMeshSource(sharedKvStoreContext, url, options); const subsources: DataSubsourceEntry[] = [ { id: "default", @@ -1086,18 +1100,19 @@ async function getMeshDataSource( }, ]; if (segmentPropertyMap !== undefined) { - const mapUrl = resolvePath(url, segmentPropertyMap); - const metadata = await getJsonMetadata( - options.chunkManager, - credentialsProvider, - mapUrl, + const mapUrl = kvstoreEnsureDirectoryPipelineUrl( + sharedKvStoreContext.kvStoreContext.resolveRelativePath( + url, + segmentPropertyMap, + ), ); - const segmentPropertyMapData = getSegmentPropertyMap( - options.chunkManager, - credentialsProvider, - metadata, + const metadata = await getJsonMetadata( + sharedKvStoreContext, mapUrl, + /*required=*/ true, + options, ); + const segmentPropertyMapData = getSegmentPropertyMap(metadata); subsources.push({ id: "properties", default: true, @@ -1233,44 +1248,7 @@ function parseInlinePropertyMap(data: unknown): InlineSegmentPropertyMap { return normalizeInlineSegmentPropertyMap({ ids, properties }); } -export const PrecomputedIndexedSegmentPropertySource = WithParameters( - WithCredentialsProvider()( - IndexedSegmentPropertySource, - ), - IndexedSegmentPropertySourceParameters, -); - -// function parseIndexedPropertyMap(data: unknown): { -// sharding: ShardingParameters|undefined, -// properties: readonly Readonly[] -// } { -// verifyObject(data); -// const sharding = verifyObjectProperty(data, 'sharding', parseShardingParameters); -// const properties = verifyObjectProperty( -// data, 'properties', -// propertiesObj => parseArray(propertiesObj, (propertyObj): IndexedSegmentProperty => { -// const id = verifyObjectProperty(propertyObj, 'id', verifyString); -// const description = verifyOptionalObjectProperty(propertyObj, 'description', verifyString); -// const type = verifyObjectProperty(propertyObj, 'type', type => { -// if (type !== 'string') { -// throw new Error(`Invalid property type: ${JSON.stringify(type)}`); -// } -// return type; -// }); -// return {id, description, type}; -// })); -// return {sharding, properties}; -// } - -export function getSegmentPropertyMap( - chunkManager: Borrowed, - credentialsProvider: SpecialProtocolCredentialsProvider, - data: unknown, - url: string, -): SegmentPropertyMap { - chunkManager; - credentialsProvider; - url; +export function getSegmentPropertyMap(data: unknown): SegmentPropertyMap { try { const t = verifyObjectProperty(data, "@type", verifyString); if (t !== "neuroglancer_segment_properties") { @@ -1283,25 +1261,13 @@ export function getSegmentPropertyMap( "inline", parseInlinePropertyMap, ); - // const indexedProperties = verifyOptionalObjectProperty(data, 'indexed', indexedObj => { - // const {sharding, properties} = parseIndexedPropertyMap(indexedObj); - // return chunkManager.getChunkSource( - // PrecomputedIndexedSegmentPropertySource, - // {credentialsProvider, properties, parameters: {sharding, url}}); - // }); return new SegmentPropertyMap({ inlineProperties }); } catch (e) { throw new Error(`Error parsing segment property map: ${e.message}`); } } -async function getSegmentPropertyMapDataSource( - options: GetDataSourceOptions, - credentialsProvider: SpecialProtocolCredentialsProvider, - url: string, - metadata: unknown, -): Promise { - options; +function getSegmentPropertyMapDataSource(metadata: unknown): DataSource { return { modelTransform: makeIdentityTransform(emptyValidCoordinateSpace), subsources: [ @@ -1309,12 +1275,7 @@ async function getSegmentPropertyMapDataSource( id: "default", default: true, subsource: { - segmentPropertyMap: getSegmentPropertyMap( - options.chunkManager, - credentialsProvider, - metadata, - url, - ), + segmentPropertyMap: getSegmentPropertyMap(metadata), }, }, ], @@ -1340,54 +1301,47 @@ export function unparseProviderUrl(url: string, parameters: any) { return url; } -export class PrecomputedDataSource extends DataSourceProvider { - get description() { - return "Precomputed file-backed data source"; +export class PrecomputedDataSource implements KvStoreBasedDataSourceProvider { + get scheme() { + return "neuroglancer-precomputed"; } - - normalizeUrl(options: NormalizeUrlOptions): string { - const { url, parameters } = parseProviderUrl(options.providerUrl); - return ( - options.providerProtocol + "://" + unparseProviderUrl(url, parameters) - ); + get expectsDirectory() { + return true; } - - convertLegacyUrl(options: ConvertLegacyUrlOptions): string { - const { url, parameters } = parseProviderUrl(options.providerUrl); - if (options.type === "mesh") { - parameters.type = "mesh"; - } - return ( - options.providerProtocol + "://" + unparseProviderUrl(url, parameters) - ); + get description() { + return "Neuroglancer Precomputed data source"; } - get(options: GetDataSourceOptions): Promise { - const { url: providerUrl, parameters } = parseProviderUrl( - options.providerUrl, + get( + options: GetKvStoreBasedDataSourceOptions, + ): Promise { + const { authorityAndPath, query, fragment } = parseUrlSuffix( + options.url.suffix, ); - return options.chunkManager.memoize.getUncounted( - { type: "precomputed:get", providerUrl, parameters }, - async (): Promise => { - const { url, credentialsProvider } = parseSpecialUrl( - providerUrl, - options.credentialsManager, + if (query) { + throw new Error( + `Invalid URL ${JSON.stringify(options.url.url)}: query parameters not supported`, + ); + } + if (authorityAndPath) { + throw new Error( + `Invalid URL ${JSON.stringify(options.url.url)}: non-empty path not supported`, + ); + } + const parameters = parseQueryStringParameters(fragment ?? ""); + const url = kvstoreEnsureDirectoryPipelineUrl(options.kvStoreUrl); + return options.registry.chunkManager.memoize.getAsync( + { type: "precomputed:get", url, parameters }, + options, + async (progressOptions) => { + const { sharedKvStoreContext } = options.registry; + const metadata = await getJsonMetadata( + sharedKvStoreContext, + url, + /*required=*/ parameters.type !== "mesh", + progressOptions, ); - let metadata: any; - try { - metadata = await getJsonMetadata( - options.chunkManager, - credentialsProvider, - url, - ); - } catch (e) { - if (isNotFoundError(e)) { - if (parameters.type === "mesh") { - return await getMeshDataSource(options, credentialsProvider, url); - } - } - throw e; - } + const canonicalUrl = `${url}|${options.url.scheme}:`; verifyObject(metadata); const redirect = verifyOptionalObjectProperty( metadata, @@ -1395,52 +1349,74 @@ export class PrecomputedDataSource extends DataSourceProvider { verifyString, ); if (redirect !== undefined) { - throw new RedirectError(redirect); + return { canonicalUrl, targetUrl: redirect }; } const t = verifyOptionalObjectProperty(metadata, "@type", verifyString); + let dataSource: DataSource; switch (t) { case "neuroglancer_skeletons": - return await getSkeletonsDataSource( - options, - credentialsProvider, + dataSource = await getSkeletonsDataSource( + sharedKvStoreContext, url, + progressOptions, ); + break; case "neuroglancer_multilod_draco": case "neuroglancer_legacy_mesh": - return await getMeshDataSource(options, credentialsProvider, url); - case "neuroglancer_annotations_v1": - return await getAnnotationDataSource( - options, - credentialsProvider, + dataSource = await getMeshDataSource( + sharedKvStoreContext, url, - metadata, + progressOptions, ); - case "neuroglancer_segment_properties": - return await getSegmentPropertyMapDataSource( - options, - credentialsProvider, + break; + case "neuroglancer_annotations_v1": + dataSource = getAnnotationDataSource( + sharedKvStoreContext, url, metadata, ); + break; + case "neuroglancer_segment_properties": + dataSource = getSegmentPropertyMapDataSource(metadata); + break; case "neuroglancer_multiscale_volume": case undefined: - return await getVolumeDataSource( - options, - credentialsProvider, + dataSource = await getVolumeDataSource( + sharedKvStoreContext, url, metadata, + progressOptions, ); + break; default: throw new Error(`Invalid type: ${JSON.stringify(t)}`); } + dataSource.canonicalUrl = canonicalUrl; + return dataSource; }, ); } - completeUrl(options: CompleteUrlOptions) { - return completeHttpPath( - options.credentialsManager, - options.providerUrl, - options.abortSignal, - ); +} + +export class PrecomputedLegacyUrlDataSource extends KvStoreBasedDataSourceLegacyUrlAdapter { + constructor() { + super(new PrecomputedDataSource(), "precomputed"); } + + convertLegacyUrl(options: ConvertLegacyUrlOptions): string { + const { url, parameters } = parseProviderUrl(options.providerUrl); + if (options.type === "mesh") { + parameters.type = "mesh"; + } + return options.providerScheme + "://" + unparseProviderUrl(url, parameters); + } +} + +export function registerAutoDetect(registry: AutoDetectRegistry) { + registry.registerDirectoryFormat( + simpleFilePresenceAutoDetectDirectorySpec(new Set(["info"]), { + suffix: "neuroglancer-precomputed:", + description: "Neuroglancer Precomputed data source", + }), + ); } diff --git a/src/datasource/precomputed/register_default.ts b/src/datasource/precomputed/register_default.ts index ae5ed2d680..154b42bb80 100644 --- a/src/datasource/precomputed/register_default.ts +++ b/src/datasource/precomputed/register_default.ts @@ -14,7 +14,17 @@ * limitations under the License. */ -import { registerProvider } from "#src/datasource/default_provider.js"; -import { PrecomputedDataSource } from "#src/datasource/precomputed/frontend.js"; +import { + dataSourceAutoDetectRegistry, + registerKvStoreBasedDataProvider, + registerProvider, +} from "#src/datasource/default_provider.js"; +import { + PrecomputedDataSource, + PrecomputedLegacyUrlDataSource, + registerAutoDetect, +} from "#src/datasource/precomputed/frontend.js"; -registerProvider("precomputed", () => new PrecomputedDataSource()); +registerKvStoreBasedDataProvider(new PrecomputedDataSource()); +registerProvider(new PrecomputedLegacyUrlDataSource()); +registerAutoDetect(dataSourceAutoDetectRegistry); diff --git a/src/datasource/precomputed/sharded.ts b/src/datasource/precomputed/sharded.ts new file mode 100644 index 0000000000..55d9bc8b71 --- /dev/null +++ b/src/datasource/precomputed/sharded.ts @@ -0,0 +1,312 @@ +/** + * @license + * Copyright 2016 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ChunkManager } from "#src/chunk_manager/backend.js"; +import { SimpleAsyncCache } from "#src/chunk_manager/generic_file_source.js"; +import { + DataEncoding, + ShardingHashFunction, + type ShardingParameters, +} from "#src/datasource/precomputed/base.js"; +import { FileByteRangeHandle } from "#src/kvstore/byte_range/file_handle.js"; +import { GzipFileHandle } from "#src/kvstore/gzip/file_handle.js"; +import type { + ByteRange, + DriverReadOptions, + FileHandle, + KvStoreWithPath, + ReadableKvStore, + ReadResponse, + StatOptions, + StatResponse, +} from "#src/kvstore/index.js"; +import { KvStoreFileHandle, readFileHandle } from "#src/kvstore/index.js"; +import type { Owned } from "#src/util/disposable.js"; +import { RefCounted } from "#src/util/disposable.js"; +import { convertEndian64, Endianness } from "#src/util/endian.js"; +import { murmurHash3_x86_128Hash64Bits_Bigint } from "#src/util/hash.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; + +const shardingHashFunctions: Map< + ShardingHashFunction, + (input: bigint) => bigint +> = new Map([ + [ + ShardingHashFunction.MURMURHASH3_X86_128, + (input) => murmurHash3_x86_128Hash64Bits_Bigint(/*seed=*/ 0, input), + ], + [ShardingHashFunction.IDENTITY, (input) => input], +]); + +export interface ShardInfo { + shardPath: string; + offset: number; +} + +interface DecodedMinishardIndex { + data: BigUint64Array; + shardPath: string; +} + +type MinishardIndexCache = SimpleAsyncCache< + bigint, + DecodedMinishardIndex | undefined +>; + +function decodeFileHandle(handle: FileHandle, encoding: DataEncoding) { + if (encoding === DataEncoding.GZIP) { + handle = new GzipFileHandle(handle, "gzip"); + } + return handle; +} + +function makeMinishardIndexCache( + chunkManager: ChunkManager, + base: KvStoreWithPath, + sharding: ShardingParameters, +): MinishardIndexCache { + return new SimpleAsyncCache(chunkManager.addRef(), { + encodeKey: (key) => key.toString(), + get: async ( + shardAndMinishard: bigint, + progressOptions: Partial, + ) => { + const minishard = + shardAndMinishard & ((1n << BigInt(sharding.minishardBits)) - 1n); + const shard = + ((1n << BigInt(sharding.shardBits)) - 1n) & + (shardAndMinishard >> BigInt(sharding.minishardBits)); + const shardPath = + base.path + + shard.toString(16).padStart(Math.ceil(sharding.shardBits / 4), "0") + + ".shard"; + + const shardFileHandle = new KvStoreFileHandle(base.store, shardPath); + + // Retrive minishard index start/end offsets. + const shardIndexSize = BigInt(16) << BigInt(sharding.minishardBits); + + // Multiply minishard by 16. + const shardIndexStart = minishard << 4n; + const response = await readFileHandle(shardFileHandle, { + ...progressOptions, + byteRange: { offset: Number(shardIndexStart), length: 16 }, + strictByteRange: true, + }); + if (response === undefined) { + return { data: undefined, size: 0 }; + } + const shardIndexResponse = await response.response.arrayBuffer(); + const shardIndexDv = new DataView(shardIndexResponse); + let minishardStartOffset = shardIndexDv.getBigUint64( + 0, + /*littleEndian=*/ true, + ); + let minishardEndOffset = shardIndexDv.getBigUint64( + 8, + /*littleEndian=*/ true, + ); + if (minishardStartOffset === minishardEndOffset) { + return { data: undefined, size: 0 }; + } + // The start/end offsets in the shard index are relative to the end of the shard + // index. + minishardStartOffset += shardIndexSize; + minishardEndOffset += shardIndexSize; + + const minishardIndexBuffer = await ( + await readFileHandle( + decodeFileHandle( + new FileByteRangeHandle(shardFileHandle, { + offset: Number(minishardStartOffset), + length: Number(minishardEndOffset - minishardStartOffset), + }), + sharding.minishardIndexEncoding, + ), + { + ...progressOptions, + strictByteRange: true, + throwIfMissing: true, + }, + ) + ).response.arrayBuffer(); + if (minishardIndexBuffer.byteLength % 24 !== 0) { + throw new Error( + `Invalid minishard index length: ${minishardIndexBuffer.byteLength}`, + ); + } + const minishardIndex = new BigUint64Array(minishardIndexBuffer); + convertEndian64(minishardIndex, Endianness.LITTLE); + + const minishardIndexSize = minishardIndex.byteLength / 24; + let prevEntryKey = 0n; + // Offsets in the minishard index are relative to the end of the shard index. + let prevStart = shardIndexSize; + for (let i = 0; i < minishardIndexSize; ++i) { + const entryKey = prevEntryKey + minishardIndex[i]; + prevEntryKey = minishardIndex[i] = entryKey; + const start = prevStart + minishardIndex[minishardIndexSize + i]; + minishardIndex[minishardIndexSize + i] = start; + const size = minishardIndex[2 * minishardIndexSize + i]; + const end = start + size; + prevStart = end; + minishardIndex[2 * minishardIndexSize + i] = end; + } + return { + data: { data: minishardIndex, shardPath }, + size: minishardIndex.byteLength, + }; + }, + }); +} + +function findMinishardEntry( + minishardIndex: DecodedMinishardIndex, + key: bigint, +): ByteRange | undefined { + const minishardIndexData = minishardIndex.data; + const minishardIndexSize = minishardIndexData.length / 3; + for (let i = 0; i < minishardIndexSize; ++i) { + if (minishardIndexData[i] !== key) { + continue; + } + const startOffset = minishardIndexData[minishardIndexSize + i]; + const endOffset = minishardIndexData[2 * minishardIndexSize + i]; + + return { + offset: Number(startOffset), + length: Number(endOffset - startOffset), + }; + } + return undefined; +} + +export class ShardedKvStore + extends RefCounted + implements ReadableKvStore +{ + private minishardIndexCache: Owned; + + constructor( + chunkManager: ChunkManager, + private base: KvStoreWithPath, + private sharding: ShardingParameters, + ) { + super(); + this.minishardIndexCache = this.registerDisposer( + makeMinishardIndexCache(chunkManager, base, sharding), + ); + } + + getUrl(key: bigint): string { + return `chunk ${key} in ${this.base.store.getUrl(this.base.path)}`; + } + + async findKey( + key: bigint, + progressOptions: Partial, + ): Promise<{ minishardEntry: ByteRange; shardInfo: ShardInfo } | undefined> { + const { sharding } = this; + const hashFunction = shardingHashFunctions.get(sharding.hash)!; + const hashCode = hashFunction(key >> BigInt(sharding.preshiftBits)); + const shardAndMinishard = + hashCode & + ((1n << BigInt(sharding.minishardBits + sharding.shardBits)) - 1n); + const minishardIndex = await this.minishardIndexCache.get( + shardAndMinishard, + progressOptions, + ); + if (minishardIndex === undefined) return undefined; + const minishardEntry = findMinishardEntry(minishardIndex, key); + if (minishardEntry === undefined) return undefined; + return { + minishardEntry, + shardInfo: { + shardPath: minishardIndex.shardPath, + offset: minishardEntry.offset, + }, + }; + } + + async readWithShardInfo( + key: bigint, + options: DriverReadOptions, + ): Promise< + | { + response: ReadResponse; + shardInfo: ShardInfo; + } + | undefined + > { + const { sharding } = this; + const findResult = await this.findKey(key, options); + if (findResult === undefined) return undefined; + const { minishardEntry, shardInfo } = findResult; + return { + response: (await decodeFileHandle( + new FileByteRangeHandle( + new KvStoreFileHandle(this.base.store, shardInfo.shardPath), + minishardEntry, + ), + sharding.dataEncoding, + ).read(options))!, + shardInfo, + }; + } + + async stat( + key: bigint, + options: StatOptions, + ): Promise { + const findResult = await this.findKey(key, options); + if (findResult === undefined) return undefined; + const { sharding } = this; + if (sharding.dataEncoding !== DataEncoding.RAW) { + return { totalSize: undefined }; + } else { + return { totalSize: findResult.minishardEntry.length }; + } + } + + async read( + key: bigint, + options: DriverReadOptions, + ): Promise { + const response = await this.readWithShardInfo(key, options); + if (response === undefined) return undefined; + return response.response; + } + + get supportsOffsetReads() { + return this.sharding.dataEncoding === DataEncoding.RAW; + } + get supportsSuffixReads() { + return this.sharding.dataEncoding === DataEncoding.RAW; + } +} + +export function getShardedKvStoreIfApplicable( + chunkSource: RefCounted & { + chunkManager: ChunkManager; + }, + base: KvStoreWithPath, + sharding: ShardingParameters | undefined, +) { + if (sharding === undefined) return undefined; + return chunkSource.registerDisposer( + new ShardedKvStore(chunkSource.chunkManager, base, sharding), + ); +} diff --git a/src/datasource/python/backend.ts b/src/datasource/python/backend.ts index a947a19311..988e7533be 100644 --- a/src/datasource/python/backend.ts +++ b/src/datasource/python/backend.ts @@ -55,7 +55,7 @@ export class PythonVolumeChunkSource extends WithParameters( chunkDecoder = chunkDecoders.get(this.parameters["encoding"])!; encoding = VolumeChunkEncoding[this.parameters.encoding].toLowerCase(); - async download(chunk: VolumeChunk, abortSignal: AbortSignal) { + async download(chunk: VolumeChunk, signal: AbortSignal) { const { parameters } = this; let path = `../../neuroglancer/${this.encoding}/${parameters.key}/${parameters.scaleKey}`; { @@ -71,9 +71,9 @@ export class PythonVolumeChunkSource extends WithParameters( } } const response = await fetchOk(new URL(path, parameters.baseUrl).href, { - signal: abortSignal, + signal: signal, }); - await this.chunkDecoder(chunk, abortSignal, await response.arrayBuffer()); + await this.chunkDecoder(chunk, signal, await response.arrayBuffer()); } } @@ -105,13 +105,13 @@ export class PythonMeshSource extends WithParameters( return Promise.resolve(undefined); } - downloadFragment(chunk: FragmentChunk, abortSignal: AbortSignal) { + downloadFragment(chunk: FragmentChunk, signal: AbortSignal) { const { parameters } = this; const requestPath = `../../neuroglancer/mesh/${parameters.key}/${ chunk.manifestChunk!.objectId }`; return fetchOk(new URL(requestPath, parameters.baseUrl).href, { - signal: abortSignal, + signal: signal, }) .then((response) => response.arrayBuffer()) .then((response) => decodeFragmentChunk(chunk, response)); @@ -123,11 +123,11 @@ export class PythonSkeletonSource extends WithParameters( SkeletonSource, SkeletonSourceParameters, ) { - download(chunk: SkeletonChunk, abortSignal: AbortSignal) { + download(chunk: SkeletonChunk, signal: AbortSignal) { const { parameters } = this; const requestPath = `../../neuroglancer/skeleton/${parameters.key}/${chunk.objectId}`; return fetchOk(new URL(requestPath, parameters.baseUrl).href, { - signal: abortSignal, + signal: signal, }) .then((response) => response.arrayBuffer()) .then((response) => diff --git a/src/datasource/python/frontend.ts b/src/datasource/python/frontend.ts index dbd188502b..a417a68e30 100644 --- a/src/datasource/python/frontend.ts +++ b/src/datasource/python/frontend.ts @@ -65,7 +65,6 @@ import { VolumeChunkSource, } from "#src/sliceview/volume/frontend.js"; import { transposeNestedArrays } from "#src/util/array.js"; -import { Borrowed, Owned } from "#src/util/disposable.js"; import { fetchOk } from "#src/util/http_request.js"; import { parseFixedLengthArray, @@ -94,20 +93,18 @@ function WithPythonDataSource< >, >(Base: TBase) { type Options = InstanceType["OPTIONS"] & { - dataSource: Borrowed; + dataSource: PythonDataSource; generation: number; }; class C extends Base { declare OPTIONS: Options; - dataSource: Owned; + dataSource: PythonDataSource; generation: number; declare parameters: PythonSourceParameters; constructor(...args: any[]) { super(...args); const options: Options = args[1]; - const dataSource = (this.dataSource = this.registerDisposer( - options.dataSource.addRef(), - )); + const dataSource = (this.dataSource = options.dataSource); this.generation = options.generation; const key = options.parameters.key; dataSource.registerSource(key, this); @@ -238,7 +235,7 @@ export class PythonMultiscaleVolumeChunkSource extends MultiscaleVolumeChunkSour // TODO(jbms): Properly handle reference counting of `dataSource`. constructor( - public dataSource: Borrowed, + public dataSource: PythonDataSource, chunkManager: ChunkManager, public key: string, public response: any, @@ -480,17 +477,19 @@ function getVolumeDataSource( options: GetDataSourceOptions, key: string, ) { - return options.chunkManager.memoize.getUncounted( + return options.registry.chunkManager.memoize.getAsync( { type: "python:VolumeDataSource", key }, - async () => { + options, + async (progressOptions) => { const response = await ( await fetchOk( new URL(`../../neuroglancer/info/${key}`, window.location.href).href, + progressOptions, ) ).json(); const volume = new PythonMultiscaleVolumeChunkSource( dataSourceProvider, - options.chunkManager, + options.registry.chunkManager, key, response, ); @@ -527,14 +526,17 @@ function getVolumeDataSource( default: true, subsourceToModelSubspaceTransform, subsource: { - mesh: options.chunkManager.getChunkSource(PythonMeshSource, { - dataSource: dataSourceProvider, - generation: volume.generation, - parameters: { - baseUrl: window.location.href, - key: key, + mesh: options.registry.chunkManager.getChunkSource( + PythonMeshSource, + { + dataSource: dataSourceProvider, + generation: volume.generation, + parameters: { + baseUrl: window.location.href, + key: key, + }, }, - }), + ), }, }); } @@ -548,15 +550,17 @@ function getSkeletonDataSource( options: GetDataSourceOptions, key: string, ) { - return options.chunkManager.memoize.getUncounted( + return options.registry.chunkManager.memoize.getAsync( { type: "python:SkeletonDataSource", key }, - async () => { + options, + async (progressOptions) => { const response = await ( await fetchOk( new URL( `../../neuroglancer/skeletoninfo/${key}`, window.location.href, ).href, + progressOptions, ) ).json(); const { baseModelSpace, subsourceToModelTransform } = @@ -567,7 +571,7 @@ function getSkeletonDataSource( (x) => verifyObjectAsMap(x, parseVertexAttributeInfo), ); const generation = verifyObjectProperty(response, "generation", (x) => x); - const skeletonSource = options.chunkManager.getChunkSource( + const skeletonSource = options.registry.chunkManager.getChunkSource( PythonSkeletonSource, { dataSource: dataSourceProvider, @@ -597,10 +601,14 @@ function getSkeletonDataSource( ); } -export class PythonDataSource extends DataSourceProvider { +export class PythonDataSource implements DataSourceProvider { private sources = new Map>(); sourceGenerations = new Map(); + get scheme() { + return "python"; + } + registerSource(key: string, source: PythonChunkSource) { let existingSet = this.sources.get(key); if (existingSet === undefined) { diff --git a/src/datasource/python/register_default.ts b/src/datasource/python/register_default.ts new file mode 100644 index 0000000000..d70ae3dfdb --- /dev/null +++ b/src/datasource/python/register_default.ts @@ -0,0 +1,20 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { registerProvider } from "#src/datasource/default_provider.js"; +import { PythonDataSource } from "#src/datasource/python/frontend.js"; + +registerProvider(new PythonDataSource()); diff --git a/src/datasource/render/backend.ts b/src/datasource/render/backend.ts index 97b5176cd4..bd78682201 100644 --- a/src/datasource/render/backend.ts +++ b/src/datasource/render/backend.ts @@ -31,15 +31,11 @@ import { registerSharedObject } from "#src/worker_rpc.js"; const chunkDecoders = new Map(); chunkDecoders.set( "jpg", - async ( - chunk: VolumeChunk, - abortSignal: AbortSignal, - response: ArrayBuffer, - ) => { + async (chunk: VolumeChunk, signal: AbortSignal, response: ArrayBuffer) => { const chunkDataSize = chunk.chunkDataSize!; const { uint8Array: decoded } = await requestAsyncComputation( decodeJpeg, - abortSignal, + signal, [response], new Uint8Array(response), undefined, @@ -48,11 +44,11 @@ chunkDecoders.set( 3, true, ); - await postProcessRawData(chunk, abortSignal, decoded); + await postProcessRawData(chunk, signal, decoded); }, ); -chunkDecoders.set("raw16", (chunk, abortSignal, response) => { - return decodeRawChunk(chunk, abortSignal, response, Endianness.BIG); +chunkDecoders.set("raw16", (chunk, signal, response) => { + return decodeRawChunk(chunk, signal, response, Endianness.BIG); }); @registerSharedObject() @@ -91,7 +87,7 @@ export class TileChunkSource extends WithParameters( return query_params.join("&"); })(); - async download(chunk: VolumeChunk, abortSignal: AbortSignal) { + async download(chunk: VolumeChunk, signal: AbortSignal) { const { parameters } = this; const { chunkGridPosition } = chunk; @@ -122,8 +118,8 @@ export class TileChunkSource extends WithParameters( const path = `/render-ws/v1/owner/${parameters.owner}/project/${parameters.project}/stack/${parameters.stack}/z/${chunkPosition[2]}/box/${chunkPosition[0]},${chunkPosition[1]},${xTileSize},${yTileSize},${scale}/${imageMethod}`; const response = await fetchOk( `${parameters.baseUrl}${path}?${this.queryString}`, - { signal: abortSignal }, + { signal: signal }, ); - await this.chunkDecoder(chunk, abortSignal, await response.arrayBuffer()); + await this.chunkDecoder(chunk, signal, await response.arrayBuffer()); } } diff --git a/src/datasource/render/frontend.ts b/src/datasource/render/frontend.ts index 6fd9b0225d..46d700430e 100644 --- a/src/datasource/render/frontend.ts +++ b/src/datasource/render/frontend.ts @@ -32,8 +32,8 @@ import type { CompletionResult, DataSource, GetDataSourceOptions, + DataSourceProvider, } from "#src/datasource/index.js"; -import { DataSourceProvider } from "#src/datasource/index.js"; import { TileChunkSourceParameters } from "#src/datasource/render/base.js"; import type { SliceViewSingleResolutionSource } from "#src/sliceview/frontend.js"; import type { VolumeSourceOptions } from "#src/sliceview/volume/base.js"; @@ -64,6 +64,7 @@ import { verifyOptionalString, verifyString, } from "#src/util/json.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; const VALID_ENCODINGS = new Set(["jpg", "raw16"]); @@ -505,11 +506,13 @@ export function getOwnerInfo( chunkManager: ChunkManager, hostname: string, owner: string, + options: Partial, ): Promise { - return chunkManager.memoize.getUncounted( + return chunkManager.memoize.getAsync( { type: "render:getOwnerInfo", hostname, owner }, - () => - fetchOk(`${hostname}/render-ws/v1/owner/${owner}/stacks`) + options, + (progressOptions) => + fetchOk(`${hostname}/render-ws/v1/owner/${owner}/stacks`, progressOptions) .then((response) => response.json()) .then(parseOwnerInfo), ); @@ -519,7 +522,11 @@ const pathPattern = /^([^/?]+)(?:\/([^/?]+))?(?:\/([^/?]+))(?:\/([^/?]*))?(?:\?(.*))?$/; const urlPattern = /^((?:(?:(?:http|https):\/\/[^,/]+)[^/?]))\/(.*)$/; -function getVolume(chunkManager: ChunkManager, datasourcePath: string) { +function getVolume( + chunkManager: ChunkManager, + datasourcePath: string, + options: Partial, +) { let hostname: string; let path: string; { @@ -543,10 +550,16 @@ function getVolume(chunkManager: ChunkManager, datasourcePath: string) { const parameters = parseQueryStringParameters(match[5] || ""); - return chunkManager.memoize.getUncounted( + return chunkManager.memoize.getAsync( { type: "render:MultiscaleVolumeChunkSource", hostname, path }, - async () => { - const ownerInfo = await getOwnerInfo(chunkManager, hostname, owner); + options, + async (progressOptions) => { + const ownerInfo = await getOwnerInfo( + chunkManager, + hostname, + owner, + progressOptions, + ); const volume = new RenderMultiscaleVolumeChunkSource( chunkManager, hostname, @@ -599,6 +612,7 @@ export async function stackAndProjectCompleter( chunkManager: ChunkManager, hostname: string, path: string, + options: Partial, ): Promise { const stackMatch = path.match( /^(?:([^/]+)(?:\/([^/]*))?(?:\/([^/]*))?(\/.*?)?)?$/, @@ -613,7 +627,12 @@ export async function stackAndProjectCompleter( } if (stackMatch[3] === undefined) { const projectPrefix = stackMatch[2] || ""; - const ownerInfo = await getOwnerInfo(chunkManager, hostname, stackMatch[1]); + const ownerInfo = await getOwnerInfo( + chunkManager, + hostname, + stackMatch[1], + options, + ); const completions = getPrefixMatchesWithDescriptions( projectPrefix, ownerInfo.projects, @@ -624,7 +643,12 @@ export async function stackAndProjectCompleter( } if (stackMatch[4] === undefined) { const stackPrefix = stackMatch[3] || ""; - const ownerInfo = await getOwnerInfo(chunkManager, hostname, stackMatch[1]); + const ownerInfo = await getOwnerInfo( + chunkManager, + hostname, + stackMatch[1], + options, + ); const projectInfo = ownerInfo.projects.get(stackMatch[2]); if (projectInfo === undefined) { throw null; @@ -643,7 +667,12 @@ export async function stackAndProjectCompleter( }; } const channelPrefix = stackMatch[4].substr(1) || ""; - const ownerInfo = await getOwnerInfo(chunkManager, hostname, stackMatch[1]); + const ownerInfo = await getOwnerInfo( + chunkManager, + hostname, + stackMatch[1], + options, + ); const projectInfo = ownerInfo.projects.get(stackMatch[2]); if (projectInfo === undefined) { throw null; @@ -673,6 +702,7 @@ export async function stackAndProjectCompleter( export async function volumeCompleter( url: string, chunkManager: ChunkManager, + options: Partial, ): Promise { const match = url.match(urlPattern); if (match === null) { @@ -686,18 +716,30 @@ export async function volumeCompleter( chunkManager, hostname, path, + options, ); return applyCompletionOffset(match![1].length + 1, completions); } -export class RenderDataSource extends DataSourceProvider { +export class RenderDataSource implements DataSourceProvider { + get scheme() { + return "render"; + } get description() { return "Render"; } get(options: GetDataSourceOptions): Promise { - return getVolume(options.chunkManager, options.providerUrl); + return getVolume( + options.registry.chunkManager, + options.providerUrl, + options, + ); } completeUrl(options: CompleteUrlOptions) { - return volumeCompleter(options.providerUrl, options.chunkManager); + return volumeCompleter( + options.providerUrl, + options.registry.chunkManager, + options, + ); } } diff --git a/src/datasource/render/register_default.ts b/src/datasource/render/register_default.ts index ae5a9241a0..3c4f1c8d27 100644 --- a/src/datasource/render/register_default.ts +++ b/src/datasource/render/register_default.ts @@ -17,4 +17,4 @@ import { registerProvider } from "#src/datasource/default_provider.js"; import { RenderDataSource } from "#src/datasource/render/frontend.js"; -registerProvider("render", () => new RenderDataSource()); +registerProvider(new RenderDataSource()); diff --git a/src/datasource/state_share.ts b/src/datasource/state_share.ts index 376ad438cd..27f3184872 100644 --- a/src/datasource/state_share.ts +++ b/src/datasource/state_share.ts @@ -1,10 +1,6 @@ -import { defaultCredentialsManager } from "#src/credentials_provider/default_manager.js"; +import { HttpKvStore } from "#src/kvstore/http/index.js"; import { StatusMessage } from "#src/status.js"; import { RefCounted } from "#src/util/disposable.js"; -import { - fetchSpecialOk, - parseSpecialUrl, -} from "#src/util/special_protocol_request.js"; import type { Viewer } from "#src/viewer.js"; import { makeIcon } from "#src/widget/icon.js"; @@ -77,24 +73,32 @@ export class StateShare extends RefCounted { const selectedStateServer = this.selectStateServerElement ? this.selectStateServerElement.value : Object.values(STATE_SERVERS)[0].url; - const protocol = new URL(selectedStateServer).protocol; - const { url: parsedUrl, credentialsProvider } = parseSpecialUrl( - selectedStateServer, - defaultCredentialsManager, - ); + + const { store, path } = + viewer.dataSourceProvider.sharedKvStoreContext.kvStoreContext.getKvStore( + selectedStateServer, + ); + + if (!(store instanceof HttpKvStore)) { + throw new Error( + `Non-HTTP protocol not supported: ${selectedStateServer}`, + ); + } StatusMessage.forPromise( - fetchSpecialOk(credentialsProvider, parsedUrl, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(viewer.state.toJSON()), - }) + store + .fetchOkImpl(store.baseUrl + path, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(viewer.state.toJSON()), + }) .then((response) => response.json()) .then((res) => { const stateUrlProtcol = new URL(res).protocol; const stateUrlWithoutProtocol = res.substring( stateUrlProtcol.length, ); + const protocol = new URL(selectedStateServer).protocol; const link = `${window.location.origin}/#!${protocol}${stateUrlWithoutProtocol}`; navigator.clipboard.writeText(link).then(() => { StatusMessage.showTemporaryMessage( diff --git a/src/datasource/vtk/backend.ts b/src/datasource/vtk/backend.ts index b2f1f3f586..a3fd399fdc 100644 --- a/src/datasource/vtk/backend.ts +++ b/src/datasource/vtk/backend.ts @@ -16,19 +16,28 @@ import { requestAsyncComputation } from "#src/async_computation/request.js"; import { parseVTKFromArrayBuffer } from "#src/async_computation/vtk_mesh_request.js"; -import { GenericSharedDataSource } from "#src/chunk_manager/generic_file_source.js"; +import { getCachedDecodedUrl } from "#src/chunk_manager/generic_file_source.js"; +import type { ReadResponse } from "#src/kvstore/index.js"; import type { SingleMesh } from "#src/single_mesh/backend.js"; import { registerSingleMeshFactory } from "#src/single_mesh/backend.js"; import { DataType } from "#src/util/data_type.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; /** - * This needs to be a global function, because it identifies the instance of GenericSharedDataSource + * This needs to be a global function, because it identifies the instance of SimpleAsyncCache * to use. */ -function parse(buffer: ArrayBuffer, abortSignal: AbortSignal) { +async function parse( + readResponse: ReadResponse | undefined, + progressOptions: Partial, +) { + if (readResponse === undefined) { + throw new Error("Not found"); + } + const buffer = await readResponse.response.arrayBuffer(); return requestAsyncComputation( parseVTKFromArrayBuffer, - abortSignal, + progressOptions.signal, [buffer], buffer, ); @@ -36,33 +45,31 @@ function parse(buffer: ArrayBuffer, abortSignal: AbortSignal) { registerSingleMeshFactory("vtk", { description: "VTK", - getMesh: (chunkManager, credentialsProvider, url, getPriority, abortSignal) => - GenericSharedDataSource.getUrl( - chunkManager, - credentialsProvider, - parse, + getMesh: async (sharedKvStoreContext, url, options) => { + const mesh = await getCachedDecodedUrl( + sharedKvStoreContext, url, - getPriority, - abortSignal, - ).then((mesh) => { - const result: SingleMesh = { - info: { - numTriangles: mesh.numTriangles, - numVertices: mesh.numVertices, - vertexAttributes: [], - }, - indices: mesh.indices, - vertexPositions: mesh.vertexPositions, + parse, + options, + ); + const result: SingleMesh = { + info: { + numTriangles: mesh.numTriangles, + numVertices: mesh.numVertices, vertexAttributes: [], - }; - for (const attribute of mesh.vertexAttributes) { - result.info.vertexAttributes.push({ - name: attribute.name, - dataType: DataType.FLOAT32, - numComponents: attribute.numComponents, - }); - result.vertexAttributes.push(attribute.data); - } - return result; - }), + }, + indices: mesh.indices, + vertexPositions: mesh.vertexPositions, + vertexAttributes: [], + }; + for (const attribute of mesh.vertexAttributes) { + result.info.vertexAttributes.push({ + name: attribute.name, + dataType: DataType.FLOAT32, + numComponents: attribute.numComponents, + }); + result.vertexAttributes.push(attribute.data); + } + return result; + }, }); diff --git a/src/datasource/vtk/frontend.ts b/src/datasource/vtk/frontend.ts index ce59208e64..824773c572 100644 --- a/src/datasource/vtk/frontend.ts +++ b/src/datasource/vtk/frontend.ts @@ -19,24 +19,30 @@ import { makeIdentityTransform, } from "#src/coordinate_transform.js"; import type { - CompleteUrlOptions, DataSource, - GetDataSourceOptions, + GetKvStoreBasedDataSourceOptions, + KvStoreBasedDataSourceProvider, } from "#src/datasource/index.js"; -import { DataSourceProvider } from "#src/datasource/index.js"; +import { ensureEmptyUrlSuffix } from "#src/kvstore/url.js"; import { getSingleMeshSource } from "#src/single_mesh/frontend.js"; -import { completeHttpPath } from "#src/util/http_path_completion.js"; -export class VtkDataSource extends DataSourceProvider { +export class VtkDataSource implements KvStoreBasedDataSourceProvider { + get scheme() { + return "vtk"; + } get description() { - return "VTK mesh file"; + return "VTK mesh"; + } + get singleFile() { + return true; } - async get(options: GetDataSourceOptions): Promise { + async get(options: GetKvStoreBasedDataSourceOptions): Promise { + ensureEmptyUrlSuffix(options.url); const meshSource = await getSingleMeshSource( - options.chunkManager, - options.credentialsManager, - options.url, + options.registry.sharedKvStoreContext, + options.kvStoreUrl, + options, ); const modelSpace = makeCoordinateSpace({ rank: 3, @@ -56,11 +62,4 @@ export class VtkDataSource extends DataSourceProvider { }; return dataSource; } - completeUrl(options: CompleteUrlOptions) { - return completeHttpPath( - options.credentialsManager, - options.providerUrl, - options.abortSignal, - ); - } } diff --git a/src/datasource/vtk/register_default.ts b/src/datasource/vtk/register_default.ts index 3c3b4e571f..e960750f7b 100644 --- a/src/datasource/vtk/register_default.ts +++ b/src/datasource/vtk/register_default.ts @@ -14,7 +14,13 @@ * limitations under the License. */ -import { registerProvider } from "#src/datasource/default_provider.js"; +import { + registerKvStoreBasedDataProvider, + registerProvider, +} from "#src/datasource/default_provider.js"; +import { KvStoreBasedDataSourceLegacyUrlAdapter } from "#src/datasource/index.js"; import { VtkDataSource } from "#src/datasource/vtk/frontend.js"; -registerProvider("vtk", () => new VtkDataSource()); +const provider = new VtkDataSource(); +registerKvStoreBasedDataProvider(provider); +registerProvider(new KvStoreBasedDataSourceLegacyUrlAdapter(provider)); diff --git a/src/datasource/zarr/backend.ts b/src/datasource/zarr/backend.ts index bbcbdda792..7370f7af0e 100644 --- a/src/datasource/zarr/backend.ts +++ b/src/datasource/zarr/backend.ts @@ -20,7 +20,6 @@ import "#src/datasource/zarr/codec/bytes/decode.js"; import "#src/datasource/zarr/codec/crc32c/decode.js"; import { WithParameters } from "#src/chunk_manager/backend.js"; -import { WithSharedCredentialsProviderCounterpart } from "#src/credentials_provider/shared_counterpart.js"; import { VolumeChunkSourceParameters } from "#src/datasource/zarr/base.js"; import { applySharding, @@ -30,30 +29,24 @@ import "#src/datasource/zarr/codec/gzip/decode.js"; import "#src/datasource/zarr/codec/sharding_indexed/decode.js"; import "#src/datasource/zarr/codec/transpose/decode.js"; import { ChunkKeyEncoding } from "#src/datasource/zarr/metadata/index.js"; -import { getSpecialProtocolKvStore } from "#src/kvstore/special/index.js"; +import { WithSharedKvStoreContextCounterpart } from "#src/kvstore/backend.js"; import { postProcessRawData } from "#src/sliceview/backend_chunk_decoders/postprocess.js"; import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; import { VolumeChunkSource } from "#src/sliceview/volume/backend.js"; -import type { SpecialProtocolCredentials } from "#src/util/special_protocol_request.js"; import { registerSharedObject } from "#src/worker_rpc.js"; @registerSharedObject() export class ZarrVolumeChunkSource extends WithParameters( - WithSharedCredentialsProviderCounterpart()( - VolumeChunkSource, - ), + WithSharedKvStoreContextCounterpart(VolumeChunkSource), VolumeChunkSourceParameters, ) { private chunkKvStore = applySharding( this.chunkManager, this.parameters.metadata.codecs, - getSpecialProtocolKvStore( - this.credentialsProvider, - this.parameters.url + "/", - ), + this.sharedKvStoreContext.kvStoreContext.getKvStore(this.parameters.url), ); - async download(chunk: VolumeChunk, abortSignal: AbortSignal) { + async download(chunk: VolumeChunk, signal: AbortSignal) { chunk.chunkDataSize = this.spec.chunkDataSize; const { parameters } = this; const { chunkGridPosition } = chunk; @@ -93,15 +86,15 @@ export class ZarrVolumeChunkSource extends WithParameters( const { chunkKvStore } = this; const response = await chunkKvStore.kvStore.read( chunkKvStore.getChunkKey(chunkGridPosition, baseKey), - { abortSignal }, + { signal }, ); if (response !== undefined) { const decoded = await decodeArray( chunkKvStore.decodeCodecs, - response.data, - abortSignal, + new Uint8Array(await response.response.arrayBuffer()), + signal, ); - await postProcessRawData(chunk, abortSignal, decoded); + await postProcessRawData(chunk, signal, decoded); } } } diff --git a/src/datasource/zarr/codec/blosc/decode.ts b/src/datasource/zarr/codec/blosc/decode.ts index a6c47d725a..6931f9c8dc 100644 --- a/src/datasource/zarr/codec/blosc/decode.ts +++ b/src/datasource/zarr/codec/blosc/decode.ts @@ -23,11 +23,11 @@ import { CodecKind } from "#src/datasource/zarr/codec/index.js"; registerCodec({ name: "blosc", kind: CodecKind.bytesToBytes, - decode(configuration: Configuration, encoded, abortSignal: AbortSignal) { + decode(configuration: Configuration, encoded, signal: AbortSignal) { configuration; return requestAsyncComputation( decodeBlosc, - abortSignal, + signal, [encoded.buffer], encoded, ); diff --git a/src/datasource/zarr/codec/bytes/decode.ts b/src/datasource/zarr/codec/bytes/decode.ts index e38c163a0f..d94353e097 100644 --- a/src/datasource/zarr/codec/bytes/decode.ts +++ b/src/datasource/zarr/codec/bytes/decode.ts @@ -28,9 +28,9 @@ registerCodec({ configuration: Configuration, decodedArrayInfo: CodecArrayInfo, encoded, - abortSignal: AbortSignal, + signal: AbortSignal, ) { - abortSignal; + signal; const { dataType, chunkShape } = decodedArrayInfo; const numElements = chunkShape.reduce((a, b) => a * b, 1); const bytesPerElement = DATA_TYPE_BYTES[dataType]; diff --git a/src/datasource/zarr/codec/crc32c/decode.ts b/src/datasource/zarr/codec/crc32c/decode.ts index f225f18cee..6cf60c8c01 100644 --- a/src/datasource/zarr/codec/crc32c/decode.ts +++ b/src/datasource/zarr/codec/crc32c/decode.ts @@ -23,13 +23,9 @@ const checksumSize = 4; registerCodec({ name: "crc32c", kind: CodecKind.bytesToBytes, - async decode( - configuration: Configuration, - encoded, - abortSignal: AbortSignal, - ) { + async decode(configuration: Configuration, encoded, signal: AbortSignal) { configuration; - abortSignal; + signal; if (encoded.length < checksumSize) { throw new Error( `Expected buffer of size at least ${checksumSize} bytes but received: ${encoded.length} bytes`, diff --git a/src/datasource/zarr/codec/decode.ts b/src/datasource/zarr/codec/decode.ts index dcc4726e0b..2cc3ea9184 100644 --- a/src/datasource/zarr/codec/decode.ts +++ b/src/datasource/zarr/codec/decode.ts @@ -20,7 +20,7 @@ import type { CodecChainSpec, } from "#src/datasource/zarr/codec/index.js"; import { CodecKind } from "#src/datasource/zarr/codec/index.js"; -import type { ReadableKvStore } from "#src/kvstore/index.js"; +import type { KvStoreWithPath, ReadableKvStore } from "#src/kvstore/index.js"; import type { RefCounted } from "#src/util/disposable.js"; export interface Codec { @@ -34,7 +34,7 @@ export interface ArrayToArrayCodec extends Codec { configuration: Configuration, decodedArrayInfo: CodecArrayInfo, encoded: ArrayBufferView, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise>; } @@ -44,7 +44,7 @@ export interface ArrayToBytesCodec extends Codec { configuration: Configuration, decodedArrayInfo: CodecArrayInfo, encoded: Uint8Array, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise>; } @@ -67,7 +67,7 @@ export interface BytesToBytesCodec extends Codec { decode( configuration: Configuration, encoded: Uint8Array, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise>; } @@ -95,7 +95,7 @@ export function registerCodec( export async function decodeArray( codecs: CodecChainSpec, encoded: Uint8Array, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise> { const bytesToBytes = codecs[CodecKind.bytesToBytes]; for (let i = bytesToBytes.length; i--; ) { @@ -104,7 +104,7 @@ export async function decodeArray( if (impl === undefined) { throw new Error(`Unsupported codec: ${JSON.stringify(codec.name)}`); } - encoded = await impl.decode(codec.configuration, encoded, abortSignal); + encoded = await impl.decode(codec.configuration, encoded, signal); } let decoded: ArrayBufferView; @@ -118,7 +118,7 @@ export async function decodeArray( codec.configuration, codecs.arrayInfo[codecs.arrayInfo.length - 1], encoded, - abortSignal, + signal, ); } @@ -133,7 +133,7 @@ export async function decodeArray( codec.configuration, codecs.arrayInfo[i], decoded, - abortSignal, + signal, ); } @@ -143,7 +143,7 @@ export async function decodeArray( export function applySharding( chunkManager: ChunkManager, codecs: CodecChainSpec, - baseKvStore: ReadableKvStore, + baseKvStore: KvStoreWithPath, ): { kvStore: ReadableKvStore; getChunkKey: ( @@ -152,7 +152,7 @@ export function applySharding( ) => unknown; decodeCodecs: CodecChainSpec; } { - let kvStore: ReadableKvStore = baseKvStore; + let kvStore: ReadableKvStore = baseKvStore.store; let curCodecs = codecs; while (true) { const { shardingInfo } = curCodecs; @@ -172,11 +172,13 @@ export function applySharding( const decodeCodecs = curCodecs; + const pathPrefix = baseKvStore.path; + function getChunkKey( chunkGridPosition: ArrayLike, baseKey: string, ): unknown { - let key: unknown = baseKey; + let key: unknown = pathPrefix + baseKey; const rank = chunkGridPosition.length; let curCodecs = codecs; while (curCodecs.shardingInfo !== undefined) { diff --git a/src/datasource/zarr/codec/gzip/decode.ts b/src/datasource/zarr/codec/gzip/decode.ts index 75a713416e..f2f9263df7 100644 --- a/src/datasource/zarr/codec/gzip/decode.ts +++ b/src/datasource/zarr/codec/gzip/decode.ts @@ -26,14 +26,10 @@ for (const [name, compressionFormat] of [ registerCodec({ name, kind: CodecKind.bytesToBytes, - async decode( - configuration: Configuration, - encoded, - abortSignal: AbortSignal, - ) { + async decode(configuration: Configuration, encoded, signal: AbortSignal) { configuration; return new Uint8Array( - await decodeGzip(encoded, compressionFormat, abortSignal), + await decodeGzip(encoded, compressionFormat, signal), ); }, }); diff --git a/src/datasource/zarr/codec/sharding_indexed/decode.ts b/src/datasource/zarr/codec/sharding_indexed/decode.ts index 68ad0adf39..dbcb08bdaf 100644 --- a/src/datasource/zarr/codec/sharding_indexed/decode.ts +++ b/src/datasource/zarr/codec/sharding_indexed/decode.ts @@ -21,27 +21,80 @@ import { registerCodec, } from "#src/datasource/zarr/codec/decode.js"; import { CodecKind } from "#src/datasource/zarr/codec/index.js"; -import type { Configuration } from "#src/datasource/zarr/codec/sharding_indexed/resolve.js"; +import type { + Configuration, + IndexConfiguration, +} from "#src/datasource/zarr/codec/sharding_indexed/resolve.js"; import { ShardIndexLocation } from "#src/datasource/zarr/codec/sharding_indexed/resolve.js"; +import { FileByteRangeHandle } from "#src/kvstore/byte_range/file_handle.js"; import type { ByteRangeRequest, ReadableKvStore, - ReadOptions, + DriverReadOptions, ReadResponse, + StatResponse, + StatOptions, + ByteRange, } from "#src/kvstore/index.js"; -import { composeByteRangeRequest } from "#src/kvstore/index.js"; +import { KvStoreFileHandle } from "#src/kvstore/index.js"; import type { Owned } from "#src/util/disposable.js"; import { RefCounted } from "#src/util/disposable.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; type ShardIndex = BigUint64Array | undefined; const MISSING_VALUE = BigInt("18446744073709551615"); +type ShardIndexCache = SimpleAsyncCache; + +function makeIndexCache( + chunkManager: ChunkManager, + base: ReadableKvStore, + configuration: IndexConfiguration, +): ShardIndexCache { + return new SimpleAsyncCache(chunkManager.addRef(), { + get: async (key: BaseKey, progressOptions: ProgressOptions) => { + const { indexCodecs } = configuration; + const encodedSize = + indexCodecs.encodedSize[indexCodecs.encodedSize.length - 1]; + let byteRange: ByteRangeRequest; + switch (configuration.indexLocation) { + case ShardIndexLocation.START: + byteRange = { offset: 0, length: encodedSize! }; + break; + case ShardIndexLocation.END: + byteRange = { suffixLength: encodedSize! }; + break; + } + const response = await base.read(key, { + ...progressOptions, + byteRange, + }); + if (response === undefined) { + return { size: 0, data: undefined }; + } + const index = await decodeArray( + configuration.indexCodecs, + new Uint8Array(await response.response.arrayBuffer()), + progressOptions.signal, + ); + return { + size: index.byteLength, + data: new BigUint64Array( + index.buffer, + index.byteOffset, + index.byteLength / 8, + ), + }; + }, + }); +} + class ShardedKvStore extends RefCounted implements ReadableKvStore<{ base: BaseKey; subChunk: number[] }> { - private indexCache: Owned>; + private indexCache: Owned>; private indexStrides: number[]; constructor( private configuration: Configuration, @@ -50,42 +103,7 @@ class ShardedKvStore ) { super(); this.indexCache = this.registerDisposer( - new SimpleAsyncCache(chunkManager.addRef(), { - get: async (key: BaseKey, abortSignal: AbortSignal) => { - const { indexCodecs } = configuration; - const encodedSize = - indexCodecs.encodedSize[indexCodecs.encodedSize.length - 1]; - let byteRange: ByteRangeRequest; - switch (configuration.indexLocation) { - case ShardIndexLocation.START: - byteRange = { offset: 0, length: encodedSize! }; - break; - case ShardIndexLocation.END: - byteRange = { suffixLength: encodedSize! }; - break; - } - const response = await base.read(key, { - abortSignal, - byteRange, - }); - if (response === undefined) { - return { size: 0, data: undefined }; - } - const index = await decodeArray( - configuration.indexCodecs, - response.data, - abortSignal, - ); - return { - size: index.byteLength, - data: new BigUint64Array( - index.buffer, - index.byteOffset, - index.byteLength / 8, - ), - }; - }, - }), + makeIndexCache(chunkManager, base, configuration), ); const { subChunkGridShape } = this.configuration; const rank = subChunkGridShape.length; @@ -105,11 +123,14 @@ class ShardedKvStore } } - async read( - key: { base: BaseKey; subChunk: number[] }, - options: ReadOptions, - ): Promise { - const shardIndex = await this.indexCache.get(key.base, options.abortSignal); + private async findKey( + key: { + base: BaseKey; + subChunk: number[]; + }, + progressOptions: Partial, + ): Promise { + const shardIndex = await this.indexCache.get(key.base, progressOptions); if (shardIndex === undefined) { // Shard not present. return undefined; @@ -128,42 +149,42 @@ class ShardedKvStore // Sub-chunk not present. return undefined; } - const fullByteRange = { + return { offset: Number(dataOffset), length: Number(dataLength), }; - const { outer: outerByteRange, inner: innerByteRange } = - composeByteRangeRequest(fullByteRange, options.byteRange); - if (outerByteRange.length === 0) { - return { - data: new Uint8Array(0), - dataRange: innerByteRange, - totalSize: fullByteRange.length, - }; - } - const response = await this.base.read(key.base, { - abortSignal: options.abortSignal, - byteRange: outerByteRange, - }); - if (response === undefined) { - // Shard unexpectedly deleted. - return undefined; - } - if ( - response.dataRange.offset !== outerByteRange.offset || - response.dataRange.length !== outerByteRange.length - ) { - throw new Error( - `Received truncated response, expected ${JSON.stringify( - outerByteRange, - )} but received ${JSON.stringify(response.dataRange)}`, - ); - } - return { - data: response.data, - dataRange: innerByteRange, - totalSize: fullByteRange.length, - }; + } + + async stat( + key: { base: BaseKey; subChunk: number[] }, + options: StatOptions, + ): Promise { + const fullByteRange = await this.findKey(key, options); + if (fullByteRange === undefined) return undefined; + return { totalSize: fullByteRange.length }; + } + + async read( + key: { base: BaseKey; subChunk: number[] }, + options: DriverReadOptions, + ): Promise { + const fullByteRange = await this.findKey(key, options); + if (fullByteRange === undefined) return undefined; + return new FileByteRangeHandle( + new KvStoreFileHandle(this.base, key.base), + fullByteRange, + ).read(options); + } + + getUrl(key: { base: BaseKey; subChunk: number[] }): string { + return `subchunk ${JSON.stringify(key.subChunk)} within shard ${this.base.getUrl(key.base)}`; + } + + get supportsOffsetReads() { + return true; + } + get supportsSuffixReads() { + return true; } } diff --git a/src/datasource/zarr/codec/sharding_indexed/resolve.ts b/src/datasource/zarr/codec/sharding_indexed/resolve.ts index 58fd2212d5..53aa858d02 100644 --- a/src/datasource/zarr/codec/sharding_indexed/resolve.ts +++ b/src/datasource/zarr/codec/sharding_indexed/resolve.ts @@ -38,9 +38,12 @@ export enum ShardIndexLocation { END, } -export interface Configuration { +export interface IndexConfiguration { indexCodecs: CodecChainSpec; indexLocation: ShardIndexLocation; +} + +export interface Configuration extends IndexConfiguration { subChunkCodecs: CodecChainSpec; subChunkShape: number[]; subChunkGridShape: number[]; diff --git a/src/datasource/zarr/codec/transpose/decode.ts b/src/datasource/zarr/codec/transpose/decode.ts index 6eafd6bb90..3b15e228a6 100644 --- a/src/datasource/zarr/codec/transpose/decode.ts +++ b/src/datasource/zarr/codec/transpose/decode.ts @@ -26,10 +26,10 @@ registerCodec({ configuration: Configuration, decodedArrayInfo: CodecArrayInfo, encoded, - abortSignal: AbortSignal, + signal: AbortSignal, ) { decodedArrayInfo; - abortSignal; + signal; configuration; return encoded; }, diff --git a/src/datasource/zarr/codec/zstd/decode.ts b/src/datasource/zarr/codec/zstd/decode.ts index eed9654ab7..157ebf7a57 100644 --- a/src/datasource/zarr/codec/zstd/decode.ts +++ b/src/datasource/zarr/codec/zstd/decode.ts @@ -23,11 +23,11 @@ import type { Configuration } from "#src/datasource/zarr/codec/zstd/resolve.js"; registerCodec({ name: "zstd", kind: CodecKind.bytesToBytes, - decode(configuration: Configuration, encoded, abortSignal: AbortSignal) { + decode(configuration: Configuration, encoded, signal: AbortSignal) { configuration; return requestAsyncComputation( decodeZstd, - abortSignal, + signal, [encoded.buffer], encoded, ); diff --git a/src/datasource/zarr/frontend.ts b/src/datasource/zarr/frontend.ts index 8cb1ee27e3..3060a75fe6 100644 --- a/src/datasource/zarr/frontend.ts +++ b/src/datasource/zarr/frontend.ts @@ -18,7 +18,6 @@ import "#src/datasource/zarr/codec/blosc/resolve.js"; import "#src/datasource/zarr/codec/zstd/resolve.js"; import { makeDataBoundsBoundingBoxAnnotationSet } from "#src/annotation/index.js"; -import type { ChunkManager } from "#src/chunk_manager/frontend.js"; import { WithParameters } from "#src/chunk_manager/frontend.js"; import type { CoordinateSpace } from "#src/coordinate_transform.js"; import { @@ -26,13 +25,12 @@ import { makeIdentityTransform, makeIdentityTransformedBoundingBox, } from "#src/coordinate_transform.js"; -import { WithCredentialsProvider } from "#src/credentials_provider/chunk_source_frontend.js"; import type { - CompleteUrlOptions, DataSource, - GetDataSourceOptions, + GetKvStoreBasedDataSourceOptions, + KvStoreBasedDataSourceProvider, } from "#src/datasource/index.js"; -import { DataSourceProvider } from "#src/datasource/index.js"; +import { getKvStorePathCompletions } from "#src/datasource/kvstore_completions.js"; import { VolumeChunkSourceParameters } from "#src/datasource/zarr/base.js"; import "#src/datasource/zarr/codec/bytes/resolve.js"; import "#src/datasource/zarr/codec/crc32c/resolve.js"; @@ -53,6 +51,16 @@ import { } from "#src/datasource/zarr/metadata/parse.js"; import type { OmeMultiscaleMetadata } from "#src/datasource/zarr/ome.js"; import { parseOmeMetadata } from "#src/datasource/zarr/ome.js"; +import type { AutoDetectRegistry } from "#src/kvstore/auto_detect.js"; +import { simpleFilePresenceAutoDetectDirectorySpec } from "#src/kvstore/auto_detect.js"; +import { WithSharedKvStoreContext } from "#src/kvstore/chunk_source_frontend.js"; +import type { CompletionResult } from "#src/kvstore/context.js"; +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; +import { + kvstoreEnsureDirectoryPipelineUrl, + parseUrlSuffix, + pipelineUrlJoin, +} from "#src/kvstore/url.js"; import type { SliceViewSingleResolutionSource } from "#src/sliceview/frontend.js"; import type { VolumeSourceOptions } from "#src/sliceview/volume/base.js"; import { @@ -69,27 +77,17 @@ import { applyCompletionOffset, completeQueryStringParametersFromTable, } from "#src/util/completion.js"; -import type { Borrowed } from "#src/util/disposable.js"; -import { completeHttpPath } from "#src/util/http_path_completion.js"; -import { isNotFoundError } from "#src/util/http_request.js"; import { parseQueryStringParameters, verifyObject, verifyOptionalObjectProperty, } from "#src/util/json.js"; import * as matrix from "#src/util/matrix.js"; -import { getObjectId } from "#src/util/object_id.js"; -import type { - SpecialProtocolCredentials, - SpecialProtocolCredentialsProvider, -} from "#src/util/special_protocol_request.js"; -import { - fetchSpecialOk, - parseSpecialUrl, -} from "#src/util/special_protocol_request.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; class ZarrVolumeChunkSource extends WithParameters( - WithCredentialsProvider()(VolumeChunkSource), + WithSharedKvStoreContext(VolumeChunkSource), VolumeChunkSourceParameters, ) {} @@ -109,11 +107,10 @@ export class MultiscaleVolumeChunkSource extends GenericMultiscaleVolumeChunkSou } constructor( - chunkManager: Borrowed, - public credentialsProvider: SpecialProtocolCredentialsProvider, + public sharedKvStoreContext: SharedKvStoreContext, public multiscale: ZarrMultiscaleInfo, ) { - super(chunkManager); + super(sharedKvStoreContext.chunkManager); this.volumeType = VolumeType.IMAGE; } @@ -160,7 +157,7 @@ export class MultiscaleVolumeChunkSource extends GenericMultiscaleVolumeChunkSou chunkSource: this.chunkManager.getChunkSource( ZarrVolumeChunkSource, { - credentialsProvider: this.credentialsProvider, + sharedKvStoreContext: this.sharedKvStoreContext, spec, parameters: { url: scale.url, @@ -177,25 +174,27 @@ export class MultiscaleVolumeChunkSource extends GenericMultiscaleVolumeChunkSou } function getJsonResource( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, + description: string, + options: Partial, ): Promise { - return chunkManager.memoize.getUncounted( + return sharedKvStoreContext.chunkManager.memoize.getAsync( { type: "zarr:json", url, - credentialsProvider: getObjectId(credentialsProvider), }, - async () => { - try { - return await fetchSpecialOk(credentialsProvider, url, {}).then( - (response) => response.json(), - ); - } catch (e) { - if (isNotFoundError(e)) return undefined; - throw e; - } + options, + async (options) => { + using _span = new ProgressSpan(options.progressListener, { + message: `Reading ${description} from ${url}`, + }); + const response = await sharedKvStoreContext.kvStoreContext.read( + url, + options, + ); + if (response === undefined) return undefined; + return await response.response.json(); }, ); } @@ -294,26 +293,19 @@ function getMultiscaleInfoForSingleArray( } async function resolveOmeMultiscale( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, multiscale: OmeMultiscaleMetadata, options: { explicitDimensionSeparator: DimensionSeparator | undefined; zarrVersion: 2 | 3; - }, + } & Partial, ): Promise { const scaleZarrMetadata = await Promise.all( multiscale.scales.map(async (scale) => { - const metadata = await getMetadata( - chunkManager, - credentialsProvider, - scale.url, - { - zarrVersion: options.zarrVersion, - expectedNodeType: "array", - explicitDimensionSeparator: options.explicitDimensionSeparator, - }, - ); + const metadata = await getMetadata(sharedKvStoreContext, scale.url, { + ...options, + expectedNodeType: "array", + }); if (metadata === undefined) { throw new Error( `zarr v{zarrVersion} array metadata not found at ${scale.url}`, @@ -383,19 +375,28 @@ async function resolveOmeMultiscale( } async function getMetadata( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContext, url: string, options: { zarrVersion?: 2 | 3 | undefined; expectedNodeType?: NodeType | undefined; explicitDimensionSeparator?: DimensionSeparator | undefined; - }, + } & Partial, ): Promise { if (options.zarrVersion === 2) { const [zarray, zattrs] = await Promise.all([ - getJsonResource(chunkManager, credentialsProvider, `${url}/.zarray`), - getJsonResource(chunkManager, credentialsProvider, `${url}/.zattrs`), + getJsonResource( + sharedKvStoreContext, + `${url}.zarray`, + "zarr v2 array metadata", + options, + ), + getJsonResource( + sharedKvStoreContext, + `${url}.zattrs`, + "zarr v2 attributes", + options, + ), ]); if (zarray === undefined) { if (zattrs === undefined) { @@ -421,9 +422,10 @@ async function getMetadata( } if (options.zarrVersion === 3) { const zarrJson = await getJsonResource( - chunkManager, - credentialsProvider, - `${url}/zarr.json`, + sharedKvStoreContext, + `${url}zarr.json`, + "zarr v3 metadata", + options, ); if (zarrJson === undefined) return undefined; if (options.explicitDimensionSeparator !== undefined) { @@ -434,11 +436,11 @@ async function getMetadata( return parseV3Metadata(zarrJson, options.expectedNodeType); } const [v2Result, v3Result] = await Promise.all([ - getMetadata(chunkManager, credentialsProvider, url, { + getMetadata(sharedKvStoreContext, url, { ...options, zarrVersion: 2, }), - getMetadata(chunkManager, credentialsProvider, url, { + getMetadata(sharedKvStoreContext, url, { ...options, zarrVersion: 3, }), @@ -449,49 +451,64 @@ async function getMetadata( return v2Result ?? v3Result; } -export class ZarrDataSource extends DataSourceProvider { - constructor(public zarrVersion: 2 | 3 | undefined = undefined) { - super(); +function resolveUrl(options: GetKvStoreBasedDataSourceOptions) { + const { + authorityAndPath: additionalPath, + query, + fragment, + } = parseUrlSuffix(options.url.suffix); + + if (query) { + throw new Error( + `Invalid URL ${JSON.stringify(options.url.url)}: query parameters not supported`, + ); + } + return { + kvStoreUrl: kvstoreEnsureDirectoryPipelineUrl(options.kvStoreUrl), + additionalPath: additionalPath ?? "", + fragment, + }; +} + +export class ZarrDataSource implements KvStoreBasedDataSourceProvider { + constructor(public zarrVersion: 2 | 3 | undefined = undefined) {} + get scheme() { + return `zarr${this.zarrVersion ?? ""}`; + } + get expectsDirectory() { + return true; } get description() { const versionStr = this.zarrVersion === undefined ? "" : ` v${this.zarrVersion}`; return `Zarr${versionStr} data source`; } - get(options: GetDataSourceOptions): Promise { - // Pattern is infallible. - let [, providerUrl, query] = - options.providerUrl.match(/([^?]*)(?:\?(.*))?$/)!; - const parameters = parseQueryStringParameters(query || ""); + get(options: GetKvStoreBasedDataSourceOptions): Promise { + let { kvStoreUrl, additionalPath, fragment } = resolveUrl(options); + kvStoreUrl = kvstoreEnsureDirectoryPipelineUrl( + pipelineUrlJoin(kvStoreUrl, additionalPath), + ); + const parameters = parseQueryStringParameters(fragment || ""); verifyObject(parameters); const dimensionSeparator = verifyOptionalObjectProperty( parameters, "dimension_separator", parseDimensionSeparator, ); - if (providerUrl.endsWith("/")) { - providerUrl = providerUrl.substring(0, providerUrl.length - 1); - } - return options.chunkManager.memoize.getUncounted( + return options.registry.chunkManager.memoize.getAsync( { type: "zarr:MultiscaleVolumeChunkSource", - providerUrl, + kvStoreUrl, dimensionSeparator, }, - async () => { - const { url, credentialsProvider } = parseSpecialUrl( - providerUrl, - options.credentialsManager, - ); - const metadata = await getMetadata( - options.chunkManager, - credentialsProvider, - url, - { - zarrVersion: this.zarrVersion, - explicitDimensionSeparator: dimensionSeparator, - }, - ); + options, + async (progressOptions) => { + const { sharedKvStoreContext } = options.registry; + const metadata = await getMetadata(sharedKvStoreContext, kvStoreUrl, { + ...progressOptions, + zarrVersion: this.zarrVersion, + explicitDimensionSeparator: dimensionSeparator, + }); if (metadata === undefined) { throw new Error("No zarr metadata found"); } @@ -499,31 +516,34 @@ export class ZarrDataSource extends DataSourceProvider { if (metadata.nodeType === "group") { // May be an OME-zarr multiscale dataset. const multiscale = parseOmeMetadata( - url, + kvStoreUrl, metadata.userAttributes, metadata.zarrVersion, ); if (multiscale === undefined) { - throw new Error("Neithre array nor OME multiscale metadata found"); + throw new Error("Neither array nor OME multiscale metadata found"); } multiscaleInfo = await resolveOmeMultiscale( - options.chunkManager, - credentialsProvider, + sharedKvStoreContext, multiscale, { + ...progressOptions, zarrVersion: metadata.zarrVersion, explicitDimensionSeparator: dimensionSeparator, }, ); } else { - multiscaleInfo = getMultiscaleInfoForSingleArray(url, metadata); + multiscaleInfo = getMultiscaleInfoForSingleArray( + kvStoreUrl, + metadata, + ); } const volume = new MultiscaleVolumeChunkSource( - options.chunkManager, - credentialsProvider, + sharedKvStoreContext, multiscaleInfo, ); return { + canonicalUrl: `${kvStoreUrl}|zarr${metadata.zarrVersion}:`, modelTransform: makeIdentityTransform(volume.modelSpace), subsources: [ { @@ -547,23 +567,46 @@ export class ZarrDataSource extends DataSourceProvider { }, ); } - - async completeUrl(options: CompleteUrlOptions) { - // Pattern is infallible. - const [, , query] = options.providerUrl.match(/([^?]*)(?:\?(.*))?$/)!; - if (query !== undefined) { - return applyCompletionOffset( - options.providerUrl.length - query.length, - await completeQueryStringParametersFromTable( - query, - supportedQueryParameters, - ), - ); + async completeUrl( + options: GetKvStoreBasedDataSourceOptions, + ): Promise { + const { kvStoreUrl, additionalPath, fragment } = resolveUrl(options); + if (fragment === undefined) { + return getKvStorePathCompletions(options.registry.sharedKvStoreContext, { + baseUrl: kvStoreUrl, + path: additionalPath, + directoryOnly: true, + signal: options.signal, + progressListener: options.progressListener, + }); + } + if (this.zarrVersion === 3) { + throw new Error("URL fragment parameters not supported"); } - return await completeHttpPath( - options.credentialsManager, - options.providerUrl, - options.abortSignal, + return applyCompletionOffset( + options.url.url.length - fragment.length, + await completeQueryStringParametersFromTable( + fragment, + supportedQueryParameters, + ), ); } } + +export function registerAutoDetectV2(registry: AutoDetectRegistry) { + registry.registerDirectoryFormat( + simpleFilePresenceAutoDetectDirectorySpec(new Set([".zarray", ".zattrs"]), { + suffix: "zarr2:", + description: "Zarr v2 data source", + }), + ); +} + +export function registerAutoDetectV3(registry: AutoDetectRegistry) { + registry.registerDirectoryFormat( + simpleFilePresenceAutoDetectDirectorySpec(new Set(["zarr.json"]), { + suffix: "zarr3:", + description: "Zarr v3 data source", + }), + ); +} diff --git a/src/datasource/zarr/ome.ts b/src/datasource/zarr/ome.ts index ebeed9b195..e3f049af56 100644 --- a/src/datasource/zarr/ome.ts +++ b/src/datasource/zarr/ome.ts @@ -188,7 +188,7 @@ function parseMultiscaleScale( "coordinateTransformations", (x) => parseOmeCoordinateTransforms(rank, x), ); - const scaleUrl = `${url}/${path}`; + const scaleUrl = `${url}${path}/`; return { url: scaleUrl, transform }; } diff --git a/src/datasource/zarr/register_default.ts b/src/datasource/zarr/register_default.ts index 7046d2bc82..a1813881b7 100644 --- a/src/datasource/zarr/register_default.ts +++ b/src/datasource/zarr/register_default.ts @@ -14,9 +14,25 @@ * limitations under the License. */ -import { registerProvider } from "#src/datasource/default_provider.js"; -import { ZarrDataSource } from "#src/datasource/zarr/frontend.js"; +import { + registerKvStoreBasedDataProvider, + dataSourceAutoDetectRegistry, + registerProvider, +} from "#src/datasource/default_provider.js"; +import { KvStoreBasedDataSourceLegacyUrlAdapter } from "#src/datasource/index.js"; +import { + ZarrDataSource, + registerAutoDetectV2, + registerAutoDetectV3, +} from "#src/datasource/zarr/frontend.js"; -registerProvider("zarr", () => new ZarrDataSource()); -registerProvider("zarr2", () => new ZarrDataSource(2)); -registerProvider("zarr3", () => new ZarrDataSource(3)); +for (const provider of [ + new ZarrDataSource(), + new ZarrDataSource(2), + new ZarrDataSource(3), +]) { + registerKvStoreBasedDataProvider(provider); + registerProvider(new KvStoreBasedDataSourceLegacyUrlAdapter(provider)); +} +registerAutoDetectV2(dataSourceAutoDetectRegistry); +registerAutoDetectV3(dataSourceAutoDetectRegistry); diff --git a/src/kvstore/auto_detect.ts b/src/kvstore/auto_detect.ts new file mode 100644 index 0000000000..d13f19482d --- /dev/null +++ b/src/kvstore/auto_detect.ts @@ -0,0 +1,332 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { KvStoreContext } from "#src/kvstore/context.js"; +import { readKvStore } from "#src/kvstore/index.js"; +import { pathIsDirectory } from "#src/kvstore/url.js"; +import { isGzipFormat } from "#src/util/gzip.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; + +export interface AutoDetectDirectoryOptions { + url: string; + fileNames: Set; + signal?: AbortSignal; +} + +export interface AutoDetectMatch { + suffix: string; + description: string; +} + +export interface AutoDetectDirectorySpec { + fileNames: Set; + match: (options: AutoDetectDirectoryOptions) => Promise; +} + +export function simpleFilePresenceAutoDetectDirectorySpec( + fileNames: Set, + match: AutoDetectMatch, +): AutoDetectDirectorySpec { + return { + fileNames, + match: async (options) => { + const detectedFileNames = options.fileNames; + for (const fileName of fileNames) { + if (detectedFileNames.has(fileName)) { + return [match]; + } + } + return []; + }, + }; +} + +export interface AutoDetectFileOptions { + url: string; + prefix: Uint8Array; + suffix?: Uint8Array; + totalSize: number | undefined; + signal?: AbortSignal; +} + +export interface AutoDetectFileSpec { + prefixLength: number; + suffixLength: number; + match: (options: AutoDetectFileOptions) => Promise; +} + +function composeMatchFunctions( + specs: { + match: (options: Options) => Promise; + }[], +): (options: Options) => Promise { + return async (options: Options) => { + const matches: AutoDetectMatch[] = []; + const results = await Promise.allSettled( + specs.map((spec) => spec.match(options)), + ); + for (const result of results) { + if (result.status !== "fulfilled") continue; + matches.push(...result.value); + } + return matches; + }; +} + +export function composeAutoDetectDirectorySpecs( + specs: AutoDetectDirectorySpec[], +): AutoDetectDirectorySpec { + const fileNames = new Set(); + for (const spec of specs) { + for (const fileName of spec.fileNames) { + fileNames.add(fileName); + } + } + return { fileNames, match: composeMatchFunctions(specs) }; +} + +export function composeAutoDetectFileSpecs( + specs: AutoDetectFileSpec[], +): AutoDetectFileSpec { + let prefixLength = 0; + let suffixLength = 0; + for (const spec of specs) { + prefixLength = Math.max(prefixLength, spec.prefixLength); + suffixLength = Math.max(suffixLength, spec.suffixLength); + } + return { prefixLength, suffixLength, match: composeMatchFunctions(specs) }; +} + +export class AutoDetectRegistry { + directorySpecs: AutoDetectDirectorySpec[] = []; + fileSpecs: AutoDetectFileSpec[] = []; + gzipFileSpecs: AutoDetectFileSpec[] = []; + private _directorySpec: AutoDetectDirectorySpec | undefined; + private _fileSpec: AutoDetectFileSpec | undefined; + + registerDirectoryFormat(spec: AutoDetectDirectorySpec) { + this.directorySpecs.push(spec); + this._directorySpec = undefined; + } + + registerFileFormat( + spec: AutoDetectFileSpec, + supportedEncodings?: { gzip?: boolean }, + ) { + this.fileSpecs.push(spec); + if (supportedEncodings?.gzip) { + this.gzipFileSpecs.push(spec); + } + this._fileSpec = undefined; + } + + copyTo(registry: AutoDetectRegistry) { + registry.directorySpecs.push(...this.directorySpecs); + registry.fileSpecs.push(...this.fileSpecs); + registry.gzipFileSpecs.push(...this.gzipFileSpecs); + registry._fileSpec = undefined; + registry._directorySpec = undefined; + } + + get directorySpec() { + return ( + this._directorySpec ?? (this._directorySpec = this.getDirectorySpec()) + ); + } + + private getDirectorySpec() { + return composeAutoDetectDirectorySpecs(this.directorySpecs); + } + + get fileSpec() { + return this._fileSpec ?? (this._fileSpec = this.getFileSpec()); + } + + private getFileSpec() { + const { fileSpecs, gzipFileSpecs } = this; + const specs = [...fileSpecs]; + if (gzipFileSpecs.length > 0) { + specs.push(getGzipFileSpec(composeAutoDetectFileSpecs(gzipFileSpecs))); + } + return composeAutoDetectFileSpecs(specs); + } +} + +export interface AutoDetectFormatOptions extends Partial { + kvStoreContext: KvStoreContext; + url: string; + autoDetectDirectory: () => AutoDetectDirectorySpec; + autoDetectFile: () => AutoDetectFileSpec; +} + +export async function autoDetectFormat( + options: AutoDetectFormatOptions, +): Promise<{ matches: AutoDetectMatch[]; url: string }> { + const kvStore = options.kvStoreContext.getKvStore(options.url); + const { progressListener } = options; + using _span = + progressListener && + new ProgressSpan(progressListener, { + message: `Auto-detecting data format at ${options.url}`, + }); + if (!pathIsDirectory(kvStore.path) || kvStore.store.singleKey === true) { + const statResponse = await kvStore.store.stat(kvStore.path, { + signal: options.signal, + progressListener: options.progressListener, + }); + if (statResponse !== undefined) { + // Match as file. + const autoDetectFile = options.autoDetectFile(); + const { totalSize } = statResponse; + let prefix: Uint8Array; + let suffix: Uint8Array | undefined; + if (totalSize !== undefined && autoDetectFile.suffixLength > 0) { + if ( + totalSize <= + autoDetectFile.prefixLength + autoDetectFile.suffixLength + ) { + // Perform a single read + const readResponse = await readKvStore(kvStore.store, kvStore.path, { + signal: options.signal, + progressListener: options.progressListener, + throwIfMissing: true, + }); + prefix = suffix = new Uint8Array( + await readResponse.response.arrayBuffer(), + ); + } else { + [prefix, suffix] = await Promise.all( + [ + { offset: 0, length: autoDetectFile.prefixLength }, + { + offset: totalSize - autoDetectFile.suffixLength, + length: autoDetectFile.suffixLength, + }, + ].map((byteRange) => + readKvStore(kvStore.store, kvStore.path, { + signal: options.signal, + progressListener: options.progressListener, + throwIfMissing: true, + byteRange, + }) + .then((readResponse) => readResponse.response.arrayBuffer()) + .then((arrayBuffer) => new Uint8Array(arrayBuffer)), + ), + ); + } + } else { + prefix = new Uint8Array( + await ( + await readKvStore(kvStore.store, kvStore.path, { + signal: options.signal, + progressListener: options.progressListener, + throwIfMissing: true, + byteRange: { offset: 0, length: autoDetectFile.prefixLength }, + }) + ).response.arrayBuffer(), + ); + } + return { + matches: await autoDetectFile.match({ + url: options.url, + prefix, + suffix, + totalSize, + signal: options.signal, + }), + url: options.url, + }; + } + + if (kvStore.store.singleKey === true) { + return { matches: [], url: options.url }; + } + kvStore.path += "/"; + } + + const autoDetectDirectory = options.autoDetectDirectory(); + const detectedFileNames = new Set(); + await Promise.all( + Array.from(autoDetectDirectory.fileNames, async (fileName) => { + const response = await kvStore.store.stat(kvStore.path + fileName, { + signal: options.signal, + progressListener: options.progressListener, + }); + if (response !== undefined) { + detectedFileNames.add(fileName); + } + }), + ); + const url = kvStore.store.getUrl(kvStore.path); + const matches = await autoDetectDirectory.match({ + url, + fileNames: detectedFileNames, + signal: options.signal, + }); + return { matches, url }; +} + +function getGzipFileSpec(decodedSpec: AutoDetectFileSpec): AutoDetectFileSpec { + // Heuristic, assume gzip adds at most 100 bytes of overhead. + const prefixLength = decodedSpec.prefixLength + 100; + + async function detect( + options: AutoDetectFileOptions, + ): Promise { + if (!isGzipFormat(options.prefix)) return []; + const decompressedStream = new Response(options.prefix).body!.pipeThrough( + new DecompressionStream("gzip"), + { signal: options.signal }, + ); + let decodedPrefix = new Uint8Array(decodedSpec.prefixLength); + // Decode as much as possible. + let totalLength = 0; + + try { + const reader = decompressedStream.getReader(); + while (true) { + let { value } = await reader.read(); + if (value === undefined) break; + const remainingLength = decodedPrefix.length - totalLength; + if (value.length > remainingLength) { + value = value.subarray(0, remainingLength); + } + decodedPrefix.set(value, totalLength); + totalLength += value.length; + if (totalLength === decodedPrefix.length) { + break; + } + } + } catch { + // Ignore failure, likely due to truncated input. + } + decodedPrefix = decodedPrefix.subarray(0, totalLength); + + const matches = await decodedSpec.match({ + url: options.url, + signal: options.signal, + prefix: decodedPrefix, + totalSize: undefined, + }); + return matches.map(({ suffix, description }) => ({ + suffix, + description: `${description} (gzip-compressed)`, + })); + } + + return { prefixLength, suffixLength: 0, match: detect }; +} diff --git a/src/kvstore/backend.ts b/src/kvstore/backend.ts new file mode 100644 index 0000000000..96748c9d13 --- /dev/null +++ b/src/kvstore/backend.ts @@ -0,0 +1,147 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ChunkManager } from "#src/chunk_manager/backend.js"; +import type { SharedCredentialsManagerCounterpart } from "#src/credentials_provider/shared_counterpart.js"; +import { KvStoreContext } from "#src/kvstore/context.js"; +import { + frontendBackendIsomorphicKvStoreProviderRegistry, + KvStoreProviderRegistry, +} from "#src/kvstore/register.js"; +import { + LIST_RPC_ID, + READ_RPC_ID, + SHARED_KVSTORE_CONTEXT_RPC_ID, + STAT_RPC_ID, +} from "#src/kvstore/shared_common.js"; +import type { RPC } from "#src/worker_rpc.js"; +import { + registerPromiseRPC, + registerSharedObject, + SharedObjectCounterpart, +} from "#src/worker_rpc.js"; +import type { ByteRange } from "."; + +@registerSharedObject(SHARED_KVSTORE_CONTEXT_RPC_ID) +export class SharedKvStoreContextCounterpart extends SharedObjectCounterpart { + kvStoreContext: KvStoreContext; + + chunkManager: ChunkManager; + credentialsManager: SharedCredentialsManagerCounterpart; + + constructor(rpc: RPC, options: any) { + super(rpc, options); + this.chunkManager = rpc.get(options.chunkManager) as ChunkManager; + this.credentialsManager = rpc.get( + options.credentialsManager, + ) as SharedCredentialsManagerCounterpart; + this.kvStoreContext = new KvStoreContext(); + frontendBackendIsomorphicKvStoreProviderRegistry.applyToContext(this); + backendOnlyKvStoreProviderRegistry.applyToContext(this); + } +} + +export const backendOnlyKvStoreProviderRegistry = + new KvStoreProviderRegistry(); + +export function WithSharedKvStoreContextCounterpart< + TBase extends { new (...args: any[]): SharedObjectCounterpart }, +>(Base: TBase) { + return class extends Base { + sharedKvStoreContext: SharedKvStoreContextCounterpart; + constructor(...args: any[]) { + super(...args); + const options = args[1]; + this.sharedKvStoreContext = this.rpc!.get(options.sharedKvStoreContext); + } + }; +} + +registerPromiseRPC( + STAT_RPC_ID, + async function ( + this: RPC, + options: { sharedKvStoreContext: number; url: string }, + progressOptions, + ) { + const sharedKvStoreContext: SharedKvStoreContextCounterpart = this.get( + options.sharedKvStoreContext, + ); + return { + value: await sharedKvStoreContext.kvStoreContext.stat( + options.url, + progressOptions, + ), + }; + }, +); + +registerPromiseRPC( + READ_RPC_ID, + async function ( + this: RPC, + options: { + sharedKvStoreContext: number; + url: string; + byteRange?: ByteRange; + throwIfMissing?: boolean; + }, + progressOptions, + ) { + const sharedKvStoreContext: SharedKvStoreContextCounterpart = this.get( + options.sharedKvStoreContext, + ); + const readResponse = await sharedKvStoreContext.kvStoreContext.read( + options.url, + { + ...progressOptions, + byteRange: options.byteRange, + throwIfMissing: options.throwIfMissing, + }, + ); + if (readResponse === undefined) { + return { value: undefined }; + } + const arrayBuffer = await readResponse.response.arrayBuffer(); + return { + value: { + data: arrayBuffer, + offset: readResponse.offset, + totalSize: readResponse.totalSize, + }, + transfers: [arrayBuffer], + }; + }, +); + +registerPromiseRPC( + LIST_RPC_ID, + async function ( + this: RPC, + options: { sharedKvStoreContext: number; url: string }, + progressOptions, + ) { + const sharedKvStoreContext: SharedKvStoreContextCounterpart = this.get( + options.sharedKvStoreContext, + ); + const { store, path } = sharedKvStoreContext.kvStoreContext.getKvStore( + options.url, + ); + return { + value: await store.list!(path, progressOptions), + }; + }, +); diff --git a/src/kvstore/byte_range/file_handle.ts b/src/kvstore/byte_range/file_handle.ts new file mode 100644 index 0000000000..865ac9068e --- /dev/null +++ b/src/kvstore/byte_range/file_handle.ts @@ -0,0 +1,94 @@ +/** + * @license + * Copyright 2023 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { + ByteRange, + ByteRangeRequest, + DriverReadOptions, + FileHandle, + ReadResponse, + StatOptions, + StatResponse, +} from "#src/kvstore/index.js"; +import { readFileHandle } from "#src/kvstore/index.js"; + +export function composeByteRangeRequest( + outer: ByteRange, + inner: ByteRangeRequest | undefined, +): { outer: ByteRange; inner: ByteRange } { + if (inner === undefined) { + return { outer, inner: { offset: 0, length: outer.length } }; + } + if ("suffixLength" in inner) { + const length = Math.min(outer.length, inner.suffixLength); + return { + outer: { offset: outer.offset + (outer.length - length), length }, + inner: { offset: outer.length - length, length }, + }; + } + if (inner.offset + inner.length > outer.length) { + throw new Error( + `Requested byte range ${JSON.stringify( + inner, + )} not valid for value of length ${outer.length}`, + ); + } + return { + outer: { offset: outer.offset + inner.offset, length: inner.length }, + inner, + }; +} + +export class FileByteRangeHandle implements FileHandle { + constructor( + public base: FileHandle, + public byteRange: ByteRange, + ) {} + + async stat(options: StatOptions): Promise { + options; + return { totalSize: this.byteRange.length }; + } + + async read(options: DriverReadOptions): Promise { + const { byteRange } = this; + const { outer: outerByteRange, inner: innerByteRange } = + composeByteRangeRequest(byteRange, options.byteRange); + if (outerByteRange.length === 0) { + return { + response: new Response(new Uint8Array(0)), + totalSize: byteRange.length, + ...innerByteRange, + }; + } + const response = await readFileHandle(this.base, { + signal: options.signal, + byteRange: outerByteRange, + strictByteRange: true, + throwIfMissing: true, + }); + return { + response: response.response, + totalSize: byteRange.length, + ...innerByteRange, + }; + } + + getUrl() { + const { offset, length } = this.byteRange; + return `${this.base.getUrl()}|range:${offset}-${offset + length}`; + } +} diff --git a/src/kvstore/byte_range/index.ts b/src/kvstore/byte_range/index.ts new file mode 100644 index 0000000000..93c9ae606e --- /dev/null +++ b/src/kvstore/byte_range/index.ts @@ -0,0 +1,75 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { FileByteRangeHandle } from "#src/kvstore/byte_range/file_handle.js"; +import type { + DriverReadOptions, + KvStore, + ReadResponse, + StatOptions, + StatResponse, + FileHandle, + ByteRange, +} from "#src/kvstore/index.js"; + +function parseKey(key: string): ByteRange { + const m = key.match(/^([0-9]+)-([0-9]+)$/); + if (m !== null) { + const begin = Number(m[1]); + const end = Number(m[2]); + if (end >= begin) { + return { offset: begin, length: end - begin }; + } + } + throw new Error( + `Invalid key ${JSON.stringify(key)} for "byte-range:", expected "-"`, + ); +} + +export class ByteRangeKvStore implements KvStore { + constructor(public base: FileHandle) {} + + getUrl(key: string) { + return this.base.getUrl() + `|byte-range:${key}`; + } + + async stat( + key: string, + options: StatOptions, + ): Promise { + const { length } = parseKey(key); + options; + return { totalSize: length }; + } + + async read( + key: string, + options: DriverReadOptions, + ): Promise { + const byteRange = parseKey(key); + return new FileByteRangeHandle(this.base, byteRange).read(options); + } + + get supportsOffsetReads() { + return true; + } + get supportsSuffixReads() { + return true; + } + get singleKey() { + return true; + } +} diff --git a/src/kvstore/byte_range/register.ts b/src/kvstore/byte_range/register.ts new file mode 100644 index 0000000000..4bdacc9f70 --- /dev/null +++ b/src/kvstore/byte_range/register.ts @@ -0,0 +1,41 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ByteRangeKvStore } from "#src/kvstore/byte_range/index.js"; +import type { KvStoreAdapterProvider } from "#src/kvstore/context.js"; +import { KvStoreFileHandle } from "#src/kvstore/index.js"; +import { frontendBackendIsomorphicKvStoreProviderRegistry } from "#src/kvstore/register.js"; +import { ensureNoQueryOrFragmentParameters } from "#src/kvstore/url.js"; + +function byteRangeProvider(): KvStoreAdapterProvider { + return { + scheme: "byte-range", + description: `byte range slicing`, + getKvStore(url, base) { + ensureNoQueryOrFragmentParameters(url); + return { + store: new ByteRangeKvStore( + new KvStoreFileHandle(base.store, base.path), + ), + path: url.suffix ?? "", + }; + }, + }; +} + +frontendBackendIsomorphicKvStoreProviderRegistry.registerKvStoreAdapterProvider( + byteRangeProvider, +); diff --git a/src/kvstore/chunk_source_frontend.ts b/src/kvstore/chunk_source_frontend.ts new file mode 100644 index 0000000000..a626923c12 --- /dev/null +++ b/src/kvstore/chunk_source_frontend.ts @@ -0,0 +1,55 @@ +/** + * @license + * Copyright 2017 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file Facilities to simplify defining subclasses of ChunkSource that use a CredentialsProvider. + */ + +import type { + ChunkManager, + ChunkSourceConstructor, + GettableChunkSource, +} from "#src/chunk_manager/frontend.js"; +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; +import type { RPC } from "#src/worker_rpc.js"; + +/** + * Mixin for adding a credentialsProvider member to a ChunkSource. + */ +export function WithSharedKvStoreContext< + TBase extends ChunkSourceConstructor< + GettableChunkSource & { chunkManager: ChunkManager } + >, +>(Base: TBase) { + type WithSharedKvStoreContextOptions = InstanceType["OPTIONS"] & { + sharedKvStoreContext: SharedKvStoreContext; + }; + class C extends Base { + sharedKvStoreContext: SharedKvStoreContext; + declare OPTIONS: WithSharedKvStoreContextOptions; + constructor(...args: any[]) { + super(...args); + const options: WithSharedKvStoreContextOptions = args[1]; + this.sharedKvStoreContext = options.sharedKvStoreContext.addRef(); + } + initializeCounterpart(rpc: RPC, options: any) { + const { sharedKvStoreContext } = this; + options.sharedKvStoreContext = sharedKvStoreContext.rpcId; + super.initializeCounterpart(rpc, options); + } + } + return C; +} diff --git a/src/kvstore/context.ts b/src/kvstore/context.ts new file mode 100644 index 0000000000..564574c027 --- /dev/null +++ b/src/kvstore/context.ts @@ -0,0 +1,138 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AutoDetectRegistry } from "#src/kvstore/auto_detect.js"; +import type { + KvStoreWithPath, + ListResponse, + ReadResponse, + ReadOptions, + ListOptions, + StatOptions, + StatResponse, +} from "#src/kvstore/index.js"; +import { listKvStore, readKvStore } from "#src/kvstore/index.js"; +import type { UrlWithParsedScheme } from "#src/kvstore/url.js"; +import { resolveRelativePath, splitPipelineUrl } from "#src/kvstore/url.js"; +import type { + BasicCompletionResult, + CompletionWithDescription, +} from "#src/util/completion.js"; + +export type CompletionResult = BasicCompletionResult; + +export interface BaseKvStoreProvider { + scheme: string; + hidden?: boolean; + description?: string; + getKvStore(parsedUrl: UrlWithParsedScheme): KvStoreWithPath; +} + +export interface KvStoreAdapterProvider { + scheme: string; + hidden?: boolean; + description?: string; + getKvStore( + parsedUrl: UrlWithParsedScheme, + base: KvStoreWithPath, + ): KvStoreWithPath; +} + +export class KvStoreContext { + baseKvStoreProviders = new Map(); + kvStoreAdapterProviders = new Map(); + autoDetectRegistry = new AutoDetectRegistry(); + + getKvStore(url: string): KvStoreWithPath { + const pipeline = splitPipelineUrl(url); + let kvStore: KvStoreWithPath; + { + const basePart = pipeline[0]; + const provider = this.baseKvStoreProviders.get(basePart.scheme); + if (provider === undefined) { + const usage = this.describeProtocolUsage(basePart.scheme); + let message = `Invalid base kvstore protocol "${basePart.scheme}:"`; + if (usage !== undefined) { + message += `; ${usage}`; + } + throw new Error(message); + } + kvStore = provider.getKvStore(basePart); + } + + for (let i = 1; i < pipeline.length; ++i) { + const part = pipeline[i]; + const provider = this.kvStoreAdapterProviders.get(part.scheme); + if (provider === undefined) { + const usage = this.describeProtocolUsage(part.scheme); + let message = `Invalid kvstore adapter protocol "${part.scheme}:" in ${JSON.stringify(url)}`; + if (usage !== undefined) { + message += `; ${usage}`; + } + throw new Error(message); + } + kvStore = provider.getKvStore(part, kvStore); + } + return kvStore; + } + + // Describes valid uses of `protocol`, for error messages indicating an + // invalid protocol. If the protocol is unknown, returns `undefined`. + describeProtocolUsage(protocol: string): string | undefined { + if (this.baseKvStoreProviders.has(protocol)) { + return `"${protocol}:" may only be used as a base kvstore protocol`; + } + if (this.kvStoreAdapterProviders.has(protocol)) { + return `"${protocol}:" may only be used as a kvstore adapter protocol`; + } + return undefined; + } + + stat( + url: string, + options: StatOptions = {}, + ): Promise { + const kvStore = this.getKvStore(url); + return kvStore.store.stat(kvStore.path, options); + } + + read( + url: string, + options: ReadOptions & { throwIfMissing: true }, + ): Promise; + + read(url: string, options?: ReadOptions): Promise; + + read( + url: string, + options: ReadOptions = {}, + ): Promise { + const kvStore = this.getKvStore(url); + return readKvStore(kvStore.store, kvStore.path, options); + } + + list(urlPrefix: string, options: ListOptions = {}): Promise { + const kvStore = this.getKvStore(urlPrefix); + return listKvStore(kvStore.store, kvStore.path, options); + } + + resolveRelativePath(baseUrl: string, relativePath: string): string { + const kvStore = this.getKvStore(baseUrl); + return kvStore.store.getUrl( + resolveRelativePath(kvStore.path, relativePath), + ); + } +} diff --git a/src/kvstore/enabled_backend_modules.ts b/src/kvstore/enabled_backend_modules.ts new file mode 100644 index 0000000000..873feeb4be --- /dev/null +++ b/src/kvstore/enabled_backend_modules.ts @@ -0,0 +1,9 @@ +// DO NOT EDIT: Generated by config/update_conditions.ts +import "#kvstore/byte_range/register"; +import "#kvstore/gcs/register"; +import "#kvstore/gzip/register"; +import "#kvstore/http/register"; +import "#kvstore/middleauth/register"; +import "#kvstore/ngauth/register"; +import "#kvstore/s3/register"; +import "#kvstore/zip/register_backend"; diff --git a/src/kvstore/enabled_frontend_modules.ts b/src/kvstore/enabled_frontend_modules.ts new file mode 100644 index 0000000000..051976e929 --- /dev/null +++ b/src/kvstore/enabled_frontend_modules.ts @@ -0,0 +1,11 @@ +// DO NOT EDIT: Generated by config/update_conditions.ts +import "#kvstore/byte_range/register"; +import "#kvstore/gcs/register"; +import "#kvstore/gzip/register"; +import "#kvstore/http/register"; +import "#kvstore/middleauth/register"; +import "#kvstore/middleauth/register_credentials_provider"; +import "#kvstore/ngauth/register"; +import "#kvstore/ngauth/register_credentials_provider"; +import "#kvstore/s3/register"; +import "#kvstore/zip/register_frontend"; diff --git a/src/kvstore/frontend.ts b/src/kvstore/frontend.ts new file mode 100644 index 0000000000..4de55f8519 --- /dev/null +++ b/src/kvstore/frontend.ts @@ -0,0 +1,151 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ChunkManager } from "#src/chunk_manager/frontend.js"; +import type { SharedCredentialsManager } from "#src/credentials_provider/shared.js"; +import { KvStoreContext } from "#src/kvstore/context.js"; +import type { + DriverListOptions, + DriverReadOptions, + ListResponse, + ReadResponse, + StatOptions, + StatResponse, +} from "#src/kvstore/index.js"; +import type { SharedKvStoreContextBase } from "#src/kvstore/register.js"; +import { + frontendBackendIsomorphicKvStoreProviderRegistry, + KvStoreProviderRegistry, +} from "#src/kvstore/register.js"; +import { + LIST_RPC_ID, + READ_RPC_ID, + SHARED_KVSTORE_CONTEXT_RPC_ID, + STAT_RPC_ID, +} from "#src/kvstore/shared_common.js"; +import { registerSharedObjectOwner, SharedObject } from "#src/worker_rpc.js"; + +@registerSharedObjectOwner(SHARED_KVSTORE_CONTEXT_RPC_ID) +export class SharedKvStoreContext + extends SharedObject + implements SharedKvStoreContextBase +{ + kvStoreContext = new KvStoreContext(); + + constructor( + public chunkManager: ChunkManager, + public credentialsManager: SharedCredentialsManager, + ) { + super(); + this.initializeCounterpart(chunkManager.rpc!, { + chunkManager: chunkManager.rpcId, + credentialsManager: credentialsManager.rpcId, + }); + frontendBackendIsomorphicKvStoreProviderRegistry.applyToContext(this); + frontendOnlyKvStoreProviderRegistry.applyToContext(this); + } +} + +export const frontendOnlyKvStoreProviderRegistry = + new KvStoreProviderRegistry(); + +export function proxyStatToBackendKvStore( + sharedKvStoreContext: SharedKvStoreContext, + url: string, + options: StatOptions, +): Promise { + return sharedKvStoreContext.rpc!.promiseInvoke( + STAT_RPC_ID, + { sharedKvStoreContext: sharedKvStoreContext.rpcId, url }, + { signal: options.signal, progressListener: options.progressListener }, + ); +} + +export async function proxyReadToBackendKvStore( + sharedKvStoreContext: SharedKvStoreContext, + url: string, + options: DriverReadOptions, +): Promise { + const response = await sharedKvStoreContext.rpc!.promiseInvoke< + | { data: ArrayBuffer; offset: number; totalSize: number | undefined } + | undefined + >( + READ_RPC_ID, + { + sharedKvStoreContext: sharedKvStoreContext.rpcId, + url, + byteRange: options.byteRange, + throwIfMissing: options.throwIfMissing, + }, + { signal: options.signal, progressListener: options.progressListener }, + ); + if (response === undefined) return undefined; + return { + response: new Response(response.data), + offset: response.offset, + length: response.data.byteLength, + totalSize: response.totalSize, + }; +} + +export function proxyListToBackendKvStore( + sharedKvStoreContext: SharedKvStoreContext, + url: string, + options: DriverListOptions, +): Promise { + return sharedKvStoreContext.rpc!.promiseInvoke( + LIST_RPC_ID, + { + sharedKvStoreContext: sharedKvStoreContext.rpcId, + url, + }, + { signal: options.signal, progressListener: options.progressListener }, + ); +} + +export abstract class ProxyReadableKvStore { + constructor(public sharedKvStoreContext: SharedKvStoreContext) {} + + abstract getUrl(key: Key): string; + + stat(key: Key, options: StatOptions): Promise { + return proxyStatToBackendKvStore( + this.sharedKvStoreContext, + this.getUrl(key), + options, + ); + } + read( + key: Key, + options: DriverReadOptions, + ): Promise { + return proxyReadToBackendKvStore( + this.sharedKvStoreContext, + this.getUrl(key), + options, + ); + } +} + +export abstract class ProxyKvStore extends ProxyReadableKvStore { + list(prefix: string, options: DriverListOptions): Promise { + return proxyListToBackendKvStore( + this.sharedKvStoreContext, + this.getUrl(prefix), + options, + ); + } +} diff --git a/src/kvstore/gcs/index.ts b/src/kvstore/gcs/index.ts new file mode 100644 index 0000000000..13e893f0d6 --- /dev/null +++ b/src/kvstore/gcs/index.ts @@ -0,0 +1,150 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { read, stat } from "#src/kvstore/http/read.js"; +import type { + KvStore, + DriverListOptions, + ListResponse, + DriverReadOptions, + ReadResponse, + StatOptions, + StatResponse, +} from "#src/kvstore/index.js"; +import { encodePathForUrl } from "#src/kvstore/url.js"; +import type { FetchOk } from "#src/util/http_request.js"; +import { fetchOk } from "#src/util/http_request.js"; +import { + parseArray, + verifyObject, + verifyObjectProperty, + verifyOptionalObjectProperty, + verifyString, + verifyStringArray, +} from "#src/util/json.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; +import { getRandomHexString } from "#src/util/random.js"; + +export class GcsKvStore implements KvStore { + constructor( + public bucket: string, + public baseUrlForDisplay: string = `gs://${bucket}/`, + private fetchOkImpl: FetchOk = fetchOk, + ) {} + + private getObjectUrl(key: string): string { + // Include random query string parameter (ignored by GCS) to bypass GCS cache + // and ensure a cached response is never used. + // + // This addresses two issues related to GCS: + // + // 1. GCS fails to send an updated `Access-Control-Allow-Origin` header in 304 + // responses to cache revalidation requests. + // + // https://bugs.chromium.org/p/chromium/issues/detail?id=1214563#c2 + // + // The random query string parameter ensures cached responses are never used. + // + // Note: This issue does not apply to gs+xml because with the XML API, the + // Access-Control-Allow-Origin response header does not vary with the Origin. + // + // 2. If the object does not prohibit caching (e.g. public bucket and default + // `cache-control` metadata value), GCS may return stale responses. + return ( + `https://storage.googleapis.com/storage/v1/b/${this.bucket}/o/` + + `${encodeURIComponent(key)}?alt=media` + + `&neuroglancer=${getRandomHexString()}` + ); + } + + stat(key: string, options: StatOptions): Promise { + return stat(this, key, this.getObjectUrl(key), options, this.fetchOkImpl); + } + + read( + key: string, + options: DriverReadOptions, + ): Promise { + return read(this, key, this.getObjectUrl(key), options, this.fetchOkImpl); + } + + async list( + prefix: string, + options: DriverListOptions, + ): Promise { + const { progressListener } = options; + using _span = + progressListener === undefined + ? undefined + : new ProgressSpan(progressListener, { + message: `Listing prefix ${this.getUrl(prefix)}`, + }); + const delimiter = "/"; + // Include `neuroglancerOrigin` query parameter that is ignored by GCS to + // workaround + // https://bugs.chromium.org/p/chromium/issues/detail?id=1214563#c2 (though + // it is not clear it would ever apply to bucket listing). + const response = await this.fetchOkImpl( + `https://storage.googleapis.com/storage/v1/b/${this.bucket}/o?` + + `delimiter=${encodeURIComponent(delimiter)}&prefix=${encodeURIComponent( + prefix, + )}&` + + `neuroglancerOrigin=${encodeURIComponent(location.origin)}`, + { + signal: options.signal, + progressListener: options.progressListener, + }, + ); + const responseJson = await response.json(); + + verifyObject(responseJson); + const directories = verifyOptionalObjectProperty( + responseJson, + "prefixes", + verifyStringArray, + [], + ).map((prefix) => prefix.substring(0, prefix.length - 1)); + + const entries = verifyOptionalObjectProperty( + responseJson, + "items", + (items) => + parseArray(items, (item) => { + verifyObject(item); + return verifyObjectProperty(item, "name", verifyString); + }), + [], + ) + .filter((name) => !name.endsWith("_$folder$")) + .map((name) => ({ key: name })); + + return { + directories, + entries, + }; + } + + getUrl(path: string) { + return this.baseUrlForDisplay + encodePathForUrl(path); + } + + get supportsOffsetReads() { + return true; + } + get supportsSuffixReads() { + return true; + } +} diff --git a/src/kvstore/gcs/register.ts b/src/kvstore/gcs/register.ts new file mode 100644 index 0000000000..4eb612bf77 --- /dev/null +++ b/src/kvstore/gcs/register.ts @@ -0,0 +1,45 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import pythonIntegration from "#python_integration_build"; +import type { BaseKvStoreProvider } from "#src/kvstore/context.js"; +import { GcsKvStore } from "#src/kvstore/gcs/index.js"; +import type { SharedKvStoreContextBase } from "#src/kvstore/register.js"; +import { frontendBackendIsomorphicKvStoreProviderRegistry } from "#src/kvstore/register.js"; + +function gcsProvider(_context: SharedKvStoreContextBase): BaseKvStoreProvider { + return { + scheme: "gs", + description: pythonIntegration + ? "Google Cloud Storage" + : "Google Cloud Storage (anonymous)", + getKvStore(url) { + const m = (url.suffix ?? "").match(/^\/\/([^/]+)(\/.*)?$/); + if (m === null) { + throw new Error("Invalid URL, expected `gs:///`"); + } + const [, bucket, path] = m; + return { + store: new GcsKvStore(bucket), + path: decodeURIComponent((path ?? "").substring(1)), + }; + }, + }; +} + +frontendBackendIsomorphicKvStoreProviderRegistry.registerBaseKvStoreProvider( + gcsProvider, +); diff --git a/src/kvstore/gzip/file_handle.ts b/src/kvstore/gzip/file_handle.ts new file mode 100644 index 0000000000..c12f83882c --- /dev/null +++ b/src/kvstore/gzip/file_handle.ts @@ -0,0 +1,235 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { + DriverReadOptions, + FileHandle, + ReadableKvStore, + ReadResponse, + StatOptions, + StatResponse, +} from "#src/kvstore/index.js"; +import { decodeGzipStream } from "#src/util/gzip.js"; + +export const EXPECTED_HEADER_OVERHEAD = 100; + +export class GzipFileHandle + implements FileHandle +{ + constructor( + public base: BaseHandle, + public format: CompressionFormat, + ) {} + + async stat(options: StatOptions): Promise { + await this.base.stat(options); + return { totalSize: undefined }; + } + + async read(options: DriverReadOptions): Promise { + const { byteRange } = options; + if (byteRange === undefined) { + const readResponse = await this.base.read(options); + if (readResponse === undefined) return undefined; + return { + response: new Response( + decodeGzipStream(readResponse.response, this.format), + ), + offset: 0, + length: undefined, + totalSize: undefined, + }; + } + if ("suffixLength" in byteRange || byteRange.offset !== 0) { + throw new Error( + `Byte range with offset not supported: ${JSON.stringify(byteRange)}`, + ); + } + + // There is no way to force a flush on a `DecompressionStream`; to ensure we + // have all available output from the input, we must close the readable + // stream, which prevents any further writes. This means if more input is + // required, we have to redo the decode. In almost all cases, though, + // `EXPECTED_HEADER_OVERHEAD` should be sufficient and it won't be necessary + // to fetch additional encoded data. + let decodedArray = new Uint8Array(byteRange.length); + const parts: Uint8Array[] = []; + let encodedBytesReceived = 0; + let expectedEncodedBytes = byteRange.length + EXPECTED_HEADER_OVERHEAD; + while (true) { + const readResponse = await this.base.read({ + ...options, + byteRange: { + offset: encodedBytesReceived, + length: expectedEncodedBytes - encodedBytesReceived, + }, + }); + if (readResponse === undefined) return undefined; + { + const part = new Uint8Array(await readResponse.response.arrayBuffer()); + parts.push(part); + encodedBytesReceived += part.length; + } + + const decompressionStream = new DecompressionStream("gzip"); + const writer = decompressionStream.writable.getWriter(); + const writePromises: Promise[] = []; + for (const part of parts) { + writePromises.push(writer.write(part)); + } + writePromises.push(writer.close()); + const reader = decompressionStream.readable.getReader(); + let decodedOffset = 0; + try { + while (decodedOffset < decodedArray.length) { + let { value } = await reader.read(); + if (value === undefined) { + // no more decoded data available + break; + } + const remainingLength = decodedArray.length - decodedOffset; + if (value.length > remainingLength) { + value = value.subarray(0, remainingLength); + } + decodedArray.set(value, decodedOffset); + decodedOffset += value.length; + } + + if ( + decodedOffset === decodedArray.length || + encodedBytesReceived === readResponse.totalSize + ) { + if (decodedOffset < decodedArray.length) { + decodedArray = decodedArray.subarray(0, decodedOffset); + } + return { + response: new Response(decodedArray), + offset: 0, + length: decodedArray.length, + totalSize: undefined, + }; + } + } finally { + await reader.cancel(); + await Promise.allSettled(writePromises); + } + + expectedEncodedBytes += Math.min( + 100, + decodedArray.length - decodedOffset, + ); + } + } + + getUrl() { + return this.base.getUrl() + "|gzip"; + } +} + +export async function gzipRead( + base: ReadableKvStore, + baseKey: Key, + format: CompressionFormat, + options: DriverReadOptions, +) { + const { byteRange } = options; + if (byteRange === undefined) { + const readResponse = await base.read(baseKey, options); + if (readResponse === undefined) return undefined; + return { + response: new Response(decodeGzipStream(readResponse.response, format)), + offset: 0, + length: undefined, + totalSize: undefined, + }; + } + if ("suffixLength" in byteRange || byteRange.offset !== 0) { + throw new Error( + `Byte range with offset not supported: ${JSON.stringify(byteRange)}`, + ); + } + + // There is no way to force a flush on a `DecompressionStream`; to ensure we + // have all available output from the input, we must close the readable + // stream, which prevents any further writes. This means if more input is + // required, we have to redo the decode. In almost all cases, though, + // `EXPECTED_HEADER_OVERHEAD` should be sufficient and it won't be necessary + // to fetch additional encoded data. + let decodedArray = new Uint8Array(byteRange.length); + const parts: Uint8Array[] = []; + let encodedBytesReceived = 0; + let expectedEncodedBytes = byteRange.length + EXPECTED_HEADER_OVERHEAD; + while (true) { + const readResponse = await base.read(baseKey, { + ...options, + byteRange: { + offset: encodedBytesReceived, + length: expectedEncodedBytes - encodedBytesReceived, + }, + }); + if (readResponse === undefined) return undefined; + { + const part = new Uint8Array(await readResponse.response.arrayBuffer()); + parts.push(part); + encodedBytesReceived += part.length; + } + + const decompressionStream = new DecompressionStream("gzip"); + const writer = decompressionStream.writable.getWriter(); + const writePromises: Promise[] = []; + for (const part of parts) { + writePromises.push(writer.write(part)); + } + writePromises.push(writer.close()); + const reader = decompressionStream.readable.getReader(); + let decodedOffset = 0; + try { + while (decodedOffset < decodedArray.length) { + let { value } = await reader.read(); + if (value === undefined) { + // no more decoded data available + break; + } + const remainingLength = decodedArray.length - decodedOffset; + if (value.length > remainingLength) { + value = value.subarray(0, remainingLength); + } + decodedArray.set(value, decodedOffset); + decodedOffset += value.length; + } + + if ( + decodedOffset === decodedArray.length || + encodedBytesReceived === readResponse.totalSize + ) { + if (decodedOffset < decodedArray.length) { + decodedArray = decodedArray.subarray(0, decodedOffset); + } + return { + response: new Response(decodedArray), + offset: 0, + length: decodedArray.length, + totalSize: undefined, + }; + } + } finally { + await reader.cancel(); + await Promise.allSettled(writePromises); + } + + expectedEncodedBytes += Math.min(100, decodedArray.length - decodedOffset); + } +} diff --git a/src/kvstore/gzip/index.ts b/src/kvstore/gzip/index.ts new file mode 100644 index 0000000000..741e67d21c --- /dev/null +++ b/src/kvstore/gzip/index.ts @@ -0,0 +1,96 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { + AutoDetectFileOptions, + AutoDetectMatch, + AutoDetectRegistry, +} from "#src/kvstore/auto_detect.js"; +import { GzipFileHandle } from "#src/kvstore/gzip/file_handle.js"; +import type { + DriverReadOptions, + KvStore, + ReadResponse, + StatOptions, + StatResponse, + FileHandle, +} from "#src/kvstore/index.js"; +import { isGzipFormat } from "#src/util/gzip.js"; + +export class GzipKvStore implements KvStore { + constructor( + public base: FileHandle, + public scheme: string, + public format: CompressionFormat, + ) {} + + getUrl(key: string) { + this.validatePath(key); + return this.base.getUrl() + `|${this.scheme}`; + } + + private validatePath(path: string) { + if (path) { + throw new Error( + `"${this.scheme}:" does not support non-empty path ${JSON.stringify(path)}`, + ); + } + } + + async stat( + key: string, + options: StatOptions, + ): Promise { + this.validatePath(key); + await this.base.stat(options); + return { totalSize: undefined }; + } + + async read( + key: string, + options: DriverReadOptions, + ): Promise { + this.validatePath(key); + return new GzipFileHandle(this.base, this.format).read(options); + } + + get supportsOffsetReads() { + return false; + } + get supportsSuffixReads() { + return false; + } + get singleKey() { + return true; + } +} + +async function detectGzip( + options: AutoDetectFileOptions, +): Promise { + if (!isGzipFormat(options.prefix)) { + return []; + } + return [{ suffix: "gzip:", description: "gzip-compressed" }]; +} + +export function registerAutoDetect(registry: AutoDetectRegistry) { + registry.registerFileFormat({ + prefixLength: 3, + suffixLength: 0, + match: detectGzip, + }); +} diff --git a/src/kvstore/gzip/register.ts b/src/kvstore/gzip/register.ts new file mode 100644 index 0000000000..2c5357ce53 --- /dev/null +++ b/src/kvstore/gzip/register.ts @@ -0,0 +1,50 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { KvStoreAdapterProvider } from "#src/kvstore/context.js"; +import { GzipKvStore, registerAutoDetect } from "#src/kvstore/gzip/index.js"; +import { KvStoreFileHandle } from "#src/kvstore/index.js"; +import { frontendBackendIsomorphicKvStoreProviderRegistry } from "#src/kvstore/register.js"; +import { ensureEmptyUrlSuffix } from "#src/kvstore/url.js"; + +function gzipProvider( + scheme: string, + format: CompressionFormat, +): KvStoreAdapterProvider { + return { + scheme, + description: `transparent ${scheme} decoding`, + getKvStore(url, base) { + ensureEmptyUrlSuffix(url); + return { + store: new GzipKvStore( + new KvStoreFileHandle(base.store, base.path), + scheme, + format, + ), + path: "", + }; + }, + }; +} + +frontendBackendIsomorphicKvStoreProviderRegistry.registerKvStoreAdapterProvider( + () => gzipProvider("gzip", "gzip"), +); + +registerAutoDetect( + frontendBackendIsomorphicKvStoreProviderRegistry.autoDetectRegistry, +); diff --git a/src/kvstore/http/html_directory_listing.ts b/src/kvstore/http/html_directory_listing.ts new file mode 100644 index 0000000000..06de731833 --- /dev/null +++ b/src/kvstore/http/html_directory_listing.ts @@ -0,0 +1,114 @@ +/** + * @license + * Copyright 2019 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { + ListEntry, + DriverListOptions, + ListResponse, +} from "#src/kvstore/index.js"; +import { extractQueryAndFragment } from "#src/kvstore/url.js"; +import type { FetchOk } from "#src/util/http_request.js"; +import { fetchOk } from "#src/util/http_request.js"; +import { + ProgressSpan, + type ProgressOptions, +} from "#src/util/progress_listener.js"; + +/** + * Obtains a directory listing from a server that supports HTML directory listings. + */ +export async function getHtmlDirectoryListing( + url: string, + options: { + fetchOkImpl?: FetchOk; + } & Partial = {}, +): Promise { + const baseUrl = extractQueryAndFragment(url).base; + const { fetchOkImpl = fetchOk, signal, progressListener } = options; + const response = await fetchOkImpl( + url, + /*init=*/ { + headers: { accept: "text/html" }, + signal: signal, + progressListener, + }, + ); + const contentType = response.headers.get("content-type"); + if (contentType === null || /\btext\/html\b/i.exec(contentType) === null) { + return []; + } + const text = await response.text(); + const doc = new DOMParser().parseFromString(text, "text/html"); + const nodes = doc.evaluate( + "//a/@href", + doc, + null, + XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE, + null, + ); + const results = new Set(); + for (let i = 0, n = nodes.snapshotLength; i < n; ++i) { + const node = nodes.snapshotItem(i)!; + const href = node.textContent; + if (href === null) continue; + const withoutQuery = extractQueryAndFragment(href).base; + if (withoutQuery) { + const resolvedUrl = new URL(withoutQuery, baseUrl).toString(); + if (resolvedUrl.startsWith(baseUrl) && resolvedUrl !== baseUrl) { + results.add(resolvedUrl); + } + } + } + return Array.from(results); +} + +export async function listFromHtmlDirectoryListing( + baseUrl: string, + prefix: string, + options: DriverListOptions, +): Promise { + const { progressListener } = options; + using _span = + progressListener && + new ProgressSpan(progressListener, { + message: `Requesting HTML directory listing for ${baseUrl}`, + }); + const { base, queryAndFragment } = extractQueryAndFragment(baseUrl); + const baseAndPrefix = base + prefix; + const fullUrl = baseAndPrefix + queryAndFragment; + const m = fullUrl.match(/^([a-z]+:\/\/.*\/)([^/?#]*)$/); + if (m === null) throw null; + const [, directoryUrl] = m; + const listing = await getHtmlDirectoryListing( + directoryUrl + queryAndFragment, + { + signal: options.signal, + progressListener: options.progressListener, + }, + ); + const entries: ListEntry[] = []; + const directories: string[] = []; + for (const entry of listing) { + if (!entry.startsWith(baseAndPrefix)) continue; + const p = decodeURIComponent(entry.substring(base.length)); + if (p.endsWith("/")) { + directories.push(p.substring(0, p.length - 1)); + } else { + entries.push({ key: p }); + } + } + return { entries, directories }; +} diff --git a/src/kvstore/http/index.ts b/src/kvstore/http/index.ts new file mode 100644 index 0000000000..4d4e1caacd --- /dev/null +++ b/src/kvstore/http/index.ts @@ -0,0 +1,95 @@ +/** + * @license + * Copyright 2023 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { listFromHtmlDirectoryListing } from "#src/kvstore/http/html_directory_listing.js"; +import { read, stat } from "#src/kvstore/http/read.js"; +import type { + KvStore, + DriverListOptions, + ListResponse, + DriverReadOptions, + ReadResponse, + StatOptions, + StatResponse, +} from "#src/kvstore/index.js"; +import { extractQueryAndFragment } from "#src/kvstore/url.js"; +import type { FetchOk } from "#src/util/http_request.js"; +import { fetchOk } from "#src/util/http_request.js"; + +function joinBaseUrlAndPath(baseUrl: string, path: string) { + const { base, queryAndFragment } = extractQueryAndFragment(baseUrl); + return base + path + queryAndFragment; +} + +export class HttpKvStore implements KvStore { + constructor( + public baseUrl: string, + public baseUrlForDisplay: string = baseUrl, + public fetchOkImpl: FetchOk = fetchOk, + ) {} + + stat(key: string, options: StatOptions): Promise { + return stat( + this, + key, + joinBaseUrlAndPath(this.baseUrl, key), + options, + this.fetchOkImpl, + ); + } + + read( + key: string, + options: DriverReadOptions, + ): Promise { + return read( + this, + key, + joinBaseUrlAndPath(this.baseUrl, key), + options, + this.fetchOkImpl, + ); + } + + list(prefix: string, options: DriverListOptions): Promise { + return listFromHtmlDirectoryListing(this.baseUrl, prefix, options); + } + + getUrl(path: string) { + return joinBaseUrlAndPath(this.baseUrlForDisplay, path); + } + + get supportsOffsetReads() { + return true; + } + get supportsSuffixReads() { + return true; + } +} + +export function getBaseUrlAndPath(url: string) { + const parsed = new URL(url); + if (parsed.hash) { + throw new Error("fragment not supported"); + } + if (parsed.username || parsed.password) { + throw new Error("basic auth credentials not supported"); + } + return { + baseUrl: `${parsed.origin}/${parsed.search}`, + path: decodeURI(parsed.pathname.substring(1)), + }; +} diff --git a/src/kvstore/http/read.ts b/src/kvstore/http/read.ts new file mode 100644 index 0000000000..721dd98a5d --- /dev/null +++ b/src/kvstore/http/read.ts @@ -0,0 +1,282 @@ +/** + * @license + * Copyright 2023 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { composeByteRangeRequest } from "#src/kvstore/byte_range/file_handle.js"; +import type { + ByteRange, + DriverReadOptions, + ReadableKvStore, + ReadResponse, + StatOptions, + StatResponse, +} from "#src/kvstore/index.js"; +import { KvStoreFileHandle, NotFoundError } from "#src/kvstore/index.js"; +import type { FetchOk } from "#src/util/http_request.js"; +import { fetchOk, HttpError, isNotFoundError } from "#src/util/http_request.js"; +import type { ProgressListener } from "#src/util/progress_listener.js"; + +function getRangeHeader(request: ByteRange | undefined): string | undefined { + if (request === undefined) return undefined; + return `bytes=${request.offset}-${request.offset + request.length - 1}`; +} + +/** + * On Chromium, multiple concurrent byte range requests to the same URL are serialized unless the + * cache is disabled. Disabling the cache works around the problem. + * + * https://bugs.chromium.org/p/chromium/issues/detail?id=969828 + */ +const byteRangeCacheMode = + navigator.userAgent.indexOf("Chrome") !== -1 ? "no-store" : "default"; + +function wasRedirectedToDirectoryListing(url: string, response: Response) { + return new URL(url).pathname + "/" === new URL(response.url).pathname; +} + +function parse206ContentRangeHeader(contentRange: string) { + const m = contentRange.match(/bytes ([0-9]+)-([0-9]+)\/([0-9]+|\*)/); + if (m === null) { + throw new Error( + `Invalid content-range header: ${JSON.stringify(contentRange)}`, + ); + } + const offset = parseInt(m[1], 10); + const endPos = parseInt(m[2], 10); + let totalSize: number | undefined; + if (m[3] !== "*") { + totalSize = parseInt(m[3], 10); + } + const length = endPos - offset + 1; + return { offset, length, totalSize }; +} + +export async function read( + store: ReadableKvStore, + key: Key, + url: string, + options: DriverReadOptions, + fetchOkImpl: FetchOk = fetchOk, +): Promise { + let resolvedByteRange: ByteRange | undefined; + try { + const { byteRange: byteRangeRequest } = options; + let rangeHeader: string | undefined; + // The HTTP spec supports suffixLength requests directly via "Range: + // bytes=-N" requests, which avoids the need for a separate HEAD request. + // However, per + // https://fetch.spec.whatwg.org/#cors-safelisted-request-header a suffix + // length byte range request header will always trigger an OPTIONS preflight + // request, which would otherwise be avoided. This negates the benefit of + // using a suffixLength request directly. Additionally, some servers such as + // the npm http-server package and https://uk1s3.embassy.ebi.ac.uk/ do not + // correctly handle suffixLength requests or do not correctly handle CORS + // preflight requests. To avoid those issues, always just issue a separate + // HEAD request to determine the length. + if (byteRangeRequest !== undefined) { + if ("suffixLength" in byteRangeRequest) { + const statResponse = await stat(store, key, url, options, fetchOkImpl); + if (statResponse === undefined) return undefined; + const { totalSize } = statResponse; + if (totalSize === undefined) { + throw new Error( + `Failed to determine total size of ${store.getUrl(key)} in order to fetch suffix ${JSON.stringify(byteRangeRequest)}`, + ); + } + resolvedByteRange = composeByteRangeRequest( + { offset: 0, length: totalSize }, + byteRangeRequest, + ).outer; + if (resolvedByteRange.length === 0) { + // Skip zero-byte read, since totalSize is already known. + return { + ...resolvedByteRange, + totalSize, + response: new Response(new Uint8Array(0)), + }; + } + rangeHeader = getRangeHeader(resolvedByteRange); + } else { + resolvedByteRange = byteRangeRequest; + if (resolvedByteRange.length === 0) { + // The HTTP range header does not support zero-length byte range + // requests. + // + // Convert zero-length byte range to length-1 byte range, and then + // discard the response. If the requested offset is 0, and the file is + // empty, then this will result in a 416 Range Not Satisfiable + // response. + rangeHeader = getRangeHeader({ + offset: Math.max(resolvedByteRange.offset - 1, 0), + length: 1, + }); + } else { + rangeHeader = getRangeHeader(resolvedByteRange); + } + } + } + const requestInit: RequestInit & { progressListener?: ProgressListener } = { + signal: options.signal, + progressListener: options.progressListener, + }; + if (rangeHeader !== undefined) { + requestInit.headers = { range: rangeHeader }; + requestInit.cache = byteRangeCacheMode; + } + let response = await fetchOkImpl(url, requestInit); + if (wasRedirectedToDirectoryListing(url, response)) { + return undefined; + } + let offset: number | undefined; + let length: number | undefined; + let totalSize: number | undefined; + if (response.status === 206) { + const contentRange = response.headers.get("content-range"); + if (contentRange === null) { + // Content-range should always be sent, but some buggy servers don't + // send it. + if (resolvedByteRange !== undefined) { + offset = resolvedByteRange.offset; + } else { + throw new Error( + "Unexpected HTTP 206 response when no byte range specified.", + ); + } + } + if (contentRange !== null) { + ({ offset, length, totalSize } = + parse206ContentRangeHeader(contentRange)); + } + } else { + length = totalSize = getBodyLength(response.headers); + } + if (offset === undefined) { + offset = 0; + } + if (length === undefined) { + length = getBodyLength(response.headers); + } + if (resolvedByteRange?.length === 0) { + response = new Response(new Uint8Array(0)); + offset = resolvedByteRange.offset; + length = 0; + } + return { + response, + offset, + length, + totalSize, + }; + } catch (e) { + if ( + e instanceof HttpError && + e.status === 416 && + resolvedByteRange?.length === 0 && + resolvedByteRange.offset === 0 + ) { + return { + response: new Response(new Uint8Array(0)), + offset: 0, + length: 0, + totalSize: 0, + }; + } + return handleThrowIfMissing(store, key, options, e); + } +} + +function getBodyLength(headers: Headers): number | undefined { + const contentLength = headers.get("content-length"); + const contentEncoding = headers.get("content-encoding"); + if (contentEncoding === null && contentLength !== null) { + const size = Number(contentLength); + if (!Number.isFinite(size) || size < 0) { + throw new Error(`Invalid content-length: {contentLength}`); + } + return size; + } + return undefined; +} + +function handleThrowIfMissing( + store: ReadableKvStore, + key: Key, + options: { throwIfMissing?: boolean }, + error: unknown, +) { + if (isNotFoundError(error)) { + if (options.throwIfMissing === true) { + throw new NotFoundError(new KvStoreFileHandle(store, key), { + cause: error, + }); + } + return undefined; + } + throw error; +} + +export async function stat( + store: ReadableKvStore, + key: Key, + url: string, + options: StatOptions, + fetchOkImpl: FetchOk = fetchOk, +): Promise { + // First try HEAD request. + try { + const response = await fetchOkImpl(url, { + method: "HEAD", + signal: options.signal, + progressListener: options.progressListener, + }); + if (wasRedirectedToDirectoryListing(url, response)) return undefined; + return { totalSize: getBodyLength(response.headers) }; + } catch (e) { + if (e instanceof HttpError && e.status === 405 /* method not allowed */) { + // HEAD may not be supported, use GET with one byte range instead. + // + // For example, + // https://data-proxy.ebrains.eu/api/v1/buckets/localizoom/14122_mPPC_BDA_s186.tif/14122_mPPC_BDA_s186.dzi + // returns HTTP 405 Method Not Allowed in response to HEAD requests. + } else { + return handleThrowIfMissing(store, key, options, e); + } + } + + // Try GET with one-byte range instead. + try { + const response = await fetchOkImpl(url, { + signal: options.signal, + progressListener: options.progressListener, + headers: { range: "bytes=0-0" }, + }); + if (wasRedirectedToDirectoryListing(url, response)) return undefined; + let totalSize: number | undefined; + if (response.status === 200) { + totalSize = getBodyLength(response.headers); + } else { + const contentRange = response.headers.get("content-range"); + if (contentRange !== null) { + ({ totalSize } = parse206ContentRangeHeader(contentRange)); + } + } + return { totalSize }; + } catch (e) { + if (e instanceof HttpError && e.status === 416) { + return { totalSize: 0 }; + } + return handleThrowIfMissing(store, key, options, e); + } +} diff --git a/src/kvstore/http/register.ts b/src/kvstore/http/register.ts new file mode 100644 index 0000000000..3b593a3148 --- /dev/null +++ b/src/kvstore/http/register.ts @@ -0,0 +1,49 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { BaseKvStoreProvider } from "#src/kvstore/context.js"; +import { getBaseUrlAndPath, HttpKvStore } from "#src/kvstore/http/index.js"; +import type { SharedKvStoreContextBase } from "#src/kvstore/register.js"; +import { frontendBackendIsomorphicKvStoreProviderRegistry } from "#src/kvstore/register.js"; + +function httpProvider( + scheme: string, + _context: SharedKvStoreContextBase, +): BaseKvStoreProvider { + return { + scheme, + description: `${scheme} (unauthenticated)`, + getKvStore(url) { + try { + const { baseUrl, path } = getBaseUrlAndPath(url.url); + return { + store: new HttpKvStore(baseUrl), + path, + }; + } catch (e) { + throw new Error(`Invalid URL ${JSON.stringify(url.url)}`, { + cause: e, + }); + } + }, + }; +} + +for (const protocol of ["http", "https"]) { + frontendBackendIsomorphicKvStoreProviderRegistry.registerBaseKvStoreProvider( + (context) => httpProvider(protocol, context), + ); +} diff --git a/src/kvstore/index.ts b/src/kvstore/index.ts index 3bd16631b7..980239a303 100644 --- a/src/kvstore/index.ts +++ b/src/kvstore/index.ts @@ -14,38 +14,13 @@ * limitations under the License. */ +import type { ProgressOptions } from "#src/util/progress_listener.js"; + export interface ByteRange { offset: number; length: number; } -export function composeByteRangeRequest( - outer: ByteRange, - inner: ByteRangeRequest | undefined, -): { outer: ByteRange; inner: ByteRange } { - if (inner === undefined) { - return { outer, inner: { offset: 0, length: outer.length } }; - } - if ("suffixLength" in inner) { - const length = Math.min(outer.length, inner.suffixLength); - return { - outer: { offset: outer.offset + (outer.length - length), length }, - inner: { offset: outer.length - length, length }, - }; - } - if (inner.offset + inner.length > outer.length) { - throw new Error( - `Requested byte range ${JSON.stringify( - inner, - )} not valid for value of length ${outer.length}`, - ); - } - return { - outer: { offset: outer.offset + inner.offset, length: inner.length }, - inner, - }; -} - export type ByteRangeRequest = | ByteRange | { @@ -53,19 +28,33 @@ export type ByteRangeRequest = }; export interface ReadResponse { - data: Uint8Array; - dataRange: ByteRange; + response: Response; + offset: number; + length: number | undefined; totalSize: number | undefined; } -export interface ReadOptions { +export interface DriverReadOptions extends Partial { byteRange?: ByteRangeRequest; - abortSignal?: AbortSignal; + throwIfMissing?: boolean; } -export interface ListOptions { - prefix: string; - abortSignal?: AbortSignal; +export class NotFoundError extends Error { + constructor(handle: FileHandle, options?: { cause: any }) { + super(`${handle.getUrl()} not found`, options); + } +} + +export interface ReadOptions extends DriverReadOptions { + strictByteRange?: boolean; +} + +export type DriverListOptions = Partial; + +export type ListResponseKeyKind = "path" | "suffix" | "url"; + +export interface ListOptions extends DriverListOptions { + responseKeys?: ListResponseKeyKind; } export interface ListEntry { @@ -77,12 +66,204 @@ export interface ListResponse { directories: string[]; } +export interface StatOptions extends Partial { + throwIfMissing?: boolean; +} + +export interface StatResponse { + totalSize: number | undefined; +} + export interface ReadableKvStore { - read(key: Key, options: ReadOptions): Promise; + stat(key: Key, options: StatOptions): Promise; + read(key: Key, options: DriverReadOptions): Promise; + getUrl(key: Key): string; + + // Reads with non-zero byte offset are supported. + supportsOffsetReads: boolean; + + // Reads with `suffixLength` byte range are supported. + supportsSuffixReads: boolean; } export interface ListableKvStore { - list(options: ListOptions): Promise; + list?: (prefix: string, options: DriverListOptions) => Promise; } -export interface KvStore extends ReadableKvStore, ListableKvStore {} +export interface KvStore extends ReadableKvStore, ListableKvStore { + // Indicates that the only valid key is the empty string. + singleKey?: boolean; +} + +export interface KvStoreWithPath { + store: KvStore; + path: string; +} + +export function getKvStoreUrl(kvstore: KvStoreWithPath): string { + return kvstore.store.getUrl(kvstore.path); +} + +export function readKvStore( + store: ReadableKvStore, + key: Key, + options: ReadOptions & { throwIfMissing: true }, +): Promise; + +export function readKvStore( + store: ReadableKvStore, + key: Key, + options?: ReadOptions, +): Promise; + +export async function readKvStore( + store: ReadableKvStore, + key: Key, + options: ReadOptions = {}, +): Promise { + return readFileHandle(new KvStoreFileHandle(store, key), options); +} + +export function readFileHandle( + handle: FileHandle, + options: ReadOptions & { throwIfMissing: true }, +): Promise; + +export function readFileHandle( + handle: FileHandle, + options?: ReadOptions, +): Promise; + +export async function readFileHandle( + handle: FileHandle, + options: ReadOptions = {}, +): Promise { + const response = await handle.read(options); + if (options?.throwIfMissing === true) { + if (response === undefined) { + throw new NotFoundError(handle); + } + } + if (options?.strictByteRange === true && response !== undefined) { + const { byteRange } = options; + const { offset, length } = response; + if (byteRange !== undefined) { + if ( + "suffixLength" in byteRange + ? length !== byteRange.suffixLength + : offset !== byteRange.offset || + (length !== undefined && length !== byteRange.length) + ) { + throw new Error( + `Received truncated response for ${handle.getUrl()}, expected ${JSON.stringify( + byteRange, + )} but received offset=${offset}, length=${length}`, + ); + } + } + } + return response; +} + +function transformListResponse( + response: ListResponse, + prefix: string, + kvStore: KvStore, + responseKeys?: ListResponseKeyKind, +) { + switch (responseKeys) { + case "suffix": { + const offset = prefix.length; + return { + directories: response.directories.map((key) => key.substring(offset)), + entries: response.entries.map(({ key, ...entry }) => ({ + ...entry, + key: key.substring(offset), + })), + }; + } + case "url": { + return { + directories: response.directories.map((key) => kvStore.getUrl(key)), + entries: response.entries.map(({ key, ...entry }) => ({ + ...entry, + key: kvStore.getUrl(key), + })), + }; + } + default: { + return response; + } + } +} + +export async function listKvStore( + kvStore: KvStore, + prefix: string, + options: ListOptions = {}, +): Promise { + if (!kvStore.list) { + throw new Error("Listing not supported"); + } + return transformListResponse( + await kvStore.list(prefix, options), + prefix, + kvStore, + options.responseKeys, + ); +} + +export async function listKvStoreRecursively( + kvStore: KvStore, + prefix: string, + options: ListOptions = {}, +): Promise { + if (!kvStore.list) { + throw new Error("Listing not supported"); + } + const entries: ListEntry[] = []; + async function process(path: string) { + const response = await kvStore.list!(path, options); + entries.push(...response.entries); + await Promise.all(response.directories.map((name) => process(name + "/"))); + } + await process(prefix); + return transformListResponse( + { entries, directories: [] }, + prefix, + kvStore, + options.responseKeys, + ).entries; +} + +export function kvStoreAppendPath( + kvstore: KvStoreWithPath, + suffix: string, +): KvStoreWithPath { + return { store: kvstore.store, path: kvstore.path + suffix }; +} + +export interface FileHandle { + stat(options: StatOptions): Promise; + read(options: DriverReadOptions): Promise; + getUrl(): string; +} + +export class KvStoreFileHandle implements FileHandle { + constructor( + public store: ReadableKvStore, + public key: Key, + ) {} + + stat(options: StatOptions): Promise { + return this.store.stat(this.key, options); + } + + read(options: DriverReadOptions): Promise { + return this.store.read(this.key, options); + } + + getUrl() { + return this.store.getUrl(this.key); + } +} diff --git a/src/datasource/middleauth/credentials_provider.ts b/src/kvstore/middleauth/credentials_provider.ts similarity index 84% rename from src/datasource/middleauth/credentials_provider.ts rename to src/kvstore/middleauth/credentials_provider.ts index 719ed4738a..554ce79933 100644 --- a/src/datasource/middleauth/credentials_provider.ts +++ b/src/kvstore/middleauth/credentials_provider.ts @@ -34,6 +34,7 @@ import { verifyString, verifyStringArray, } from "#src/util/json.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; export type MiddleAuthToken = { tokenType: string; @@ -59,7 +60,7 @@ function openPopupCenter(url: string, width: number, height: number) { function waitForAuthResponseMessage( serverUrl: string, source: Window, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise { return new Promise((resolve, reject) => { window.addEventListener( @@ -91,17 +92,17 @@ function waitForAuthResponseMessage( console.error("Response received: ", event.data); } }, - { signal: abortSignal }, + { signal: signal }, ); }); } async function waitForLogin( serverUrl: string, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise { const abortController = new AbortController(); - abortSignal = AbortSignal.any([abortController.signal, abortSignal]); + signal = AbortSignal.any([abortController.signal, signal]); try { const newWindow = openPopupCenter( `${serverUrl}/api/v1/authorize`, @@ -114,7 +115,7 @@ async function waitForLogin( monitorAuthPopupWindow(newWindow, abortController); return await raceWithAbort( waitForAuthResponseMessage(serverUrl, newWindow, abortController.signal), - abortSignal, + signal, ); } finally { abortController.abort(); @@ -144,7 +145,7 @@ export class MiddleAuthCredentialsProvider extends CredentialsProvider { + get = makeCredentialsGetter(async (options) => { let token = undefined; if (!this.alreadyTriedLocalStorage) { @@ -153,13 +154,16 @@ export class MiddleAuthCredentialsProvider extends CredentialsProvider waitForLogin(this.serverUrl, abortSignal), + get: (signal) => waitForLogin(this.serverUrl, signal), }, - abortSignal, + options.signal, ); saveAuthTokenToLocalStorage(this.serverUrl, token); } @@ -188,16 +192,23 @@ export class MiddleAuthAppCredentialsProvider extends CredentialsProvider { - const authInfo = await fetch(`${this.serverUrl}/auth_info`).then((res) => - res.json(), - ); + get = makeCredentialsGetter(async (options) => { + let authInfo: any; + { + using _span = new ProgressSpan(options.progressListener, { + message: `Determining authentication server for ${this.serverUrl}`, + }); + const response = await fetch(`${this.serverUrl}/auth_info`, { + signal: options.signal, + }); + authInfo = await response.json(); + } const provider = this.credentialsManager.getCredentialsProvider( "middleauth", authInfo.login_url, ) as MiddleAuthCredentialsProvider; - this.credentials = await provider.get(this.credentials); + this.credentials = await provider.get(this.credentials, options); if (this.credentials.credentials.appUrls.includes(this.serverUrl)) { return this.credentials.credentials; diff --git a/src/kvstore/middleauth/register.ts b/src/kvstore/middleauth/register.ts new file mode 100644 index 0000000000..e772a05bb8 --- /dev/null +++ b/src/kvstore/middleauth/register.ts @@ -0,0 +1,74 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { + CredentialsManager, + CredentialsProvider, +} from "#src/credentials_provider/index.js"; +import type { OAuth2Credentials } from "#src/credentials_provider/oauth2.js"; +import { fetchOkWithOAuth2CredentialsAdapter } from "#src/credentials_provider/oauth2.js"; +import type { BaseKvStoreProvider } from "#src/kvstore/context.js"; +import { getBaseUrlAndPath, HttpKvStore } from "#src/kvstore/http/index.js"; +import type { SharedKvStoreContextBase } from "#src/kvstore/register.js"; +import { frontendBackendIsomorphicKvStoreProviderRegistry } from "#src/kvstore/register.js"; + +const SCHEME_PREFIX = "middleauth+"; + +function getMiddleAuthCredentialsProvider( + credentialsManager: CredentialsManager, + url: string, +): CredentialsProvider { + return credentialsManager.getCredentialsProvider( + "middleauthapp", + new URL(url).origin, + ); +} + +function middleauthProvider( + scheme: string, + context: SharedKvStoreContextBase, +): BaseKvStoreProvider { + return { + scheme: SCHEME_PREFIX + scheme, + description: `${scheme} with middleauth`, + getKvStore(url) { + const httpUrl = url.url.substring(SCHEME_PREFIX.length); + const credentialsProvider = getMiddleAuthCredentialsProvider( + context.credentialsManager, + httpUrl, + ); + try { + const { baseUrl, path } = getBaseUrlAndPath(httpUrl); + return { + store: new HttpKvStore( + baseUrl, + SCHEME_PREFIX + baseUrl, + fetchOkWithOAuth2CredentialsAdapter(credentialsProvider), + ), + path, + }; + } catch (e) { + throw new Error(`Invalid URL ${JSON.stringify(url.url)}`, { + cause: e, + }); + } + }, + }; +} + +frontendBackendIsomorphicKvStoreProviderRegistry.registerBaseKvStoreProvider( + (context) => middleauthProvider("https", context), +); diff --git a/src/datasource/middleauth/register_credentials_provider.ts b/src/kvstore/middleauth/register_credentials_provider.ts similarity index 75% rename from src/datasource/middleauth/register_credentials_provider.ts rename to src/kvstore/middleauth/register_credentials_provider.ts index 90f6a3b634..42ad01323e 100644 --- a/src/datasource/middleauth/register_credentials_provider.ts +++ b/src/kvstore/middleauth/register_credentials_provider.ts @@ -14,19 +14,19 @@ * limitations under the License. */ -import { defaultCredentialsManager } from "#src/credentials_provider/default_manager.js"; +import { registerDefaultCredentialsProvider } from "#src/credentials_provider/default_manager.js"; import type { CredentialsManager } from "#src/credentials_provider/index.js"; import { MiddleAuthCredentialsProvider, MiddleAuthAppCredentialsProvider, -} from "#src/datasource/middleauth/credentials_provider.js"; +} from "#src/kvstore/middleauth/credentials_provider.js"; -defaultCredentialsManager.register( +registerDefaultCredentialsProvider( "middleauth", - (serverUrl) => new MiddleAuthCredentialsProvider(serverUrl), + (serverUrl: string) => new MiddleAuthCredentialsProvider(serverUrl), ); -defaultCredentialsManager.register( +registerDefaultCredentialsProvider( "middleauthapp", (serverUrl: string, credentialsManager: CredentialsManager) => new MiddleAuthAppCredentialsProvider(serverUrl, credentialsManager), diff --git a/src/datasource/ngauth/README.md b/src/kvstore/ngauth/README.md similarity index 100% rename from src/datasource/ngauth/README.md rename to src/kvstore/ngauth/README.md diff --git a/src/datasource/ngauth/credentials_provider.ts b/src/kvstore/ngauth/credentials_provider.ts similarity index 71% rename from src/datasource/ngauth/credentials_provider.ts rename to src/kvstore/ngauth/credentials_provider.ts index ed24973eba..7d2208f0ee 100644 --- a/src/datasource/ngauth/credentials_provider.ts +++ b/src/kvstore/ngauth/credentials_provider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { fetchWithCredentials } from "#src/credentials_provider/http_request.js"; +import { fetchOkWithCredentials } from "#src/credentials_provider/http_request.js"; import { CredentialsProvider, makeCredentialsGetter, @@ -25,12 +25,13 @@ import { } from "#src/credentials_provider/interactive_credentials_provider.js"; import type { OAuth2Credentials } from "#src/credentials_provider/oauth2.js"; import { raceWithAbort } from "#src/util/abort.js"; -import { HttpError } from "#src/util/http_request.js"; +import { fetchOk, HttpError } from "#src/util/http_request.js"; import { verifyObject, verifyObjectProperty, verifyString, } from "#src/util/json.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; function makeOriginError(serverUrl: string): Error { return new Error( @@ -45,10 +46,10 @@ export interface Credentials { async function waitForLogin( serverUrl: string, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise { const abortController = new AbortController(); - abortSignal = AbortSignal.any([abortController.signal, abortSignal]); + signal = AbortSignal.any([abortController.signal, signal]); try { const newWindow = window.open( `${serverUrl}/login?origin=${encodeURIComponent(self.origin)}`, @@ -59,7 +60,7 @@ async function waitForLogin( monitorAuthPopupWindow(newWindow, abortController); return await raceWithAbort( waitForAuthResponseMessage(serverUrl, newWindow, abortController.signal), - abortSignal, + signal, ); } finally { abortController.abort(); @@ -69,7 +70,7 @@ async function waitForLogin( function waitForAuthResponseMessage( serverUrl: string, source: Window, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise { return new Promise((resolve, reject) => { window.addEventListener( @@ -102,7 +103,7 @@ function waitForAuthResponseMessage( ); } }, - { signal: abortSignal }, + { signal: signal }, ); }); } @@ -111,28 +112,34 @@ export class NgauthCredentialsProvider extends CredentialsProvider constructor(public serverUrl: string) { super(); } - get = makeCredentialsGetter(async (abortSignal) => { - const response = await fetch(`${this.serverUrl}/token`, { - method: "POST", - credentials: "include", - signal: abortSignal, + get = makeCredentialsGetter(async (options) => { + using _span = new ProgressSpan(options.progressListener, { + message: `Requesting ngauth login token from ${this.serverUrl}`, }); - switch (response.status) { - case 200: - return { token: await response.text() }; - case 401: - return await getCredentialsWithStatus( - { - description: `ngauth server ${this.serverUrl}`, - requestDescription: "login", - get: (abortSignal) => waitForLogin(this.serverUrl, abortSignal), - }, - abortSignal, - ); - case 403: - throw makeOriginError(this.serverUrl); - default: - throw HttpError.fromResponse(response); + try { + const response = await fetchOk(`${this.serverUrl}/token`, { + method: "POST", + credentials: "include", + signal: options.signal, + }); + return { token: await response.text() }; + } catch (e) { + if (e instanceof HttpError) { + switch (e.status) { + case 401: + return await getCredentialsWithStatus( + { + description: `ngauth server ${this.serverUrl}`, + requestDescription: "login", + get: (signal) => waitForLogin(this.serverUrl, signal), + }, + options.signal, + ); + case 403: + throw makeOriginError(this.serverUrl); + } + } + throw e; } }); } @@ -145,11 +152,14 @@ export class NgauthGcsCredentialsProvider extends CredentialsProvider { - const response = await fetchWithCredentials( + get = makeCredentialsGetter(async (options) => { + using _span = new ProgressSpan(options.progressListener, { + message: `Requesting access token for gs://${this.bucket} from ${this.serverUrl}`, + }); + const response = await fetchOkWithCredentials( this.ngauthCredentialsProvider, `${this.serverUrl}/gcs_token`, - { method: "POST" }, + { method: "POST", signal: options.signal }, (credentials, init) => { return { ...init, diff --git a/src/kvstore/ngauth/register.ts b/src/kvstore/ngauth/register.ts new file mode 100644 index 0000000000..a7ef1067cf --- /dev/null +++ b/src/kvstore/ngauth/register.ts @@ -0,0 +1,84 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import pythonIntegration from "#python_integration_build"; +import type { + CredentialsManager, + CredentialsProvider, +} from "#src/credentials_provider/index.js"; +import type { OAuth2Credentials } from "#src/credentials_provider/oauth2.js"; +import { fetchOkWithOAuth2CredentialsAdapter } from "#src/credentials_provider/oauth2.js"; +import type { BaseKvStoreProvider } from "#src/kvstore/context.js"; +import { GcsKvStore } from "#src/kvstore/gcs/index.js"; +import type { SharedKvStoreContextBase } from "#src/kvstore/register.js"; +import { frontendBackendIsomorphicKvStoreProviderRegistry } from "#src/kvstore/register.js"; + +function getNgauthCredentialsProvider( + credentialsManager: CredentialsManager, + authServer: string, + bucket: string, +): CredentialsProvider { + return pythonIntegration + ? credentialsManager.getCredentialsProvider("gcs", { bucket }) + : credentialsManager.getCredentialsProvider("ngauth_gcs", { + authServer: authServer, + bucket, + }); +} + +const SCHEME_PREFIX = "gs+ngauth+"; + +function gcsNgauthProvider( + scheme: string, + context: SharedKvStoreContextBase, +): BaseKvStoreProvider { + return { + scheme, + description: pythonIntegration + ? "Google Cloud Storage" + : "Google Cloud Storage (ngauth)", + getKvStore(url) { + const m = (url.suffix ?? "").match(/^\/\/([^/]+)\/([^/]+)(\/.*)?$/); + if (m === null) { + throw new Error( + `Invalid URL, expected ${url.scheme}:////`, + ); + } + const [, authHost, bucket, path] = m; + const authUrl = + url.scheme.substring(SCHEME_PREFIX.length) + "://" + authHost; + const credentialsProvider = getNgauthCredentialsProvider( + context.credentialsManager, + authUrl, + bucket, + ); + return { + store: new GcsKvStore( + bucket, + `${url.scheme}://${authHost}/${bucket}/`, + fetchOkWithOAuth2CredentialsAdapter(credentialsProvider), + ), + path: decodeURIComponent((path ?? "").substring(1)), + }; + }, + }; +} + +for (const scheme of ["http", "https"]) { + frontendBackendIsomorphicKvStoreProviderRegistry.registerBaseKvStoreProvider( + (context) => gcsNgauthProvider(`${SCHEME_PREFIX}${scheme}`, context), + ); +} diff --git a/src/datasource/ngauth/register_credentials_provider.ts b/src/kvstore/ngauth/register_credentials_provider.ts similarity index 83% rename from src/datasource/ngauth/register_credentials_provider.ts rename to src/kvstore/ngauth/register_credentials_provider.ts index d500e1ee54..ff7ce554f7 100644 --- a/src/datasource/ngauth/register_credentials_provider.ts +++ b/src/kvstore/ngauth/register_credentials_provider.ts @@ -14,18 +14,18 @@ * limitations under the License. */ -import { defaultCredentialsManager } from "#src/credentials_provider/default_manager.js"; +import { registerDefaultCredentialsProvider } from "#src/credentials_provider/default_manager.js"; import type { CredentialsManager } from "#src/credentials_provider/index.js"; import { NgauthCredentialsProvider, NgauthGcsCredentialsProvider, -} from "#src/datasource/ngauth/credentials_provider.js"; +} from "#src/kvstore/ngauth/credentials_provider.js"; -defaultCredentialsManager.register( +registerDefaultCredentialsProvider( "ngauth", (serverUrl) => new NgauthCredentialsProvider(serverUrl), ); -defaultCredentialsManager.register( +registerDefaultCredentialsProvider( "ngauth_gcs", ( parameters: { authServer: string; bucket: string }, diff --git a/src/kvstore/register.ts b/src/kvstore/register.ts new file mode 100644 index 0000000000..86f316b61d --- /dev/null +++ b/src/kvstore/register.ts @@ -0,0 +1,74 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { CredentialsManager } from "#src/credentials_provider/index.js"; +import { AutoDetectRegistry } from "#src/kvstore/auto_detect.js"; +import type { + BaseKvStoreProvider, + KvStoreAdapterProvider, + KvStoreContext, +} from "#src/kvstore/context.js"; + +export interface SharedKvStoreContextBase { + kvStoreContext: KvStoreContext; + credentialsManager: CredentialsManager; +} + +export class KvStoreProviderRegistry< + SharedKvStoreContext extends SharedKvStoreContextBase, +> { + baseKvStoreProviders: Array< + (context: SharedKvStoreContext) => BaseKvStoreProvider + > = []; + kvStoreAdapterProviders: Array< + (context: SharedKvStoreContext) => KvStoreAdapterProvider + > = []; + autoDetectRegistry = new AutoDetectRegistry(); + + registerBaseKvStoreProvider( + provider: (context: SharedKvStoreContext) => BaseKvStoreProvider, + ) { + this.baseKvStoreProviders.push(provider); + } + + registerKvStoreAdapterProvider( + provider: (context: SharedKvStoreContext) => KvStoreAdapterProvider, + ) { + this.kvStoreAdapterProviders.push(provider); + } + + applyToContext(context: SharedKvStoreContext) { + const { kvStoreContext } = context; + for (const key of [ + "baseKvStoreProviders", + "kvStoreAdapterProviders", + ] as const) { + const map = kvStoreContext[key]; + for (const providerFactory of this[key]) { + const provider = providerFactory(context); + const { scheme } = provider; + if (map.has(scheme)) { + throw new Error(`Duplicate kvstore scheme ${scheme}`); + } + map.set(scheme, provider as any); + } + } + this.autoDetectRegistry.copyTo(context.kvStoreContext.autoDetectRegistry); + } +} + +export const frontendBackendIsomorphicKvStoreProviderRegistry = + new KvStoreProviderRegistry(); diff --git a/src/kvstore/s3/index.ts b/src/kvstore/s3/index.ts new file mode 100644 index 0000000000..d412406174 --- /dev/null +++ b/src/kvstore/s3/index.ts @@ -0,0 +1,78 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { read, stat } from "#src/kvstore/http/read.js"; +import type { + KvStore, + DriverListOptions, + ListResponse, + DriverReadOptions, + ReadResponse, + StatOptions, + StatResponse, +} from "#src/kvstore/index.js"; +import { getS3BucketListing } from "#src/kvstore/s3/list.js"; +import { encodePathForUrl } from "#src/kvstore/url.js"; +import type { FetchOk } from "#src/util/http_request.js"; +import { fetchOk } from "#src/util/http_request.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; + +export class S3KvStore implements KvStore { + constructor( + public baseUrl: string, + public baseUrlForDisplay: string, + private fetchOkImpl: FetchOk = fetchOk, + ) {} + + stat(key: string, options: StatOptions): Promise { + const url = `${this.baseUrl}${key}`; + return stat(this, key, url, options, this.fetchOkImpl); + } + + read( + key: string, + options: DriverReadOptions, + ): Promise { + const url = `${this.baseUrl}${key}`; + return read(this, key, url, options, this.fetchOkImpl); + } + + list(prefix: string, options: DriverListOptions): Promise { + const { progressListener } = options; + using _span = + progressListener === undefined + ? undefined + : new ProgressSpan(progressListener, { + message: `Listing prefix ${this.getUrl(prefix)}`, + }); + return getS3BucketListing(this.baseUrl, prefix, { + fetchOkImpl: this.fetchOkImpl, + signal: options.signal, + progressListener, + }); + } + + getUrl(path: string) { + return this.baseUrlForDisplay + encodePathForUrl(path); + } + + get supportsOffsetReads() { + return true; + } + get supportsSuffixReads() { + return true; + } +} diff --git a/src/kvstore/s3/list.ts b/src/kvstore/s3/list.ts new file mode 100644 index 0000000000..3580689ddd --- /dev/null +++ b/src/kvstore/s3/list.ts @@ -0,0 +1,77 @@ +/** + * @license + * Copyright 2019 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ListEntry, ListResponse } from "#src/kvstore/index.js"; +import type { FetchOk } from "#src/util/http_request.js"; +import { fetchOk } from "#src/util/http_request.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; + +export async function getS3BucketListing( + bucketUrl: string, + prefix: string, + options: { + delimiter?: string; + fetchOkImpl?: FetchOk; + } & Partial = {}, +): Promise { + const { + delimiter = "/", + fetchOkImpl = fetchOk, + signal, + progressListener, + } = options; + const response = await fetchOkImpl( + `${bucketUrl}?prefix=${encodeURIComponent(prefix)}` + + `&delimiter=${encodeURIComponent(delimiter)}`, + /*init=*/ { + signal: signal, + progressListener, + }, + ); + const text = await response.text(); + const doc = new DOMParser().parseFromString(text, "application/xml"); + const namespaceResolver: XPathNSResolver = () => + "http://doc.s3.amazonaws.com/2006-03-01/"; + const commonPrefixNodes = doc.evaluate( + "//CommonPrefixes/Prefix", + doc, + namespaceResolver, + XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE, + null, + ); + const directories: string[] = []; + for (let i = 0, n = commonPrefixNodes.snapshotLength; i < n; ++i) { + const name = commonPrefixNodes.snapshotItem(i)!.textContent; + if (name === null) continue; + // Exclude delimiter from end of `name`. + directories.push(name.substring(0, name.length - delimiter.length)); + } + + const entries: ListEntry[] = []; + const contents = doc.evaluate( + "//Contents/Key", + doc, + namespaceResolver, + XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE, + null, + ); + for (let i = 0, n = contents.snapshotLength; i < n; ++i) { + const name = contents.snapshotItem(i)!.textContent; + if (name === null) continue; + entries.push({ key: name }); + } + return { directories, entries }; +} diff --git a/src/kvstore/s3/register.ts b/src/kvstore/s3/register.ts new file mode 100644 index 0000000000..b6545d5a01 --- /dev/null +++ b/src/kvstore/s3/register.ts @@ -0,0 +1,46 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { BaseKvStoreProvider } from "#src/kvstore/context.js"; +import type { SharedKvStoreContextBase } from "#src/kvstore/register.js"; +import { frontendBackendIsomorphicKvStoreProviderRegistry } from "#src/kvstore/register.js"; + +import { S3KvStore } from "#src/kvstore/s3/index.js"; + +function s3Provider(_context: SharedKvStoreContextBase): BaseKvStoreProvider { + return { + scheme: "s3", + description: "S3 (anonymous)", + getKvStore(url) { + const m = (url.suffix ?? "").match(/^\/\/([^/]+)(\/.*)?$/); + if (m === null) { + throw new Error("Invalid URL, expected `s3:///`"); + } + const [, bucket, path] = m; + return { + store: new S3KvStore( + `https://${bucket}.s3.amazonaws.com/`, + `s3://${bucket}/`, + ), + path: decodeURIComponent((path ?? "").substring(1)), + }; + }, + }; +} + +frontendBackendIsomorphicKvStoreProviderRegistry.registerBaseKvStoreProvider( + s3Provider, +); diff --git a/src/kvstore/shared_common.ts b/src/kvstore/shared_common.ts new file mode 100644 index 0000000000..a3bfb367ce --- /dev/null +++ b/src/kvstore/shared_common.ts @@ -0,0 +1,21 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const SHARED_KVSTORE_CONTEXT_RPC_ID = "SharedKvStoreContext"; + +export const STAT_RPC_ID = "SharedKvStoreContext.stat"; +export const READ_RPC_ID = "SharedKvStoreContext.read"; +export const LIST_RPC_ID = "SharedKvStoreContext.list"; diff --git a/src/kvstore/special/index.ts b/src/kvstore/special/index.ts deleted file mode 100644 index f8f6e3a503..0000000000 --- a/src/kvstore/special/index.ts +++ /dev/null @@ -1,175 +0,0 @@ -/** - * @license - * Copyright 2023 Google Inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import type { - ByteRange, - ByteRangeRequest, - ReadableKvStore, - ReadOptions, - ReadResponse, -} from "#src/kvstore/index.js"; -import { composeByteRangeRequest } from "#src/kvstore/index.js"; -import { isNotFoundError } from "#src/util/http_request.js"; -import type { SpecialProtocolCredentialsProvider } from "#src/util/special_protocol_request.js"; -import { fetchSpecialOk } from "#src/util/special_protocol_request.js"; - -function getRangeHeader( - request: ByteRangeRequest | undefined, -): string | undefined { - if (request === undefined) return undefined; - if ("suffixLength" in request) { - return `bytes=-${request.suffixLength}`; - } - return `bytes=${request.offset}-${request.offset + request.length - 1}`; -} - -/** - * On Chromium, multiple concurrent byte range requests to the same URL are serialized unless the - * cache is disabled. Disabling the cache works around the problem. - * - * https://bugs.chromium.org/p/chromium/issues/detail?id=969828 - */ -const byteRangeCacheMode = - navigator.userAgent.indexOf("Chrome") !== -1 ? "no-store" : "default"; - -class SpecialProtocolKvStore implements ReadableKvStore { - constructor( - public credentialsProvider: SpecialProtocolCredentialsProvider, - public baseUrl: string, - ) {} - - async getObjectLength(url: string, options: ReadOptions) { - // Use a HEAD request to get the length of an object - const headResponse = await fetchSpecialOk(this.credentialsProvider, url, { - method: "HEAD", - signal: options.abortSignal, - }); - - if (headResponse.status !== 200) { - throw new Error( - "Failed to determine total size in order to fetch suffix", - ); - } - const contentLength = headResponse.headers.get("content-length"); - if (contentLength === undefined) { - throw new Error( - "Failed to determine total size in order to fetch suffix", - ); - } - const contentLengthNumber = Number(contentLength); - return contentLengthNumber; - } - - async read( - key: string, - options: ReadOptions, - ): Promise { - let { byteRange: byteRangeRequest } = options; - const url = this.baseUrl + key; - - try { - // The HTTP spec supports suffixLength requests directly via "Range: - // bytes=-N" requests, which avoids the need for a separate HEAD request. - // However, per - // https://fetch.spec.whatwg.org/#cors-safelisted-request-header a suffix - // length byte range request header will always trigger an OPTIONS preflight - // request, which would otherwise be avoided. This negates the benefit of - // using a suffixLength request directly. Additionally, some servers such as - // the npm http-server package and https://uk1s3.embassy.ebi.ac.uk/ do not - // correctly handle suffixLength requests or do not correctly handle CORS - // preflight requests. To avoid those issues, always just issue a separate - // HEAD request to determine the length. - let totalSize: number | undefined; - if ( - byteRangeRequest !== undefined && - "suffixLength" in byteRangeRequest - ) { - const totalSize = await this.getObjectLength(url, options); - byteRangeRequest = composeByteRangeRequest( - { offset: 0, length: totalSize }, - byteRangeRequest, - ).outer; - } - const requestInit: RequestInit = { signal: options.abortSignal }; - const rangeHeader = getRangeHeader(byteRangeRequest); - if (rangeHeader !== undefined) { - requestInit.headers = { range: rangeHeader }; - requestInit.cache = byteRangeCacheMode; - } - const response = await fetchSpecialOk( - this.credentialsProvider, - url, - requestInit, - ); - const data = await response.arrayBuffer(); - let byteRange: ByteRange | undefined; - if (response.status === 206) { - const contentRange = response.headers.get("content-range"); - if (contentRange === null) { - // Content-range should always be sent, but some buggy servers don't - // send it. - if (byteRangeRequest !== undefined) { - byteRange = { - offset: byteRangeRequest.offset, - length: data.byteLength, - }; - } else { - throw new Error( - "Unexpected HTTP 206 response when no byte range specified.", - ); - } - } - if (contentRange !== null) { - const m = contentRange.match(/bytes ([0-9]+)-([0-9]+)\/([0-9]+|\*)/); - if (m === null) { - throw new Error( - `Invalid content-range header: ${JSON.stringify(contentRange)}`, - ); - } - const beginPos = parseInt(m[1], 10); - const endPos = parseInt(m[2], 10); - if (endPos !== beginPos + data.byteLength - 1) { - throw new Error( - `Length in content-range header ${JSON.stringify( - contentRange, - )} does not match content length ${data.byteLength}`, - ); - } - if (m[3] !== "*") { - totalSize = parseInt(m[3], 10); - } - byteRange = { offset: beginPos, length: data.byteLength }; - } - } - if (byteRange === undefined) { - byteRange = { offset: 0, length: data.byteLength }; - totalSize = data.byteLength; - } - return { data: new Uint8Array(data), dataRange: byteRange, totalSize }; - } catch (e) { - if (isNotFoundError(e)) { - return undefined; - } - throw e; - } - } -} -export function getSpecialProtocolKvStore( - credentialsProvider: SpecialProtocolCredentialsProvider, - baseUrl: string, -): ReadableKvStore { - return new SpecialProtocolKvStore(credentialsProvider, baseUrl); -} diff --git a/src/kvstore/url.ts b/src/kvstore/url.ts new file mode 100644 index 0000000000..89a29298b7 --- /dev/null +++ b/src/kvstore/url.ts @@ -0,0 +1,200 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function kvstoreEnsureDirectoryPipelineUrl(url: string): string { + const m = url.match( + /^((?:.*?|)?)([a-zA-Z][a-zA-Z0-9-+.]*)(?:(:[^?#|]*)((?:[?#][^|]*)?))?$/, + ); + if (m === null) { + throw new Error(`Invalid URL: ${url}`); + } + const [, pipelinePrefix, scheme, path, queryAndFragment] = m; + if (path === undefined) { + return `${pipelinePrefix}${scheme}:`; + } + if (path === ":" || path.endsWith("/")) return url; + return `${pipelinePrefix}${scheme}${path}/${queryAndFragment ?? ""}`; +} + +export function finalPipelineUrlComponent(url: string) { + // match is infallible + const m = url.match(/.*?([^|]*)$/)!; + return m[1]; +} + +export const schemePattern = /^(?:([a-zA-Z][a-zA-Z0-9-+.]*):)?(.*)$/; + +export function parsePipelineUrlComponent(url: string): UrlWithParsedScheme { + // schemePattern always matches + const m = url.match(schemePattern)!; + const scheme = m[1]; + const suffix = m[2]; + if (scheme === undefined) { + return { url, scheme: url, suffix: undefined }; + } else { + return { url, scheme: scheme, suffix }; + } +} + +export const urlComponentPattern = + /^(?:([a-zA-Z][a-zA-Z0-9-+.]*):)?([^?#]*)(?:\?([^#]*))?(?:#(.*))?$/; + +export function parseUrlSuffix(suffix: string | undefined): { + authorityAndPath: string | undefined; + query: string | undefined; + fragment: string | undefined; +} { + if (suffix === undefined) { + return { + authorityAndPath: undefined, + query: undefined, + fragment: undefined, + }; + } + // Infallible pattern. + const [, authorityAndPath, query, fragment] = suffix.match( + /^([^?#]*)(?:\?([^#]*))?(?:#(.*))?$/, + )!; + return { + authorityAndPath, + query: query ?? undefined, + fragment: fragment ?? undefined, + }; +} + +export interface UrlWithParsedScheme { + // Full original URL. + url: string; + + // Scheme (excluding ":"). + scheme: string; + + // Suffix following ":", including initial "//" if present. + suffix: string | undefined; +} + +// Splits a URL containing multiple "|"-separate parts. +export function splitPipelineUrl(url: string): UrlWithParsedScheme[] { + return url.split("|").map(parsePipelineUrlComponent); +} + +export function pipelineUrlJoin( + baseUrl: string, + ...additionalParts: string[] +): string { + // Strip off any ? or # parameters, since they are not part of the path. + // Infallible pattern + let [, base, queryAndFragment] = baseUrl.match(/^(.*?[^|?#]*)([^|]*)$/)!; + for (let part of additionalParts) { + if (part.startsWith("/")) { + part = part.substring(1); + } + if (part === "") continue; + base = kvstoreEnsureDirectoryPipelineUrl(base); + base += part; + } + return base + queryAndFragment; +} + +export function joinPath(base: string, ...additionalParts: string[]) { + for (let part of additionalParts) { + if (part.startsWith("/")) { + part = part.substring(1); + } + if (part === "") continue; + base = ensurePathIsDirectory(base); + base += part; + } + return base; +} + +export function ensurePathIsDirectory(path: string) { + if (!pathIsDirectory(path)) { + path += "/"; + } + return path; +} + +export function ensureNoQueryOrFragmentParameters(url: UrlWithParsedScheme) { + const { suffix } = url; + if (suffix === undefined) return; + if (suffix.match(/[#?]/)) { + throw new Error( + `Invalid URL ${url.url}: query parameters and/or fragment not supported`, + ); + } +} + +export function ensureEmptyUrlSuffix(url: UrlWithParsedScheme) { + if (url.suffix) { + throw new Error( + `Invalid URL syntax ${JSON.stringify(url.url)}, expected "${url.scheme}:"`, + ); + } +} + +export function extractQueryAndFragment(url: string): { + base: string; + queryAndFragment: string; +} { + const [, base, queryAndFragment] = url.match(/^(.*?[^|?#]*)([^|]*)$/)!; + return { base, queryAndFragment }; +} + +// Resolves `relativePath` relative to `basePath`. +// +// Note that the parameters are both expected to be plain paths, not full URLs +// or URL pipelines. +export function resolveRelativePath(basePath: string, relativePath: string) { + const origBasePath = basePath; + if (basePath.endsWith("/")) { + basePath = basePath.substring(0, basePath.length - 1); + } + for (const component of relativePath.split("/")) { + if (component === "" || component === ".") { + continue; + } + if (component === "..") { + const prevSlash = basePath.lastIndexOf("/"); + if (prevSlash <= 0) { + throw new Error( + `Invalid relative path ${JSON.stringify(relativePath)} from base path ${JSON.stringify(origBasePath)}`, + ); + } + basePath = basePath.substring(0, prevSlash - 1); + continue; + } + basePath += "/"; + basePath += component; + } + if (relativePath.endsWith("/")) { + basePath += "/"; + } + return basePath; +} + +export function pathIsDirectory(path: string) { + return path === "" || path.endsWith("/"); +} + +// Plain paths can have arbitrary characters, but to be included in a URL +// pipeline, special characters must be percent encoded. +export function encodePathForUrl(path: string) { + return encodeURI(path).replace( + /[?#]/g, + (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`, + ); +} diff --git a/src/kvstore/zip/auto_detect.ts b/src/kvstore/zip/auto_detect.ts new file mode 100644 index 0000000000..521002f6f7 --- /dev/null +++ b/src/kvstore/zip/auto_detect.ts @@ -0,0 +1,46 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { + AutoDetectFileOptions, + AutoDetectMatch, + AutoDetectRegistry, +} from "#src/kvstore/auto_detect.js"; +import { + EOCDR_WITHOUT_COMMENT_SIZE, + parseEndOfCentralDirectoryRecord, +} from "#src/kvstore/zip/metadata.js"; + +async function detectZip( + options: AutoDetectFileOptions, +): Promise { + const { suffix } = options; + if (suffix === undefined) return []; + if (parseEndOfCentralDirectoryRecord(suffix) === undefined) return []; + return [{ suffix: "zip:", description: "ZIP archive" }]; +} + +export function registerAutoDetect(registry: AutoDetectRegistry) { + registry.registerFileFormat({ + prefixLength: 0, + // To ensure all valid zip file are detected, this should be set to + // `EOCDR_WITHOUT_COMMENT_SIZE + MAX_COMMENT_SIZE`. In practice, though, zip + // files with comments are rare and 4096 should be sufficient for most + // cases while avoiding reading an excessive amount for auto-detection. + suffixLength: EOCDR_WITHOUT_COMMENT_SIZE + 4096, + match: detectZip, + }); +} diff --git a/src/kvstore/zip/backend.ts b/src/kvstore/zip/backend.ts new file mode 100644 index 0000000000..00d021fe98 --- /dev/null +++ b/src/kvstore/zip/backend.ts @@ -0,0 +1,255 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ChunkManager } from "#src/chunk_manager/backend.js"; +import { makeSimpleAsyncCache } from "#src/chunk_manager/generic_file_source.js"; +import { FileByteRangeHandle } from "#src/kvstore/byte_range/file_handle.js"; +import { GzipFileHandle } from "#src/kvstore/gzip/file_handle.js"; +import type { + DriverListOptions, + DriverReadOptions, + FileHandle, + KvStore, + ListEntry, + ListResponse, + ReadResponse, + StatOptions, + StatResponse, +} from "#src/kvstore/index.js"; +import { readFileHandle } from "#src/kvstore/index.js"; +import { encodePathForUrl } from "#src/kvstore/url.js"; +import type { + ZipMetadata, + Reader, + ZipEntry, +} from "#src/kvstore/zip/metadata.js"; +import { + readZipMetadata, + readEntryDataHeader, + ZipCompressionMethod, +} from "#src/kvstore/zip/metadata.js"; +import { + binarySearch, + binarySearchLowerBound, + filterArrayInplace, +} from "#src/util/array.js"; +import { + ProgressSpan, + type ProgressOptions, +} from "#src/util/progress_listener.js"; +import { defaultStringCompare } from "#src/util/string.js"; + +function makeZipReader(base: FileHandle): Reader { + return async ( + offset: number, + length: number, + options: Partial, + ) => { + const readResponse = await readFileHandle(base, { + throwIfMissing: true, + byteRange: { offset, length }, + strictByteRange: true, + signal: options.signal, + progressListener: options.progressListener, + }); + return new Uint8Array(await readResponse.response.arrayBuffer()); + }; +} + +interface CachedZipEntry extends ZipEntry { + fileDataStart?: number; +} + +interface CachedZipMetadata extends ZipMetadata { + entries: CachedZipEntry[]; +} + +function getZipMetadataCache(chunkManager: ChunkManager, base: FileHandle) { + const url = base.getUrl(); + return makeSimpleAsyncCache(chunkManager, `zipMetadata:${url}`, { + get: async (_unusedCacheKey: undefined, progressOptions) => { + using _span = new ProgressSpan(progressOptions.progressListener, { + message: `Reading ZIP central directory from ${url}`, + }); + const statResponse = await base.stat(progressOptions); + if (statResponse?.totalSize === undefined) { + throw new Error(`Failed to determine ZIP file size: ${url}`); + } + const metadata = await readZipMetadata( + makeZipReader(base), + statResponse.totalSize, + progressOptions, + ); + // Zip files sometimes contain zero-length files corresponding to + // directories. + filterArrayInplace( + metadata.entries, + (entry) => !entry.fileName.endsWith("/"), + ); + metadata.entries.sort((a, b) => + defaultStringCompare(a.fileName, b.fileName), + ); + return { data: metadata, size: metadata.sizeEstimate }; + }, + }); +} + +async function getZipMetadata( + chunkManager: ChunkManager, + base: FileHandle, + options: Partial, +): Promise { + const cache = getZipMetadataCache(chunkManager, base); + try { + return (await cache.get(undefined, options)) as CachedZipMetadata; + } finally { + cache.dispose(); + } +} + +function findEntry( + metadata: CachedZipMetadata, + key: string, +): CachedZipEntry | undefined { + const { entries } = metadata; + const index = binarySearch(entries, key, (key, entry) => + defaultStringCompare(key, entry.fileName), + ); + if (index < 0) return undefined; + return entries[index]; +} + +function list(metadata: ZipMetadata, prefix: string) { + const { entries } = metadata; + const startIndex = binarySearchLowerBound( + 0, + entries.length, + (index) => entries[index].fileName >= prefix, + ); + + const endIndex = binarySearchLowerBound( + Math.min(entries.length, startIndex + 1), + entries.length, + (index) => !entries[index].fileName.startsWith(prefix), + ); + + const listEntries: ListEntry[] = []; + const directories: string[] = []; + + for (let index = startIndex; index < endIndex; ) { + const entry = entries[index]; + const i = entry.fileName.indexOf("/", prefix.length); + if (i === -1) { + // Filename + listEntries.push({ key: entry.fileName }); + ++index; + } else { + // Directory + directories.push(entry.fileName.substring(0, i)); + const directoryPrefix = entry.fileName.substring(0, i + 1); + index = binarySearchLowerBound( + index + 1, + endIndex, + (index) => !entries[index].fileName.startsWith(directoryPrefix), + ); + } + } + + return { entries: listEntries, directories }; +} + +export class ZipKvStore + implements KvStore +{ + constructor( + public chunkManager: ChunkManager, + public base: BaseFileHandle, + ) {} + + private metadata: ZipMetadata | undefined; + + private async getMetadata(options: Partial) { + let { metadata } = this; + if (metadata === undefined) { + metadata = this.metadata = await getZipMetadata( + this.chunkManager, + this.base, + options, + ); + } + return metadata; + } + + getUrl(key: string) { + return this.base.getUrl() + `|zip:${encodePathForUrl(key)}`; + } + + async stat( + key: string, + options: StatOptions, + ): Promise { + const entry = findEntry(await this.getMetadata(options), key); + if (entry === undefined) return undefined; + return { totalSize: entry.uncompressedSize }; + } + + async read( + key: string, + options: DriverReadOptions, + ): Promise { + const entry = findEntry(await this.getMetadata(options), key); + if (entry === undefined) return undefined; + let { fileDataStart } = entry; + if (fileDataStart === undefined) { + fileDataStart = entry.fileDataStart = await readEntryDataHeader( + makeZipReader(this.base), + entry, + options, + ); + } + let handle: FileHandle = new FileByteRangeHandle(this.base, { + offset: fileDataStart, + length: entry.compressedSize, + }); + switch (entry.compressionMethod) { + case ZipCompressionMethod.STORE: + break; + case ZipCompressionMethod.DEFLATE: + handle = new GzipFileHandle(handle, "deflate-raw"); + break; + default: + throw new Error( + `Unsupported compression method: ${entry.compressionMethod}`, + ); + } + return handle.read(options); + } + + async list( + prefix: string, + options: DriverListOptions, + ): Promise { + const metadata = await this.getMetadata(options); + return list(metadata, prefix); + } + + get supportsOffsetReads() { + return true; + } + get supportsSuffixReads() { + return true; + } +} diff --git a/src/kvstore/zip/frontend.ts b/src/kvstore/zip/frontend.ts new file mode 100644 index 0000000000..959254c444 --- /dev/null +++ b/src/kvstore/zip/frontend.ts @@ -0,0 +1,40 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; +import { ProxyKvStore } from "#src/kvstore/frontend.js"; +import type { KvStore, KvStoreFileHandle } from "#src/kvstore/index.js"; +import { encodePathForUrl } from "#src/kvstore/url.js"; + +export class ZipKvStore extends ProxyKvStore implements KvStore { + constructor( + sharedKvStoreContext: SharedKvStoreContext, + public base: KvStoreFileHandle, + ) { + super(sharedKvStoreContext); + } + + getUrl(key: string) { + return this.base.getUrl() + `|zip:${encodePathForUrl(key)}`; + } + + get supportsOffsetReads() { + return true; + } + get supportsSuffixReads() { + return true; + } +} diff --git a/src/kvstore/zip/metadata.ts b/src/kvstore/zip/metadata.ts new file mode 100644 index 0000000000..dbf864ab05 --- /dev/null +++ b/src/kvstore/zip/metadata.ts @@ -0,0 +1,658 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Derived from https://github.com/greggman/unzipit/blob/4d94c9b77f7815062ff4460311e8b3ce4f7d5deb/src/unzipit.js + * + * Includes only parsing of raw entries. + * + * @license + * + * The MIT License (MIT) + * + * Copyright (c) 2014 Josh Wolfe + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * MIT License + * + * Copyright (c) 2019 Gregg Tavares + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { buf as crc32buf } from "crc-32"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; + +export interface ZipEntry { + versionMadeBy: number; + versionNeededToExtract: number; + generalPurposeBitFlag: number; + compressionMethod: number; + lastModFileTime: number; + lastModFileDate: number; + crc32: number; + compressedSize: number; + uncompressedSize: number; + nameBytes: Uint8Array; + commentBytes: Uint8Array; + internalFileAttributes: number; + externalFileAttributes: number; + relativeOffsetOfLocalHeader: number; + fileName: string; +} + +export interface ZipMetadata { + entries: ZipEntry[]; + commentBytes: Uint8Array; + // Estimated size in bytes of metadata. + sizeEstimate: number; +} + +export const EOCDR_WITHOUT_COMMENT_SIZE = 22; +export const MAX_COMMENT_SIZE = 0xffff; // 2-byte size +const EOCDR_SIGNATURE = 0x06054b50; +const ZIP64_EOCDR_SIGNATURE = 0x06064b50; + +export interface Reader { + ( + offset: number, + length: number, + progressOptions: Partial, + ): Promise>; +} + +function lastReadCachingReader(base: Reader) { + let lastReadOffset: number = 0; + let lastReadBuffer: Uint8Array | undefined; + + return async function lastReadCachingRead( + offset: number, + length: number, + progressOptions: Partial, + ): Promise> { + if (lastReadBuffer !== undefined) { + if ( + offset > lastReadOffset && + offset + length <= lastReadOffset + lastReadBuffer.length + ) { + return lastReadBuffer.subarray( + offset - lastReadOffset, + offset + length - lastReadOffset, + ); + } + } + + const newBuffer = await base(offset, length, progressOptions); + lastReadOffset = offset; + lastReadBuffer = newBuffer; + return newBuffer; + }; +} + +export function parseEndOfCentralDirectoryRecord(data: Uint8Array): + | { + eocdrOffset: number; + diskNumber: number; + entryCount: number; + centralDirectorySize: number; + centralDirectoryOffset: number; + } + | undefined { + const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); + const size = data.length; + for (let i = size - EOCDR_WITHOUT_COMMENT_SIZE; i >= 0; --i) { + // 0 - End of central directory signature + if (dv.getUint32(i, /*littleEndian=*/ true) !== EOCDR_SIGNATURE) { + continue; + } + + // 20 - Comment length + const commentLength = dv.getUint16(i + 20, /*littleEndian=*/ true); + const expectedCommentLength = size - i - EOCDR_WITHOUT_COMMENT_SIZE; + if (commentLength !== expectedCommentLength) { + continue; + } + + // 4 - Number of this disk + const diskNumber = dv.getUint16(i + 4, /*littleEndian=*/ true); + + // 6 - Disk where central directory starts + // 8 - Number of central directory records on this disk + // 10 - Total number of central directory records + const entryCount = dv.getUint16(i + 10, /*littleEndian=*/ true); + // 12 - Size of central directory (bytes) + const centralDirectorySize = dv.getUint32(i + 12, /*littleEndian=*/ true); + // 16 - Offset of start of central directory, relative to start of archive + const centralDirectoryOffset = dv.getUint32(i + 16, /*littleEndian=*/ true); + + return { + eocdrOffset: i, + diskNumber, + entryCount, + centralDirectorySize, + centralDirectoryOffset, + }; + } + + return undefined; +} + +async function findEndOfCentralDirectory( + reader: Reader, + totalLength: number, + options: Partial, +) { + const size = Math.min( + EOCDR_WITHOUT_COMMENT_SIZE + MAX_COMMENT_SIZE, + totalLength, + ); + const readStart = totalLength - size; + const data = await reader(readStart, size, options); + const record = parseEndOfCentralDirectoryRecord(data); + if (record === undefined) { + throw new Error( + "End of central directory record signature not found; either not a zip file or file is truncated.", + ); + } + const { + eocdrOffset, + diskNumber, + entryCount, + centralDirectorySize, + centralDirectoryOffset, + } = record; + if (diskNumber !== 0) { + throw new Error( + `Multi-volume zip files are not supported. This is volume: ${diskNumber}`, + ); + } + + // 22 - Comment + // the encoding is always cp437. + const commentBytes = data.slice(eocdrOffset + 22, data.length); + + if (entryCount === 0xffff || centralDirectoryOffset === 0xffffffff) { + return await readZip64CentralDirectory( + reader, + eocdrOffset, + commentBytes, + options, + ); + } else { + return await readEntries( + reader, + centralDirectoryOffset, + centralDirectorySize, + entryCount, + commentBytes, + options, + ); + } +} + +const END_OF_CENTRAL_DIRECTORY_LOCATOR_SIGNATURE = 0x07064b50; + +async function readZip64CentralDirectory( + reader: Reader, + offset: number, + commentBytes: Uint8Array, + progressOptions: Partial, +) { + // ZIP64 Zip64 end of central directory locator + const zip64EocdlOffset = offset - 20; + const eocdl = await reader(zip64EocdlOffset, 20, progressOptions); + + const eocdlDv = new DataView( + eocdl.buffer, + eocdl.byteOffset, + eocdl.byteLength, + ); + + // 0 - zip64 end of central dir locator signature + if ( + eocdlDv.getUint32(0, /*littleEndian=*/ true) !== + END_OF_CENTRAL_DIRECTORY_LOCATOR_SIGNATURE + ) { + throw new Error("invalid zip64 end of central directory locator signature"); + } + + // 4 - number of the disk with the start of the zip64 end of central directory + // 8 - relative offset of the zip64 end of central directory record + const zip64EocdrOffset = eocdlDv.getBigUint64(8, /*littleEndian=*/ true); + // 16 - total number of disks + + // ZIP64 end of central directory record + const zip64Eocdr = await reader( + Number(zip64EocdrOffset), + 56, + progressOptions, + ); + + const zip64EocdrDv = new DataView( + zip64Eocdr.buffer, + zip64Eocdr.byteOffset, + zip64Eocdr.byteLength, + ); + + // 0 - zip64 end of central dir signature 4 bytes (0x06064b50) + if ( + zip64EocdrDv.getUint32(0, /*littleEndian=*/ true) !== ZIP64_EOCDR_SIGNATURE + ) { + throw new Error("invalid zip64 end of central directory record signature"); + } + // 4 - size of zip64 end of central directory record 8 bytes + // 12 - version made by 2 bytes + // 14 - version needed to extract 2 bytes + // 16 - number of this disk 4 bytes + // 20 - number of the disk with the start of the central directory 4 bytes + // 24 - total number of entries in the central directory on this disk 8 bytes + // 32 - total number of entries in the central directory 8 bytes + const entryCount = zip64EocdrDv.getBigUint64(32, /*littleEndian=*/ true); + // 40 - size of the central directory 8 bytes + const centralDirectorySize = zip64EocdrDv.getBigUint64( + 40, + /*littleEndian=*/ true, + ); + // 48 - offset of start of central directory with respect to the starting disk number 8 bytes + const centralDirectoryOffset = zip64EocdrDv.getBigUint64( + 48, + /*littleEndian=*/ true, + ); + // 56 - zip64 extensible data sector (variable size) + return readEntries( + reader, + Number(centralDirectoryOffset), + Number(centralDirectorySize), + Number(entryCount), + commentBytes, + progressOptions, + ); +} + +const CENTRAL_DIRECTORY_FILE_HEADER_SIGNATURE = 0x02014b50; + +async function readEntries( + reader: Reader, + centralDirectoryOffset: number, + centralDirectorySize: number, + rawEntryCount: number, + commentBytes: Uint8Array, + progressOptions: Partial, +): Promise { + let readEntryCursor = 0; + const allEntriesBuffer = await reader( + centralDirectoryOffset, + centralDirectorySize, + progressOptions, + ); + const rawEntries = []; + + const dv = new DataView( + allEntriesBuffer.buffer, + allEntriesBuffer.byteOffset, + allEntriesBuffer.byteLength, + ); + + const textDecoder = new TextDecoder(); + + for (let e = 0; e < rawEntryCount; ++e) { + // 0 - Central directory file header signature + const signature = dv.getUint32(readEntryCursor + 0, /*littleEndian=*/ true); + if (signature !== CENTRAL_DIRECTORY_FILE_HEADER_SIGNATURE) { + throw new Error( + `invalid central directory file header signature: 0x${signature.toString(16)}`, + ); + } + // 4 - Version made by + const versionMadeBy = dv.getUint16( + readEntryCursor + 4, + /*littleEndian=*/ true, + ); + // 6 - Version needed to extract (minimum) + const versionNeededToExtract = dv.getUint16( + readEntryCursor + 6, + /*littleEndian=*/ true, + ); + // 8 - General purpose bit flag + const generalPurposeBitFlag = dv.getUint16( + readEntryCursor + 8, + /*littleEndian=*/ true, + ); + // 10 - Compression method + const compressionMethod = dv.getUint16( + readEntryCursor + 10, + /*littleEndian=*/ true, + ); + // 12 - File last modification time + const lastModFileTime = dv.getUint16( + readEntryCursor + 12, + /*littleEndian=*/ true, + ); + // 14 - File last modification date + const lastModFileDate = dv.getUint16( + readEntryCursor + 14, + /*littleEndian=*/ true, + ); + // 16 - CRC-32 + const crc32 = dv.getUint32(readEntryCursor + 16, /*littleEndian=*/ true); + // 20 - Compressed size + let compressedSize = dv.getUint32( + readEntryCursor + 20, + /*littleEndian=*/ true, + ); + // 24 - Uncompressed size + let uncompressedSize = dv.getUint32( + readEntryCursor + 24, + /*littleEndian=*/ true, + ); + // 28 - File name length (n) + const fileNameLength = dv.getUint16( + readEntryCursor + 28, + /*littleEndian=*/ true, + ); + // 30 - Extra field length (m) + const extraFieldLength = dv.getUint16( + readEntryCursor + 30, + /*littleEndian=*/ true, + ); + // 32 - File comment length (k) + const fileCommentLength = dv.getUint16( + readEntryCursor + 32, + /*littleEndian=*/ true, + ); + // 34 - Disk number where file starts + // 36 - Internal file attributes + const internalFileAttributes = dv.getUint16( + readEntryCursor + 36, + /*littleEndian=*/ true, + ); + // 38 - External file attributes + const externalFileAttributes = dv.getUint32( + readEntryCursor + 38, + /*littleEndian=*/ true, + ); + // 42 - Relative offset of local file header + let relativeOffsetOfLocalHeader = dv.getUint32( + readEntryCursor + 42, + /*littleEndian=*/ true, + ); + + if (generalPurposeBitFlag & 0x40) { + throw new Error("strong encryption is not supported"); + } + + readEntryCursor += 46; + + // 46 - File name + let nameBytes = allEntriesBuffer.subarray( + readEntryCursor, + (readEntryCursor += fileNameLength), + ); + + let isUTF8 = (generalPurposeBitFlag & 0x800) !== 0; + + // 46+n - Extra field + const extraFields = []; + for (let i = 0; i < extraFieldLength - 3; ) { + const headerId = dv.getUint16( + readEntryCursor + i + 0, + /*littleEndian=*/ true, + ); + const dataSize = dv.getUint16( + readEntryCursor + i + 2, + /*littleEndian=*/ true, + ); + const dataStart = i + 4; + const dataEnd = dataStart + dataSize; + if (dataEnd > extraFieldLength) { + throw new Error("extra field length exceeds extra field buffer size"); + } + extraFields.push({ + id: headerId, + offset: readEntryCursor + dataStart, + length: dataSize, + }); + i = dataEnd; + } + readEntryCursor += extraFieldLength; + + // 46+n+m - File comment + const commentBytes = allEntriesBuffer.slice( + readEntryCursor, + (readEntryCursor += fileCommentLength), + ); + + if ( + uncompressedSize === 0xffffffff || + compressedSize === 0xffffffff || + relativeOffsetOfLocalHeader === 0xffffffff + ) { + // ZIP64 format + // find the Zip64 Extended Information Extra Field + const zip64ExtraField = extraFields.find((e) => e.id === 0x0001); + if (zip64ExtraField === undefined) { + throw new Error("expected zip64 extended information extra field"); + } + const { offset: zip64EiefBufferOffset, length: zip64EiefBufferLength } = + zip64ExtraField; + let index = 0; + // 0 - Original Size 8 bytes + if (uncompressedSize === 0xffffffff) { + if (index + 8 > zip64EiefBufferLength) { + throw new Error( + "zip64 extended information extra field does not include uncompressed size", + ); + } + uncompressedSize = Number( + dv.getBigUint64( + zip64EiefBufferOffset + index, + /*littleEndian=*/ true, + ), + ); + index += 8; + } + // 8 - Compressed Size 8 bytes + if (compressedSize === 0xffffffff) { + if (index + 8 > zip64EiefBufferLength) { + throw new Error( + "zip64 extended information extra field does not include compressed size", + ); + } + compressedSize = Number( + dv.getBigUint64( + zip64EiefBufferOffset + index, + /*littleEndian=*/ true, + ), + ); + index += 8; + } + // 16 - Relative Header Offset 8 bytes + if (relativeOffsetOfLocalHeader === 0xffffffff) { + if (index + 8 > zip64EiefBufferLength) { + throw new Error( + "zip64 extended information extra field does not include relative header offset", + ); + } + relativeOffsetOfLocalHeader = Number( + dv.getBigUint64( + zip64EiefBufferOffset + index, + /*littleEndian=*/ true, + ), + ); + index += 8; + } + // 24 - Disk Start Number 4 bytes + } + + // check for Info-ZIP Unicode Path Extra Field (0x7075) + // see https://github.com/thejoshwolfe/yauzl/issues/33 + const nameField = extraFields.find( + (e) => + e.id === 0x7075 && + e.length >= 6 && // too short to be meaningful + allEntriesBuffer[e.offset] === 1 && // Version 1 byte version of this extra field, currently 1 + dv.getInt32(e.offset + 1, /*littleEndian=*/ true) === + crc32buf(nameBytes), + ); // NameCRC32 4 bytes File Name Field CRC32 Checksum + // > If the CRC check fails, this UTF-8 Path Extra Field should be + // > ignored and the File Name field in the header should be used instead. + if (nameField) { + nameBytes = allEntriesBuffer.slice( + nameField.offset + 5, + nameField.offset + nameField.length, + ); + isUTF8 = true; + } + + // validate file size + if (compressionMethod === 0) { + let expectedCompressedSize = uncompressedSize; + if ((generalPurposeBitFlag & 0x1) !== 0) { + // traditional encryption prefixes the file data with a header + expectedCompressedSize += 12; + } + if (compressedSize !== expectedCompressedSize) { + throw new Error( + `compressed/uncompressed size mismatch for stored file: ${compressedSize} != ${expectedCompressedSize}`, + ); + } + } + + // Just decode as UTF-8 regardless of `isUTF8`, because the non-UTF8 + // encoding is difficult/impossible to determine correctly. + let fileName = textDecoder.decode(nameBytes); + fileName = fileName.replaceAll("\\", "/"); + isUTF8; + + const rawEntry: ZipEntry = { + versionMadeBy, + versionNeededToExtract, + generalPurposeBitFlag, + compressionMethod, + lastModFileTime, + lastModFileDate, + crc32, + compressedSize, + uncompressedSize, + nameBytes, + commentBytes, + internalFileAttributes, + externalFileAttributes, + relativeOffsetOfLocalHeader, + fileName, + }; + rawEntries.push(rawEntry); + } + return { + commentBytes, + entries: rawEntries, + // Estimate that the JavaScript representation consumes twice the memory of + // the encoded representation. + sizeEstimate: commentBytes.length + allEntriesBuffer.length * 2, + }; +} + +export async function readEntryDataHeader( + reader: Reader, + rawEntry: ZipEntry, + options: Partial, +) { + if (rawEntry.generalPurposeBitFlag & 0x1) { + throw new Error("encrypted entries not supported"); + } + const data = await reader(rawEntry.relativeOffsetOfLocalHeader, 30, options); + const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); + + // 0 - Local file header signature = 0x04034b50 + const signature = dv.getUint32(0, /*littleEndian=*/ true); + if (signature !== 0x04034b50) { + throw new Error( + `invalid local file header signature: 0x${signature.toString(16)}`, + ); + } + + // all this should be redundant + // 4 - Version needed to extract (minimum) + // 6 - General purpose bit flag + // 8 - Compression method + // 10 - File last modification time + // 12 - File last modification date + // 14 - CRC-32 + // 18 - Compressed size + // 22 - Uncompressed size + // 26 - File name length (n) + const fileNameLength = dv.getUint16(26, /*littleEndian=*/ true); + // 28 - Extra field length (m) + const extraFieldLength = dv.getUint16(28, /*littleEndian=*/ true); + // 30 - File name + // 30+n - Extra field + const localFileHeaderEnd = + rawEntry.relativeOffsetOfLocalHeader + + data.length + + fileNameLength + + extraFieldLength; + + return localFileHeaderEnd; +} + +export async function readZipMetadata( + reader: Reader, + totalLength: number, + options: Partial, +): Promise { + return await findEndOfCentralDirectory( + lastReadCachingReader(reader), + totalLength, + options, + ); +} + +export enum ZipCompressionMethod { + STORE = 0, + DEFLATE = 8, +} diff --git a/src/kvstore/zip/register_backend.ts b/src/kvstore/zip/register_backend.ts new file mode 100644 index 0000000000..e90a4eed17 --- /dev/null +++ b/src/kvstore/zip/register_backend.ts @@ -0,0 +1,43 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { SharedKvStoreContextCounterpart } from "#src/kvstore/backend.js"; +import { backendOnlyKvStoreProviderRegistry } from "#src/kvstore/backend.js"; +import type { KvStoreAdapterProvider } from "#src/kvstore/context.js"; +import { KvStoreFileHandle } from "#src/kvstore/index.js"; +import { ensureNoQueryOrFragmentParameters } from "#src/kvstore/url.js"; +import { ZipKvStore } from "#src/kvstore/zip/backend.js"; + +function zipProvider( + sharedKvStoreContext: SharedKvStoreContextCounterpart, +): KvStoreAdapterProvider { + return { + scheme: "zip", + description: "ZIP archive", + getKvStore(parsedUrl, base) { + ensureNoQueryOrFragmentParameters(parsedUrl); + return { + store: new ZipKvStore( + sharedKvStoreContext.chunkManager, + new KvStoreFileHandle(base.store, base.path), + ), + path: decodeURIComponent(parsedUrl.suffix ?? ""), + }; + }, + }; +} + +backendOnlyKvStoreProviderRegistry.registerKvStoreAdapterProvider(zipProvider); diff --git a/src/kvstore/zip/register_frontend.ts b/src/kvstore/zip/register_frontend.ts new file mode 100644 index 0000000000..ff47de0d26 --- /dev/null +++ b/src/kvstore/zip/register_frontend.ts @@ -0,0 +1,46 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { KvStoreAdapterProvider } from "#src/kvstore/context.js"; +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; +import { frontendOnlyKvStoreProviderRegistry } from "#src/kvstore/frontend.js"; +import { KvStoreFileHandle } from "#src/kvstore/index.js"; +import { ensureNoQueryOrFragmentParameters } from "#src/kvstore/url.js"; +import { registerAutoDetect } from "#src/kvstore/zip/auto_detect.js"; +import { ZipKvStore } from "#src/kvstore/zip/frontend.js"; + +function zipProvider( + sharedKvStoreContext: SharedKvStoreContext, +): KvStoreAdapterProvider { + return { + scheme: "zip", + description: "ZIP archive", + getKvStore(parsedUrl, base) { + ensureNoQueryOrFragmentParameters(parsedUrl); + return { + store: new ZipKvStore( + sharedKvStoreContext, + new KvStoreFileHandle(base.store, base.path), + ), + path: decodeURIComponent(parsedUrl.suffix ?? ""), + }; + }, + }; +} + +frontendOnlyKvStoreProviderRegistry.registerKvStoreAdapterProvider(zipProvider); + +registerAutoDetect(frontendOnlyKvStoreProviderRegistry.autoDetectRegistry); diff --git a/src/layer/annotation/index.ts b/src/layer/annotation/index.ts index 1d20308cf3..663b76e052 100644 --- a/src/layer/annotation/index.ts +++ b/src/layer/annotation/index.ts @@ -29,7 +29,7 @@ import { import type { CoordinateTransformSpecification } from "#src/coordinate_transform.js"; import { makeCoordinateSpace } from "#src/coordinate_transform.js"; import type { DataSourceSpecification } from "#src/datasource/index.js"; -import { localAnnotationsUrl, LocalDataSource } from "#src/datasource/index.js"; +import { localAnnotationsUrl, LocalDataSource } from "#src/datasource/local.js"; import type { LayerManager, ManagedUserLayer } from "#src/layer/index.js"; import { LayerReference, diff --git a/src/layer/index.ts b/src/layer/index.ts index 95fbf6502a..fea87063da 100644 --- a/src/layer/index.ts +++ b/src/layer/index.ts @@ -32,7 +32,7 @@ import { TrackableCoordinateSpace, } from "#src/coordinate_transform.js"; import type { - DataSourceProviderRegistry, + DataSourceRegistry, DataSourceSpecification, DataSubsource, } from "#src/datasource/index.js"; @@ -2080,7 +2080,7 @@ export abstract class LayerListSpecification extends RefCounted { abstract rpc: RPC; - abstract dataSourceProviderRegistry: Borrowed; + abstract dataSourceProviderRegistry: Borrowed; abstract layerManager: Borrowed; abstract chunkManager: Borrowed; abstract layerSelectedValues: Borrowed; @@ -2110,7 +2110,7 @@ export class TopLevelLayerListSpecification extends LayerListSpecification { constructor( public display: DisplayContext, - public dataSourceProviderRegistry: DataSourceProviderRegistry, + public dataSourceProviderRegistry: DataSourceRegistry, public layerManager: LayerManager, public chunkManager: ChunkManager, public selectionState: Borrowed, diff --git a/src/layer/layer_data_source.ts b/src/layer/layer_data_source.ts index ccc93bcd61..f3cb7b2691 100644 --- a/src/layer/layer_data_source.ts +++ b/src/layer/layer_data_source.ts @@ -28,6 +28,7 @@ import { import type { DataSource, DataSourceSpecification, + DataSourceWithRedirectInfo, DataSubsourceEntry, DataSubsourceSpecification, } from "#src/datasource/index.js"; @@ -39,6 +40,7 @@ import type { WatchableValueInterface } from "#src/trackable_value.js"; import { arraysEqual } from "#src/util/array.js"; import type { Borrowed, Owned } from "#src/util/disposable.js"; import { disposableOnce, RefCounted } from "#src/util/disposable.js"; +import { formatErrorMessage } from "#src/util/error.js"; import { verifyBoolean, verifyObject, @@ -49,6 +51,7 @@ import { } from "#src/util/json.js"; import * as matrix from "#src/util/matrix.js"; import { MessageList, MessageSeverity } from "#src/util/message_list.js"; +import { MultiConsumerProgressListener } from "#src/util/progress_listener.js"; import { NullarySignal } from "#src/util/signal.js"; export function parseDataSubsourceSpecificationFromJson( @@ -308,6 +311,7 @@ export type LayerDataSourceLoadState = export class LayerDataSource extends RefCounted { changed = new NullarySignal(); messages = new MessageList(); + progressListener = new MultiConsumerProgressListener(); private loadState_: LayerDataSourceLoadState = undefined; private spec_: DataSourceSpecification; private specGeneration = -1; @@ -396,23 +400,18 @@ export class LayerDataSource extends RefCounted { } this.refCounted_ = refCounted; this.spec_ = spec; - const chunkManager = layer.manager.chunkManager; const registry = layer.manager.dataSourceProviderRegistry; const abortController = new AbortController(); - this.messages.addMessage({ - severity: MessageSeverity.info, - message: "Loading data source", - }); registry .get({ - chunkManager, url: spec.url, - abortSignal: abortController.signal, + signal: abortController.signal, globalCoordinateSpace: layer.manager.root.coordinateSpace, transform: spec.transform, state: spec.state, + progressListener: this.progressListener, }) - .then((source: DataSource) => { + .then((source: DataSourceWithRedirectInfo) => { if (refCounted.wasDisposed) return; this.messages.clearMessages(); const loaded = refCounted.registerDisposer( @@ -437,15 +436,24 @@ export class LayerDataSource extends RefCounted { }), ); } + const { originalCanonicalUrl } = source; + const layerType = this.layer.type; + if ( + (layerType === "auto" || layerType === "new" || spec.setManually) && + originalCanonicalUrl !== undefined && + originalCanonicalUrl !== spec.url + ) { + this.spec = { ...spec, url: originalCanonicalUrl }; + } retainer(); }) - .catch((error: Error) => { + .catch((error) => { if (this.wasDisposed) return; this.loadState_ = { error }; this.messages.clearMessages(); this.messages.addMessage({ severity: MessageSeverity.error, - message: error.message, + message: formatErrorMessage(error), }); this.changed.dispatch(); }); diff --git a/src/layer/segmentation/index.ts b/src/layer/segmentation/index.ts index 5efdc0ca8f..8551e8c576 100644 --- a/src/layer/segmentation/index.ts +++ b/src/layer/segmentation/index.ts @@ -22,7 +22,7 @@ import type { DataSourceSpecification } from "#src/datasource/index.js"; import { LocalDataSource, localEquivalencesUrl, -} from "#src/datasource/index.js"; +} from "#src/datasource/local.js"; import type { LayerActionContext, ManagedUserLayer } from "#src/layer/index.js"; import { LinkedLayerGroup, diff --git a/src/main.bundle.js b/src/main.bundle.js index 511bdfb00c..2cc375487b 100644 --- a/src/main.bundle.js +++ b/src/main.bundle.js @@ -1,4 +1,5 @@ import "#src/util/polyfills.js"; import "#src/layer/enabled_frontend_modules.js"; import "#src/datasource/enabled_frontend_modules.js"; +import "#src/kvstore/enabled_frontend_modules.js"; import "#main"; diff --git a/src/main_module.ts b/src/main_module.ts index b232673900..79af4f9d24 100644 --- a/src/main_module.ts +++ b/src/main_module.ts @@ -1,2 +1,4 @@ +import "#src/util/polyfills.js"; import "#src/layer/enabled_frontend_modules.js"; import "#src/datasource/enabled_frontend_modules.js"; +import "#src/kvstore/enabled_frontend_modules.js"; diff --git a/src/main_python.ts b/src/main_python.ts index 64bf3075a7..8357627dd5 100644 --- a/src/main_python.ts +++ b/src/main_python.ts @@ -21,8 +21,7 @@ import { debounce } from "lodash-es"; import { CachingCredentialsManager } from "#src/credentials_provider/index.js"; -import { getDefaultDataSourceProvider } from "#src/datasource/default_provider.js"; -import { PythonDataSource } from "#src/datasource/python/frontend.js"; +import type { PythonDataSource } from "#src/datasource/python/frontend.js"; import { Client, ClientStateReceiver, @@ -114,21 +113,19 @@ const client = new Client(); const credentialsManager = new PythonCredentialsManager(client); -const dataSourceProvider = getDefaultDataSourceProvider({ +const viewer = ((window).viewer = makeDefaultViewer({ + showLayerDialog: false, + resetStateWhenEmpty: false, credentialsManager: new CachingCredentialsManager(credentialsManager), -}); -const pythonDataSource = new PythonDataSource(); -dataSourceProvider.register("python", pythonDataSource); +})); + +const pythonDataSource = viewer.dataSourceProvider.dataSources.get( + "python", +) as PythonDataSource; configState.add( "sourceGenerations", makeTrackableBasedSourceGenerationHandler(pythonDataSource), ); - -const viewer = ((window).viewer = makeDefaultViewer({ - showLayerDialog: false, - resetStateWhenEmpty: false, - dataSourceProvider, -})); setDefaultInputEventBindings(viewer.inputEventBindings); configState.add( "inputEventBindings", @@ -151,7 +148,10 @@ let sharedState: Trackable | undefined = viewer.state; if (window.location.hash) { const hashBinding = viewer.registerDisposer( - new UrlHashBinding(viewer.state, credentialsManager), + new UrlHashBinding( + viewer.state, + viewer.dataSourceProvider.sharedKvStoreContext, + ), ); hashBinding.updateFromUrlHash(); sharedState = undefined; @@ -159,7 +159,7 @@ if (window.location.hash) { const prefetchManager = new PrefetchManager( viewer.display, - dataSourceProvider, + viewer.dataSourceProvider, viewer.dataContext.addRef(), viewer.uiConfiguration, ); diff --git a/src/mesh/backend.ts b/src/mesh/backend.ts index ce44423e07..fe8191a1b5 100644 --- a/src/mesh/backend.ts +++ b/src/mesh/backend.ts @@ -369,10 +369,7 @@ export function decodeTriangleVertexPositionsAndIndices( export interface MeshSource { // TODO(jbms): Move this declaration to class definition below and declare abstract once // TypeScript supports mixins with abstract classes. - downloadFragment( - chunk: FragmentChunk, - abortSignal: AbortSignal, - ): Promise; + downloadFragment(chunk: FragmentChunk, signal: AbortSignal): Promise; } // eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging @@ -423,8 +420,8 @@ export class MeshSource extends ChunkSource { @registerSharedObject(FRAGMENT_SOURCE_RPC_ID) export class FragmentSource extends ChunkSource { meshSource: MeshSource | null = null; - download(chunk: FragmentChunk, abortSignal: AbortSignal) { - return this.meshSource!.downloadFragment(chunk, abortSignal); + download(chunk: FragmentChunk, signal: AbortSignal) { + return this.meshSource!.downloadFragment(chunk, signal); } } @@ -569,7 +566,7 @@ export interface MultiscaleMeshSource { // TypeScript supports mixins with abstract classes. downloadFragment( chunk: MultiscaleFragmentChunk, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise; } @@ -621,8 +618,8 @@ export class MultiscaleMeshSource extends ChunkSource { @registerSharedObject(MULTISCALE_FRAGMENT_SOURCE_RPC_ID) export class MultiscaleFragmentSource extends ChunkSource { meshSource: MultiscaleMeshSource | null = null; - download(chunk: MultiscaleFragmentChunk, abortSignal: AbortSignal) { - return this.meshSource!.downloadFragment(chunk, abortSignal); + download(chunk: MultiscaleFragmentChunk, signal: AbortSignal) { + return this.meshSource!.downloadFragment(chunk, signal); } } diff --git a/src/python_integration/prefetch.ts b/src/python_integration/prefetch.ts index c75d7caf12..28f9ae0370 100644 --- a/src/python_integration/prefetch.ts +++ b/src/python_integration/prefetch.ts @@ -20,7 +20,8 @@ */ import { debounce } from "lodash-es"; -import type { DataSourceProviderRegistry } from "#src/datasource/index.js"; +import type { DataManagementContext } from "#src/data_management_context.js"; +import type { DataSourceRegistry } from "#src/datasource/index.js"; import type { DisplayContext } from "#src/display_context.js"; import type { Borrowed, Owned } from "#src/util/disposable.js"; import { RefCounted } from "#src/util/disposable.js"; @@ -31,10 +32,7 @@ import { verifyObjectProperty, } from "#src/util/json.js"; import { NullarySignal } from "#src/util/signal.js"; -import type { - DataManagementContext, - ViewerUIConfiguration, -} from "#src/viewer.js"; +import type { ViewerUIConfiguration } from "#src/viewer.js"; import { Viewer } from "#src/viewer.js"; import { WatchableVisibilityPriority } from "#src/visibility_priority/frontend.js"; @@ -46,7 +44,7 @@ export class PrefetchManager extends RefCounted { constructor( public display: Borrowed, - public dataSourceProvider: DataSourceProviderRegistry, + public dataSourceProvider: DataSourceRegistry, public dataContext: Owned, public uiConfiguration: ViewerUIConfiguration, ) { diff --git a/src/python_integration/volume.ts b/src/python_integration/volume.ts index a3a9f7fbd0..39d1d5a93a 100644 --- a/src/python_integration/volume.ts +++ b/src/python_integration/volume.ts @@ -420,7 +420,7 @@ export class VolumeRequestHandler extends RefCounted { private maybeHandleRequest( request: VolumeRequest, - abortSignal: AbortSignal, + signal: AbortSignal, ): boolean { const layer = this.viewer.layerManager.getLayerByName(request.layer); if (layer === undefined) { @@ -586,7 +586,7 @@ export class VolumeRequestHandler extends RefCounted { order: info.order, }; }, - abortSignal, + { signal }, ); } catch (e) { response = { error: e.message }; diff --git a/src/single_mesh/backend.ts b/src/single_mesh/backend.ts index ba1c8f0f62..3357260995 100644 --- a/src/single_mesh/backend.ts +++ b/src/single_mesh/backend.ts @@ -14,17 +14,14 @@ * limitations under the License. */ -import type { ChunkManager } from "#src/chunk_manager/backend.js"; import { Chunk, ChunkSource, withChunkManager, WithParameters, } from "#src/chunk_manager/backend.js"; -import { ChunkPriorityTier } from "#src/chunk_manager/base.js"; -import type { PriorityGetter } from "#src/chunk_manager/generic_file_source.js"; -import type { SharedCredentialsProviderCounterpart } from "#src/credentials_provider/shared_counterpart.js"; -import { WithSharedCredentialsProviderCounterpart } from "#src/credentials_provider/shared_counterpart.js"; +import type { SharedKvStoreContextCounterpart } from "#src/kvstore/backend.js"; +import { WithSharedKvStoreContextCounterpart } from "#src/kvstore/backend.js"; import { computeVertexNormals } from "#src/mesh/backend.js"; import type { SingleMeshData, @@ -40,10 +37,7 @@ import { } from "#src/single_mesh/base.js"; import type { TypedArray } from "#src/util/array.js"; import { stableStringify } from "#src/util/json.js"; -import type { - SpecialProtocolCredentials, - SpecialProtocolCredentialsProvider, -} from "#src/util/special_protocol_request.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; import { getBasePriority, getPriorityTier, @@ -112,11 +106,9 @@ export interface SingleMeshVertexAttributes { interface SingleMeshFactory { description?: string; getMesh: ( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContextCounterpart, url: string, - getPriority: PriorityGetter, - abortSignal: AbortSignal, + options: Partial, ) => Promise; } @@ -149,20 +141,12 @@ function getDataSource( } export function getMesh( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContextCounterpart, url: string, - getPriority: PriorityGetter, - abortSignal: AbortSignal, + options: Partial, ) { const [factory, path] = getDataSource(singleMeshFactories, url); - return factory.getMesh( - chunkManager, - credentialsProvider, - path, - getPriority, - abortSignal, - ); + return factory.getMesh(sharedKvStoreContext, path, options); } export function getMinMax(array: TypedArray): [number, number] { @@ -176,26 +160,16 @@ export function getMinMax(array: TypedArray): [number, number] { } export function getCombinedMesh( - chunkManager: ChunkManager, - credentialsProvider: SpecialProtocolCredentialsProvider, + sharedKvStoreContext: SharedKvStoreContextCounterpart, parameters: SingleMeshSourceParameters, - getPriority: PriorityGetter, - abortSignal: AbortSignal, + options: Partial, ) { - return getMesh( - chunkManager, - credentialsProvider, - parameters.meshSourceUrl, - getPriority, - abortSignal, - ); + return getMesh(sharedKvStoreContext, parameters.meshSourceUrl, options); } @registerSharedObject() export class SingleMeshSource extends WithParameters( - WithSharedCredentialsProviderCounterpart()( - ChunkSource, - ), + WithSharedKvStoreContextCounterpart(ChunkSource), SingleMeshSourceParametersWithInfo, ) { getChunk() { @@ -209,31 +183,24 @@ export class SingleMeshSource extends WithParameters( return chunk; } - download(chunk: SingleMeshChunk, abortSignal: AbortSignal) { - const getPriority = () => ({ - priorityTier: chunk.priorityTier, - priority: chunk.priority, - }); - return getCombinedMesh( - this.chunkManager, - this.credentialsProvider, + async download(chunk: SingleMeshChunk, signal: AbortSignal) { + const data = await getCombinedMesh( + this.sharedKvStoreContext, this.parameters, - getPriority, - abortSignal, - ).then((data) => { - if ( - stableStringify(data.info) !== stableStringify(this.parameters.info) - ) { - throw new Error("Mesh info has changed."); - } - if (data.vertexNormals === undefined) { - data.vertexNormals = computeVertexNormals( - data.vertexPositions, - data.indices, - ); - } - chunk.data = data; - }); + { + signal, + }, + ); + if (stableStringify(data.info) !== stableStringify(this.parameters.info)) { + throw new Error("Mesh info has changed."); + } + if (data.vertexNormals === undefined) { + data.vertexNormals = computeVertexNormals( + data.vertexPositions, + data.indices, + ); + } + chunk.data = data; } } @@ -273,33 +240,18 @@ export class SingleMeshLayer extends SingleMeshLayerBase { } } -const INFO_PRIORITY = 1000; - registerPromiseRPC( GET_SINGLE_MESH_INFO_RPC_ID, - async function (x, abortSignal): RPCPromise { - const chunkManager = this.getRef(x.chunkManager); - const credentialsProvider = this.getOptionalRef< - SharedCredentialsProviderCounterpart< - Exclude - > - >(x.credentialsProvider); - try { - const parameters = x.parameters; - const mesh = await getCombinedMesh( - chunkManager, - credentialsProvider, - parameters, - () => ({ - priorityTier: ChunkPriorityTier.VISIBLE, - priority: INFO_PRIORITY, - }), - abortSignal, - ); - return { value: mesh.info }; - } finally { - chunkManager.dispose(); - credentialsProvider?.dispose(); - } + async function (x, progressOptions): RPCPromise { + const sharedKvStoreContext = this.get( + x.sharedKvStoreContext, + ) as SharedKvStoreContextCounterpart; + const parameters = x.parameters; + const mesh = await getCombinedMesh( + sharedKvStoreContext, + parameters, + progressOptions, + ); + return { value: mesh.info }; }, ); diff --git a/src/single_mesh/frontend.ts b/src/single_mesh/frontend.ts index 8aa26f86b8..0fae89510b 100644 --- a/src/single_mesh/frontend.ts +++ b/src/single_mesh/frontend.ts @@ -15,17 +15,13 @@ */ import { ChunkState } from "#src/chunk_manager/base.js"; -import type { ChunkManager } from "#src/chunk_manager/frontend.js"; import { Chunk, ChunkSource, WithParameters, } from "#src/chunk_manager/frontend.js"; -import { - getCredentialsProviderCounterpart, - WithCredentialsProvider, -} from "#src/credentials_provider/chunk_source_frontend.js"; -import type { CredentialsManager } from "#src/credentials_provider/index.js"; +import { WithSharedKvStoreContext } from "#src/kvstore/chunk_source_frontend.js"; +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; import type { PickState, VisibleLayerInfo } from "#src/layer/index.js"; import type { PerspectivePanel } from "#src/perspective_view/panel.js"; import type { PerspectiveViewRenderContext } from "#src/perspective_view/render_layer.js"; @@ -47,8 +43,7 @@ import { WatchableValue } from "#src/trackable_value.js"; import { DataType } from "#src/util/data_type.js"; import type { mat4 } from "#src/util/geom.js"; import { vec3 } from "#src/util/geom.js"; -import type { SpecialProtocolCredentials } from "#src/util/special_protocol_request.js"; -import { parseSpecialUrl } from "#src/util/special_protocol_request.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; import { withSharedVisibility } from "#src/visibility_priority/frontend.js"; import type { Buffer } from "#src/webgl/buffer.js"; import { glsl_COLORMAPS } from "#src/webgl/colormaps.js"; @@ -439,7 +434,7 @@ export function getAttributeTextureFormats( } export class SingleMeshSource extends WithParameters( - WithCredentialsProvider()(ChunkSource), + WithSharedKvStoreContext(ChunkSource), SingleMeshSourceParametersWithInfo, ) { attributeTextureFormats = getAttributeTextureFormats( @@ -650,46 +645,39 @@ export class SingleMeshLayer extends PerspectiveViewRenderLayer, ) { - return chunkManager.memoize.getUncounted( + return sharedKvStoreContext.chunkManager.memoize.getAsync( { type: "single_mesh:getMeshInfo", url }, - async () => { - const { url: parsedUrl, credentialsProvider } = parseSpecialUrl( - url, - credentialsManager, - ); - const info = await chunkManager.rpc!.promiseInvoke( - GET_SINGLE_MESH_INFO_RPC_ID, - { - chunkManager: chunkManager.addCounterpartRef(), - credentialsProvider: - getCredentialsProviderCounterpart( - chunkManager, - credentialsProvider, - ), - parameters: { meshSourceUrl: parsedUrl }, - }, - ); - return { info, url: parsedUrl, credentialsProvider }; + options, + async (progressOptions) => { + const info = + await sharedKvStoreContext.chunkManager.rpc!.promiseInvoke( + GET_SINGLE_MESH_INFO_RPC_ID, + { + sharedKvStoreContext: sharedKvStoreContext.rpcId, + parameters: { meshSourceUrl: url }, + }, + { + signal: progressOptions.signal, + progressListener: options.progressListener, + }, + ); + return info; }, ); } export async function getSingleMeshSource( - chunkManager: ChunkManager, - credentialsManager: CredentialsManager, + sharedKvStoreContext: SharedKvStoreContext, url: string, + options: Partial, ) { - const { - info, - url: parsedUrl, - credentialsProvider, - } = await getSingleMeshInfo(chunkManager, credentialsManager, url); - return chunkManager.getChunkSource(SingleMeshSource, { - credentialsProvider, - parameters: { meshSourceUrl: parsedUrl, info }, + const info = await getSingleMeshInfo(sharedKvStoreContext, url, options); + return sharedKvStoreContext.chunkManager.getChunkSource(SingleMeshSource, { + sharedKvStoreContext, + parameters: { meshSourceUrl: url, info }, }); } diff --git a/src/sliceview/backend.ts b/src/sliceview/backend.ts index 1612ea580f..a921901d2e 100644 --- a/src/sliceview/backend.ts +++ b/src/sliceview/backend.ts @@ -535,7 +535,7 @@ registerPromiseRPC( SLICEVIEW_REQUEST_CHUNK_RPC_ID, async function ( x: { this: RPC; source: number; chunkGridPosition: Float32Array }, - abortSignal: AbortSignal, + progressOptions, ): RPCPromise { const source = this.get(x.source) as SliceViewChunkSourceBackend; const { chunkManager } = source; @@ -572,7 +572,7 @@ registerPromiseRPC( }); source.registerChunkListener(key, listener!); try { - await raceWithAbort(promise, abortSignal); + await raceWithAbort(promise, progressOptions.signal); return { value: undefined }; } finally { source.unregisterChunkListener(key, listener!); diff --git a/src/sliceview/backend_chunk_decoders/bossNpz.ts b/src/sliceview/backend_chunk_decoders/bossNpz.ts index 105dea4e34..250cc3ba4e 100644 --- a/src/sliceview/backend_chunk_decoders/bossNpz.ts +++ b/src/sliceview/backend_chunk_decoders/bossNpz.ts @@ -31,7 +31,7 @@ import { parseNpy } from "#src/util/npy.js"; export async function decodeBossNpzChunk( chunk: VolumeChunk, - abortSignal: AbortSignal, + signal: AbortSignal, response: ArrayBuffer, ) { const parseResult = parseNpy( @@ -61,5 +61,5 @@ export async function decodeBossNpzChunk( } does not match expected data type ${DataType[spec.dataType]}`, ); } - await postProcessRawData(chunk, abortSignal, parseResult.data); + await postProcessRawData(chunk, signal, parseResult.data); } diff --git a/src/sliceview/backend_chunk_decoders/compressed_segmentation.ts b/src/sliceview/backend_chunk_decoders/compressed_segmentation.ts index 534bb78c86..2a9a6d8358 100644 --- a/src/sliceview/backend_chunk_decoders/compressed_segmentation.ts +++ b/src/sliceview/backend_chunk_decoders/compressed_segmentation.ts @@ -18,9 +18,9 @@ import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; export async function decodeCompressedSegmentationChunk( chunk: VolumeChunk, - abortSignal: AbortSignal, + signal: AbortSignal, response: ArrayBuffer, ) { - abortSignal; + signal; chunk.data = new Uint32Array(response); } diff --git a/src/sliceview/backend_chunk_decoders/compresso.ts b/src/sliceview/backend_chunk_decoders/compresso.ts index 322a725e86..c4e3ac680e 100644 --- a/src/sliceview/backend_chunk_decoders/compresso.ts +++ b/src/sliceview/backend_chunk_decoders/compresso.ts @@ -21,15 +21,15 @@ import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; export async function decodeCompressoChunk( chunk: VolumeChunk, - abortSignal: AbortSignal, + signal: AbortSignal, response: ArrayBuffer, ) { const image = await requestAsyncComputation( decodeCompresso, - abortSignal, + signal, [response], new Uint8Array(response), ); - await decodeRawChunk(chunk, abortSignal, image.buffer); + await decodeRawChunk(chunk, signal, image.buffer); } diff --git a/src/sliceview/backend_chunk_decoders/index.ts b/src/sliceview/backend_chunk_decoders/index.ts index 169af5c267..f9e4e94e83 100644 --- a/src/sliceview/backend_chunk_decoders/index.ts +++ b/src/sliceview/backend_chunk_decoders/index.ts @@ -22,6 +22,6 @@ import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; export type ChunkDecoder = ( chunk: VolumeChunk, - abortSignal: AbortSignal, + signal: AbortSignal, response: ArrayBuffer, ) => Promise; diff --git a/src/sliceview/backend_chunk_decoders/jpeg.ts b/src/sliceview/backend_chunk_decoders/jpeg.ts index 34bdc8138f..6a9ede1553 100644 --- a/src/sliceview/backend_chunk_decoders/jpeg.ts +++ b/src/sliceview/backend_chunk_decoders/jpeg.ts @@ -21,13 +21,13 @@ import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; export async function decodeJpegChunk( chunk: VolumeChunk, - abortSignal: AbortSignal, + signal: AbortSignal, response: ArrayBuffer, ) { const chunkDataSize = chunk.chunkDataSize!; const { uint8Array: decoded } = await requestAsyncComputation( decodeJpeg, - abortSignal, + signal, [response], new Uint8Array(response), undefined, @@ -36,5 +36,5 @@ export async function decodeJpegChunk( chunkDataSize[3] || 1, false, ); - await postProcessRawData(chunk, abortSignal, decoded); + await postProcessRawData(chunk, signal, decoded); } diff --git a/src/sliceview/backend_chunk_decoders/jxl.ts b/src/sliceview/backend_chunk_decoders/jxl.ts index 38e529d508..f515d8f856 100644 --- a/src/sliceview/backend_chunk_decoders/jxl.ts +++ b/src/sliceview/backend_chunk_decoders/jxl.ts @@ -21,18 +21,18 @@ import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; export async function decodeJxlChunk( chunk: VolumeChunk, - abortSignal: AbortSignal, + signal: AbortSignal, response: ArrayBuffer, ) { const chunkDataSize = chunk.chunkDataSize!; const { uint8Array: decoded } = await requestAsyncComputation( decodeJxl, - abortSignal, + signal, [response], new Uint8Array(response), chunkDataSize[0] * chunkDataSize[1] * chunkDataSize[2], chunkDataSize[3] || 1, 1, // bytesPerPixel ); - await postProcessRawData(chunk, abortSignal, decoded); + await postProcessRawData(chunk, signal, decoded); } diff --git a/src/sliceview/backend_chunk_decoders/ndstoreNpz.ts b/src/sliceview/backend_chunk_decoders/ndstoreNpz.ts index 44d554d83c..210129fd1d 100644 --- a/src/sliceview/backend_chunk_decoders/ndstoreNpz.ts +++ b/src/sliceview/backend_chunk_decoders/ndstoreNpz.ts @@ -31,7 +31,7 @@ import { parseNpy } from "#src/util/npy.js"; export async function decodeNdstoreNpzChunk( chunk: VolumeChunk, - abortSignal: AbortSignal, + signal: AbortSignal, response: ArrayBuffer, ) { const parseResult = parseNpy( @@ -55,5 +55,5 @@ export async function decodeNdstoreNpzChunk( `expected data type ${DataType[spec.dataType]}`, ); } - await postProcessRawData(chunk, abortSignal, parseResult.data); + await postProcessRawData(chunk, signal, parseResult.data); } diff --git a/src/sliceview/backend_chunk_decoders/png.ts b/src/sliceview/backend_chunk_decoders/png.ts index 7f6404e161..22bb9f37e9 100644 --- a/src/sliceview/backend_chunk_decoders/png.ts +++ b/src/sliceview/backend_chunk_decoders/png.ts @@ -22,14 +22,14 @@ import { DATA_TYPE_BYTES } from "#src/util/data_type.js"; export async function decodePngChunk( chunk: VolumeChunk, - abortSignal: AbortSignal, + signal: AbortSignal, response: ArrayBuffer, ) { const chunkDataSize = chunk.chunkDataSize!; const dataType = chunk.source!.spec.dataType; const { uint8Array: image } = await requestAsyncComputation( decodePng, - abortSignal, + signal, [response], /*buffer=*/ new Uint8Array(response), /*width=*/ undefined, @@ -40,5 +40,5 @@ export async function decodePngChunk( /*convertToGrayscale=*/ false, ); - await decodeRawChunk(chunk, abortSignal, image.buffer); + await decodeRawChunk(chunk, signal, image.buffer); } diff --git a/src/sliceview/backend_chunk_decoders/postprocess.ts b/src/sliceview/backend_chunk_decoders/postprocess.ts index de445db5d1..23bbf0d89a 100644 --- a/src/sliceview/backend_chunk_decoders/postprocess.ts +++ b/src/sliceview/backend_chunk_decoders/postprocess.ts @@ -28,7 +28,7 @@ import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; export async function postProcessRawData( chunk: VolumeChunk, - abortSignal: AbortSignal, + signal: AbortSignal, data: ArrayBufferView, ) { const { spec } = chunk.source!; @@ -45,7 +45,7 @@ export async function postProcessRawData( case DataType.UINT32: chunk.data = await requestAsyncComputation( encodeCompressedSegmentationUint32, - abortSignal, + signal, [data.buffer], data as Uint32Array, shape, @@ -55,7 +55,7 @@ export async function postProcessRawData( case DataType.UINT64: chunk.data = await requestAsyncComputation( encodeCompressedSegmentationUint64, - abortSignal, + signal, [data.buffer], data as Uint32Array, shape, diff --git a/src/sliceview/backend_chunk_decoders/raw.ts b/src/sliceview/backend_chunk_decoders/raw.ts index b7bc763f24..2a883c16d3 100644 --- a/src/sliceview/backend_chunk_decoders/raw.ts +++ b/src/sliceview/backend_chunk_decoders/raw.ts @@ -23,13 +23,13 @@ import * as vector from "#src/util/vector.js"; export async function decodeRawChunk( chunk: VolumeChunk, - abortSignal: AbortSignal, + signal: AbortSignal, response: ArrayBuffer, endianness: Endianness = ENDIANNESS, byteOffset = 0, byteLength: number = response.byteLength, ) { - abortSignal; + signal; const { spec } = chunk.source!; const { dataType } = spec; const numElements = vector.prod(chunk.chunkDataSize!); @@ -48,5 +48,5 @@ export async function decodeRawChunk( byteLength, ); convertEndian(data, endianness, bytesPerElement); - await postProcessRawData(chunk, abortSignal, data); + await postProcessRawData(chunk, signal, data); } diff --git a/src/sliceview/compressed_segmentation/encode.benchmark.ts b/src/sliceview/compressed_segmentation/encode.benchmark.ts index dcd19fbc49..022c72ce66 100644 --- a/src/sliceview/compressed_segmentation/encode.benchmark.ts +++ b/src/sliceview/compressed_segmentation/encode.benchmark.ts @@ -32,7 +32,12 @@ describe("64x64x64 example", async () => { "testdata", ); const exampleChunkDataUint8Array = await fs.readFile( - path.resolve(testDataDir, "64x64x64-raw-uint64-segmentation.dat"), + path.resolve( + testDataDir, + "codec", + "compressed_segmentation", + "64x64x64-raw-uint64-segmentation.dat", + ), ); const exampleChunkData64 = new Uint32Array(exampleChunkDataUint8Array.buffer); const exampleChunkData32 = exampleChunkData64.filter((_element, index) => { diff --git a/src/sliceview/frontend.ts b/src/sliceview/frontend.ts index e61dd0ab01..18c6ee50d7 100644 --- a/src/sliceview/frontend.ts +++ b/src/sliceview/frontend.ts @@ -68,6 +68,7 @@ import type { vec4 } from "#src/util/geom.js"; import { kOneVec, kZeroVec4, mat4, vec3 } from "#src/util/geom.js"; import { MessageList, MessageSeverity } from "#src/util/message_list.js"; import { getObjectId } from "#src/util/object_id.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; import { NullarySignal } from "#src/util/signal.js"; import { withSharedVisibility } from "#src/visibility_priority/frontend.js"; import type { GL } from "#src/webgl/context.js"; @@ -650,7 +651,7 @@ export abstract class SliceViewChunkSource< async fetchChunk( chunkGridPosition: Float32Array, transform: (chunk: Chunk) => T, - abortSignal?: AbortSignal, + progressOptions: Partial, ): Promise { const key = chunkGridPosition.join(); const existingChunk = this.chunks.get(key); @@ -679,7 +680,7 @@ export abstract class SliceViewChunkSource< await this.rpc!.promiseInvoke( SLICEVIEW_REQUEST_CHUNK_RPC_ID, { source: this.rpcId, chunkGridPosition }, - abortSignal, + progressOptions, ); return await promise; } finally { diff --git a/src/status.css b/src/status.css index a5c9c922a8..bf4ab50873 100644 --- a/src/status.css +++ b/src/status.css @@ -63,6 +63,7 @@ background-color: #808080; padding: 20px; padding-top: 30px; + border: 1px solid white; } #neuroglancer-status-container-modal > div > div:first-child { diff --git a/src/status.ts b/src/status.ts index 07b2deb6d9..2507cdc6f4 100644 --- a/src/status.ts +++ b/src/status.ts @@ -21,6 +21,9 @@ import { makeCloseButton } from "#src/widget/close_button.js"; let statusContainer: HTMLElement | undefined; let modalStatusContainer: HTMLElement | undefined; +// Exported for use by #tests/fixtures/status_message_handler.js +export const statusMessages = new Set(); + export const DEFAULT_STATUS_DELAY = 200; export type Delay = boolean | number; @@ -57,6 +60,11 @@ function getModalStatusContainer() { return modalStatusContainer; } +// For use by #tests/fixtures/status_message_handler.js +export function getStatusMessageContainers() { + return [getStatusContainer(), getModalStatusContainer()]; +} + export class StatusMessage { element: HTMLElement; private modalElementWrapper: HTMLElement | undefined; @@ -75,6 +83,7 @@ export class StatusMessage { } else { this.timer = null; } + statusMessages.add(this); } [Symbol.dispose]() { @@ -90,6 +99,7 @@ export class StatusMessage { if (this.timer !== null) { clearTimeout(this.timer); } + statusMessages.delete(this); } setText(text: string, makeVisible?: boolean) { this.element.textContent = text; diff --git a/src/ui/default_viewer_setup.ts b/src/ui/default_viewer_setup.ts index fc5d5aeaee..fc6bb1a406 100644 --- a/src/ui/default_viewer_setup.ts +++ b/src/ui/default_viewer_setup.ts @@ -37,7 +37,7 @@ export function setupDefaultViewer(options?: Partial) { const hashBinding = viewer.registerDisposer( new UrlHashBinding( viewer.state, - viewer.dataSourceProvider.credentialsManager, + viewer.dataSourceProvider.sharedKvStoreContext, { defaultFragment: typeof NEUROGLANCER_DEFAULT_STATE_FRAGMENT !== "undefined" diff --git a/src/ui/layer_data_sources_tab.css b/src/ui/layer_data_sources_tab.css index 0fa84d0b20..f72fe24c2e 100644 --- a/src/ui/layer_data_sources_tab.css +++ b/src/ui/layer_data_sources_tab.css @@ -112,3 +112,13 @@ li.neuroglancer-message-info { .neuroglancer-layer-data-sources-tab-type-detection-type { font-weight: bold; } + +.neuroglancer-layer-data-sources-tab .neuroglancer-progress { + margin: 0px; + list-style: none; + padding: 0px; +} + +.neuroglancer-layer-data-sources-tab .neuroglancer-progress > li { + color: #ccc; +} diff --git a/src/ui/layer_data_sources_tab.ts b/src/ui/layer_data_sources_tab.ts index d3f94ef32a..866acd4850 100644 --- a/src/ui/layer_data_sources_tab.ts +++ b/src/ui/layer_data_sources_tab.ts @@ -19,7 +19,7 @@ */ import "#src/ui/layer_data_sources_tab.css"; -import { LocalDataSource } from "#src/datasource/index.js"; +import { LocalDataSource } from "#src/datasource/local.js"; import type { UserLayer, UserLayerConstructor } from "#src/layer/index.js"; import { changeLayerName, @@ -50,34 +50,62 @@ import { } from "#src/util/dom.js"; import type { MessageList } from "#src/util/message_list.js"; import { MessageSeverity } from "#src/util/message_list.js"; +import type { ProgressListener } from "#src/util/progress_listener.js"; import { makeAddButton } from "#src/widget/add_button.js"; import { CoordinateSpaceTransformWidget } from "#src/widget/coordinate_transform.js"; -import type { Completer } from "#src/widget/multiline_autocomplete.js"; +import type { + Completer, + SyntaxHighlighter, +} from "#src/widget/multiline_autocomplete.js"; import { AutocompleteTextInput, makeCompletionElementWithDescription, } from "#src/widget/multiline_autocomplete.js"; +import { ProgressListenerWidget } from "#src/widget/progress_listener.js"; import { Tab } from "#src/widget/tab_view.js"; +const dataSourceUrlSyntaxHighlighter: SyntaxHighlighter = { + splitPattern: /\|?[^|:/_]*(?:[:/_]+)?/g, + getSeparatorNode: (text: string) => { + if (text.startsWith("|") && text.length > 1) { + // Create an empty span with CSS class that adds `::after` node with + // content "\a". This prevents the linebreak from affecting text selection. + const node = document.createElement("span"); + node.classList.add("neuroglancer-multiline-autocomplete-linebreak"); + return node; + } else { + return document.createElement("wbr"); + } + }, +}; + class SourceUrlAutocomplete extends AutocompleteTextInput { dataSourceView: DataSourceView; dirty: WatchableValueInterface; constructor(dataSourceView: DataSourceView) { const { manager } = dataSourceView.source.layer; - const sourceCompleter: Completer = ({ value }, abortSignal: AbortSignal) => - manager.dataSourceProviderRegistry - .completeUrl({ + const sourceCompleter: Completer = async ( + { value }, + signal: AbortSignal, + progressListener: ProgressListener, + ) => { + const originalResult = + await manager.dataSourceProviderRegistry.completeUrl({ url: value, - chunkManager: manager.chunkManager, - abortSignal, - }) - .then((originalResult) => ({ - completions: originalResult.completions, - makeElement: makeCompletionElementWithDescription, - offset: originalResult.offset, - showSingleResult: true, - })); - super({ completer: sourceCompleter, delay: 0 }); + signal, + progressListener, + }); + return { + ...originalResult, + makeElement: makeCompletionElementWithDescription, + showSingleResult: true, + }; + }; + super({ + completer: sourceCompleter, + syntaxHighlighter: dataSourceUrlSyntaxHighlighter, + delay: 0, + }); this.placeholder = "Data source URL"; this.dataSourceView = dataSourceView; this.element.classList.add("neuroglancer-layer-data-source-url-input"); @@ -306,14 +334,6 @@ export class DataSourceView extends RefCounted { const { source } = this; const existingSpec = source.spec; const userLayer = this.source.layer; - url = userLayer.manager.dataSourceProviderRegistry.normalizeUrl({ url }); - if (url !== urlInput.value) { - urlInput.disableCompletion(); - urlInput.setValueAndSelection(url, { - begin: url.length, - end: url.length, - }); - } urlInput.dirty.value = false; // If url is non-empty and unchanged, don't set spec, as that would trigger a reload of the // data source. If the url is empty, always set spec in order to possible remove the empty @@ -330,12 +350,14 @@ export class DataSourceView extends RefCounted { try { const newName = userLayer.manager.dataSourceProviderRegistry.suggestLayerName(url); - changeLayerName(userLayer.managedLayer, newName); + if (newName) { + changeLayerName(userLayer.managedLayer, newName); + } } catch { // Ignore errors obtaining a suggested layer name. } } - source.spec = { ...existingSpec, url }; + source.spec = { ...existingSpec, url, setManually: true }; }; urlInput.onCommit.add(updateUrlFromView); @@ -345,6 +367,12 @@ export class DataSourceView extends RefCounted { element.appendChild( this.registerDisposer(new MessagesView(source.messages)).element, ); + const progressListenerWidget = new ProgressListenerWidget(); + element.appendChild(progressListenerWidget.element); + source.progressListener.addListener(progressListenerWidget); + this.registerDisposer(() => + source.progressListener.removeListener(progressListenerWidget), + ); this.updateView(); } diff --git a/src/ui/url_hash_binding.ts b/src/ui/url_hash_binding.ts index cca2bdc77d..978942494c 100644 --- a/src/ui/url_hash_binding.ts +++ b/src/ui/url_hash_binding.ts @@ -15,15 +15,11 @@ */ import { debounce } from "lodash-es"; -import type { CredentialsManager } from "#src/credentials_provider/index.js"; +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; import { StatusMessage } from "#src/status.js"; import { WatchableValue } from "#src/trackable_value.js"; import { RefCounted } from "#src/util/disposable.js"; import { urlSafeParse, verifyObject } from "#src/util/json.js"; -import { - fetchSpecialOk, - parseSpecialUrl, -} from "#src/util/special_protocol_request.js"; import type { Trackable } from "#src/util/trackable.js"; import { getCachedJson } from "#src/util/trackable.js"; @@ -70,7 +66,7 @@ export class UrlHashBinding extends RefCounted { constructor( public root: Trackable, - public credentialsManager: CredentialsManager, + public sharedKvStoreContext: SharedKvStoreContext, options: UrlHashBindingOptions = {}, ) { super(); @@ -121,13 +117,10 @@ export class UrlHashBinding extends RefCounted { // Handle remote JSON state if (s.match(/^#!([a-z][a-z\d+-.]*):\/\//)) { const url = s.substring(2); - const { url: parsedUrl, credentialsProvider } = parseSpecialUrl( - url, - this.credentialsManager, - ); StatusMessage.forPromise( - fetchSpecialOk(credentialsProvider, parsedUrl, {}) - .then((response) => response.json()) + this.sharedKvStoreContext.kvStoreContext + .read(url, { throwIfMissing: true }) + .then((response) => response.response.json()) .then((json) => { verifyObject(json); this.root.reset(); diff --git a/src/util/abort.ts b/src/util/abort.ts index 7b45a2a0e0..da3b37908f 100644 --- a/src/util/abort.ts +++ b/src/util/abort.ts @@ -44,10 +44,10 @@ export class SharedAbortController { return this.controller.signal; } - addConsumer(abortSignal: AbortSignal | undefined): void { + addConsumer(signal: AbortSignal | undefined): void { if (this.controller.signal.aborted) return undefined; - if (abortSignal !== undefined) { - if (abortSignal.aborted) return; + if (signal !== undefined) { + if (signal.aborted) return; const self = this; function wrappedCallback(this: AbortSignal) { self.consumers.delete(wrappedCallback); @@ -56,14 +56,14 @@ export class SharedAbortController { self[Symbol.dispose](); } } - abortSignal.addEventListener("abort", wrappedCallback, { once: true }); + signal.addEventListener("abort", wrappedCallback, { once: true }); } ++this.retainCount; } [Symbol.dispose](): void { - for (const [wrappedCallback, abortSignal] of this.consumers) { - abortSignal.removeEventListener("abort", wrappedCallback); + for (const [wrappedCallback, signal] of this.consumers) { + signal.removeEventListener("abort", wrappedCallback); } this.consumers.clear(); this.retainCount = 0; @@ -78,7 +78,7 @@ export class SharedAbortController { } export function promiseWithResolversAndAbortCallback( - abortSignal: AbortSignal, + signal: AbortSignal, abortCallback: (reason: any) => void, ): { promise: Promise; @@ -86,7 +86,7 @@ export function promiseWithResolversAndAbortCallback( reject: (reason: any) => void; } { const { promise, resolve, reject } = Promise.withResolvers(); - const cleanup = scopedAbortCallback(abortSignal, abortCallback); + const cleanup = scopedAbortCallback(signal, abortCallback); return { promise, resolve: (value: T) => { @@ -102,13 +102,13 @@ export function promiseWithResolversAndAbortCallback( export function raceWithAbort( promise: Promise, - abortSignal: AbortSignal | undefined, + signal: AbortSignal | undefined, ): Promise { - if (abortSignal === undefined) return promise; - if (abortSignal.aborted) return Promise.reject(abortSignal.reason); + if (signal === undefined) return promise; + if (signal.aborted) return Promise.reject(signal.reason); return new Promise((resolve, reject) => { - const cleanup = scopedAbortCallback(abortSignal, (reason) => { + const cleanup = scopedAbortCallback(signal, (reason) => { reject(reason); }); promise.then( @@ -124,12 +124,12 @@ export function raceWithAbort( }); } -export function abortPromise(abortSignal: AbortSignal) { +export function abortPromise(signal: AbortSignal) { return new Promise((_resolve, reject) => { - abortSignal.addEventListener( + signal.addEventListener( "abort", () => { - reject(abortSignal.reason); + reject(signal.reason); }, { once: true }, ); diff --git a/src/util/array.ts b/src/util/array.ts index 240f5b23ce..64d30a29a2 100644 --- a/src/util/array.ts +++ b/src/util/array.ts @@ -170,13 +170,13 @@ export function tile2dArray( return result; } -export function binarySearch( - haystack: ArrayLike, - needle: T, - compare: (a: T, b: T) => number, +export function binarySearch( + haystack: ArrayLike, + needle: Needle, + compare: (a: Needle, b: Hay) => number, low = 0, high = haystack.length, -) { +): number { while (low < high) { const mid = (low + high - 1) >> 1; const compareResult = compare(needle, haystack[mid]); diff --git a/src/util/byte_range_http_requests.ts b/src/util/byte_range_http_requests.ts deleted file mode 100644 index 4e4f505d39..0000000000 --- a/src/util/byte_range_http_requests.ts +++ /dev/null @@ -1,43 +0,0 @@ -/** - * @license - * Copyright 2019 Google Inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { getByteRangeHeader } from "#src/util/http_request.js"; -import type { SpecialProtocolCredentialsProvider } from "#src/util/special_protocol_request.js"; -import { fetchSpecialOk } from "#src/util/special_protocol_request.js"; -import type { Uint64 } from "#src/util/uint64.js"; - -/** - * On Chromium, multiple concurrent byte range requests to the same URL are serialized unless the - * cache is disabled. Disabling the cache works around the problem. - * - * https://bugs.chromium.org/p/chromium/issues/detail?id=969828 - */ -const cacheMode = - navigator.userAgent.indexOf("Chrome") !== -1 ? "no-store" : "default"; - -export function fetchSpecialHttpByteRange( - credentialsProvider: SpecialProtocolCredentialsProvider, - url: string, - startOffset: Uint64 | number, - endOffset: Uint64 | number, - abortSignal: AbortSignal, -): Promise { - return fetchSpecialOk(credentialsProvider, url, { - headers: getByteRangeHeader(startOffset, endOffset), - cache: cacheMode, - signal: abortSignal, - }).then((response) => response.arrayBuffer()); -} diff --git a/src/util/completion.ts b/src/util/completion.ts index a406a70b0d..18fb332501 100644 --- a/src/util/completion.ts +++ b/src/util/completion.ts @@ -26,6 +26,12 @@ export interface CompletionWithDescription extends Completion { export interface BasicCompletionResult { completions: C[]; + // Default completion to show. + // + // If not specified, the longest common prefix of all completions is the + // "default completion" to show inline as a hint and to append if the user + // presses TAB. This option overrides that. + defaultCompletion?: string; offset: number; } diff --git a/src/util/error.ts b/src/util/error.ts index 3dd27e9554..64801ab29a 100644 --- a/src/util/error.ts +++ b/src/util/error.ts @@ -28,3 +28,19 @@ export function valueOrThrow(x: ValueOrError): T { if (x.error !== undefined) throw new Error(x.error); return x; } + +export function formatErrorMessage(error: unknown): string { + if (typeof error === "string") return error; + if (error instanceof Error) { + const { message, cause } = error; + if (cause !== undefined) { + return `${message}: ${formatErrorMessage(cause)}`; + } + return message; + } + try { + return "" + error; + } catch { + return "Unknown error"; + } +} diff --git a/src/util/gcs_bucket_listing.ts b/src/util/gcs_bucket_listing.ts deleted file mode 100644 index ec77de5910..0000000000 --- a/src/util/gcs_bucket_listing.ts +++ /dev/null @@ -1,88 +0,0 @@ -/** - * @license - * Copyright 2020 Google Inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { fetchWithOAuth2Credentials } from "#src/credentials_provider/oauth2.js"; -import type { BasicCompletionResult } from "#src/util/completion.js"; -import { - parseArray, - verifyObject, - verifyObjectProperty, - verifyOptionalObjectProperty, - verifyString, - verifyStringArray, -} from "#src/util/json.js"; -import type { SpecialProtocolCredentialsProvider } from "#src/util/special_protocol_request.js"; - -export async function getGcsBucketListing( - credentialsProvider: SpecialProtocolCredentialsProvider, - bucket: string, - prefix: string, - delimiter: string, - abortSignal: AbortSignal, -): Promise { - // Include origin as `neuroglancerOrigin` query string parameter. See comment in - // `special_protocol_request.ts` for details. - const response = await fetchWithOAuth2Credentials( - credentialsProvider, - `https://www.googleapis.com/storage/v1/b/${bucket}/o?` + - `delimiter=${encodeURIComponent(delimiter)}&prefix=${encodeURIComponent( - prefix, - )}&` + - `neuroglancerOrigin=${encodeURIComponent(location.origin)}`, - { signal: abortSignal }, - ).then((response) => response.json()); - verifyObject(response); - const prefixes = verifyOptionalObjectProperty( - response, - "prefixes", - verifyStringArray, - [], - ); - const items = verifyOptionalObjectProperty( - response, - "items", - (items) => - parseArray(items, (item) => { - verifyObject(item); - return verifyObjectProperty(item, "name", verifyString); - }), - [], - ).filter((name) => !name.endsWith("_$folder$")); - return [...prefixes, ...items]; -} - -export async function getGcsPathCompletions( - credentialsProvider: SpecialProtocolCredentialsProvider, - enteredBucketUrl: string, - bucket: string, - path: string, - abortSignal: AbortSignal, -): Promise { - const prefix = path; - if (!prefix.startsWith("/")) throw null; - const paths = await getGcsBucketListing( - credentialsProvider, - bucket, - path.substring(1), - "/", - abortSignal, - ); - const offset = path.lastIndexOf("/"); - return { - offset: offset + enteredBucketUrl.length + 1, - completions: paths.map((x) => ({ value: x.substring(offset) })), - }; -} diff --git a/src/util/google_oauth2.ts b/src/util/google_oauth2.ts index baa8a312a9..50e8f6241f 100644 --- a/src/util/google_oauth2.ts +++ b/src/util/google_oauth2.ts @@ -29,6 +29,7 @@ import { verifyObjectProperty, verifyString, } from "#src/util/json.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; import { getRandomHexString } from "#src/util/random.js"; export const EMAIL_SCOPE = "email"; @@ -60,11 +61,11 @@ function extractEmailFromIdToken(idToken: string): string { } } -// Note: `abortSignal` is guaranteed to be aborted once the operation completes. +// Note: `signal` is guaranteed to be aborted once the operation completes. function waitForAuthResponseMessage( source: Window, state: string, - abortSignal: AbortSignal, + signal: AbortSignal, ): Promise { return new Promise((resolve, reject) => { window.addEventListener( @@ -108,7 +109,7 @@ function waitForAuthResponseMessage( console.error("Response received: ", event.data); } }, - { signal: abortSignal }, + { signal: signal }, ); }); } @@ -193,7 +194,7 @@ export async function authenticateGoogleOAuth2( immediate?: boolean; authUser?: number; }, - abortSignal: AbortSignal, + signal: AbortSignal, ) { const state = getRandomHexString(); const nonce = getRandomHexString(); @@ -208,7 +209,7 @@ export async function authenticateGoogleOAuth2( authUser: options.authUser, }); const abortController = new AbortController(); - abortSignal = AbortSignal.any([abortController.signal, abortSignal]); + signal = AbortSignal.any([abortController.signal, signal]); try { let source: Window; if (options.immediate) { @@ -223,7 +224,7 @@ export async function authenticateGoogleOAuth2( } return await raceWithAbort( waitForAuthResponseMessage(source, state, abortController.signal), - abortSignal, + signal, ); } finally { abortController.abort(); @@ -237,12 +238,15 @@ export class GoogleOAuth2CredentialsProvider extends CredentialsProvider - getCredentialsWithStatus( + get = makeCredentialsGetter(async (options) => { + using _span = new ProgressSpan(options.progressListener, { + message: `Requesting ${this.options.description} OAuth2 access token`, + }); + return await getCredentialsWithStatus( { description: this.options.description, supportsImmediate: true, - get: (abortSignal, immediate) => + get: (signal, immediate) => authenticateGoogleOAuth2( { clientId: this.options.clientId, @@ -250,10 +254,10 @@ export class GoogleOAuth2CredentialsProvider extends CredentialsProvider 2 && view[0] === 0x1f && view[1] === 0x8b; + return ( + view.length >= 3 && view[0] === 0x1f && view[1] === 0x8b && view[2] === 0x08 + ); } export async function decodeGzip( - data: ArrayBuffer | ArrayBufferView, + data: ArrayBuffer | ArrayBufferView | Response, format: CompressionFormat, - abortSignal?: AbortSignal, + signal?: AbortSignal, ) { try { - const decompressedStream = new Response(data).body!.pipeThrough( - new DecompressionStream(format), - { signal: abortSignal }, + const decompressedStream = decodeGzipStream( + data instanceof Response ? data : new Response(data), + format, + signal, ); return await new Response(decompressedStream).arrayBuffer(); } catch { - abortSignal?.throwIfAborted(); + signal?.throwIfAborted(); throw new Error(`Failed to decode ${format}`); } } +export function decodeGzipStream( + response: Response, + format: CompressionFormat, + signal?: AbortSignal, +): ReadableStream { + return response.body!.pipeThrough(new DecompressionStream(format), { + signal: signal, + }); +} + /** * Decompress `data` if it is in gzip format, otherwise just return it. */ diff --git a/src/util/hash.ts b/src/util/hash.ts index 70ed1de192..964c6283af 100644 --- a/src/util/hash.ts +++ b/src/util/hash.ts @@ -143,3 +143,56 @@ export function murmurHash3_x86_128Hash64Bits( out.high = h2; return out; } + +export function murmurHash3_x86_128Hash64Bits_Bigint( + seed: number, + input: bigint, +): bigint { + let h1 = seed; + let h2 = seed; + let h3 = seed; + let h4 = seed; + const c1 = 0x239b961b; + const c2 = 0xab0e9789; + const c3 = 0x38b34ae5; + // const c4 = 0xa1e38b93; + + let k2 = Math.imul(Number(input >> BigInt(32)), c2); + k2 = rotl32(k2, 16); + k2 = Math.imul(k2, c3); + h2 ^= k2; + + let k1 = Math.imul(Number(input & BigInt(0xffffffff)), c1); + k1 = rotl32(k1, 15); + k1 = Math.imul(k1, c2); + h1 ^= k1; + + const len = 8; + + h1 ^= len; + h2 ^= len; + h3 ^= len; + h4 ^= len; + + h1 = (h1 + h2) >>> 0; + h1 = (h1 + h3) >>> 0; + h1 = (h1 + h4) >>> 0; + h2 = (h2 + h1) >>> 0; + h3 = (h3 + h1) >>> 0; + h4 = (h4 + h1) >>> 0; + + h1 = murmurHash3_x86_128Mix(h1); + h2 = murmurHash3_x86_128Mix(h2); + h3 = murmurHash3_x86_128Mix(h3); + h4 = murmurHash3_x86_128Mix(h4); + + h1 = (h1 + h2) >>> 0; + h1 = (h1 + h3) >>> 0; + h1 = (h1 + h4) >>> 0; + h2 = (h2 + h1) >>> 0; + + // h3 = (h3 + h1) >>> 0; + // h4 = (h4 + h1) >>> 0; + + return BigInt(h1) | (BigInt(h2) << BigInt(32)); +} diff --git a/src/util/http_path_completion.ts b/src/util/http_path_completion.ts deleted file mode 100644 index 1aadcb2138..0000000000 --- a/src/util/http_path_completion.ts +++ /dev/null @@ -1,195 +0,0 @@ -/** - * @license - * Copyright 2019 Google Inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import type { CredentialsManager } from "#src/credentials_provider/index.js"; -import type { - BasicCompletionResult, - Completion, - CompletionWithDescription, -} from "#src/util/completion.js"; -import { getPrefixMatchesWithDescriptions } from "#src/util/completion.js"; -import { getGcsPathCompletions } from "#src/util/gcs_bucket_listing.js"; -import { parseUrl } from "#src/util/http_request.js"; -import { getS3PathCompletions } from "#src/util/s3.js"; -import { getS3CompatiblePathCompletions } from "#src/util/s3_bucket_listing.js"; -import type { SpecialProtocolCredentialsProvider } from "#src/util/special_protocol_request.js"; -import { - fetchSpecialOk, - parseSpecialUrl, -} from "#src/util/special_protocol_request.js"; - -/** - * Obtains a directory listing from a server that supports HTML directory listings. - */ -export async function getHtmlDirectoryListing( - url: string, - abortSignal: AbortSignal, - credentialsProvider?: SpecialProtocolCredentialsProvider, -): Promise { - const response = await fetchSpecialOk( - credentialsProvider, - url, - /*init=*/ { headers: { accept: "text/html" }, signal: abortSignal }, - ); - const contentType = response.headers.get("content-type"); - if (contentType === null || /\btext\/html\b/i.exec(contentType) === null) { - return []; - } - const text = await response.text(); - const doc = new DOMParser().parseFromString(text, "text/html"); - const nodes = doc.evaluate( - "//a/@href", - doc, - null, - XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE, - null, - ); - const results: string[] = []; - for (let i = 0, n = nodes.snapshotLength; i < n; ++i) { - const node = nodes.snapshotItem(i)!; - const href = node.textContent; - if (href) { - results.push(new URL(href, url).toString()); - } - } - return results; -} - -export async function getHtmlPathCompletions( - url: string, - abortSignal: AbortSignal, - credentialsProvider?: SpecialProtocolCredentialsProvider, -): Promise { - console.log("getHtmlPathCompletions"); - const m = url.match(/^([a-z]+:\/\/.*\/)([^/?#]*)$/); - if (m === null) throw null; - const entries = await getHtmlDirectoryListing( - m[1], - abortSignal, - credentialsProvider, - ); - const offset = m[1].length; - const matches: Completion[] = []; - for (const entry of entries) { - if (!entry.startsWith(url)) continue; - matches.push({ value: entry.substring(offset) }); - } - return { - offset, - completions: matches, - }; -} - -const specialProtocolEmptyCompletions: CompletionWithDescription[] = [ - { value: "gs://", description: "Google Cloud Storage (JSON API)" }, - { value: "gs+xml://", description: "Google Cloud Storage (XML API)" }, - { - value: "gs+ngauth+http://", - description: "Google Cloud Storage (JSON API) authenticated via ngauth", - }, - { - value: "gs+ngauth+https://", - description: "Google Cloud Storage (JSON API) authenticated via ngauth", - }, - { - value: "gs+xml+ngauth+http://", - description: "Google Cloud Storage (XML API) authenticated via ngauth", - }, - { - value: "gs+xml+ngauth+https://", - description: "Google Cloud Storage (XML API) authenticated via ngauth", - }, - { value: "s3://", description: "Amazon Simple Storage Service (S3)" }, - { value: "https://" }, - { value: "http://" }, -]; - -export async function completeHttpPath( - credentialsManager: CredentialsManager, - url: string, - abortSignal: AbortSignal, -): Promise> { - if (!url.includes("://")) { - return { - offset: 0, - completions: getPrefixMatchesWithDescriptions( - url, - specialProtocolEmptyCompletions, - (x) => x.value, - (x) => x.description, - ), - }; - } - const { url: parsedUrl, credentialsProvider } = parseSpecialUrl( - url, - credentialsManager, - ); - const offset = url.length - parsedUrl.length; - let result; - try { - result = parseUrl(parsedUrl); - } catch { - throw null; - } - const { protocol, host, path } = result; - const completions = await (async () => { - if (protocol === "gs+xml" && path.length > 0) { - return await getS3CompatiblePathCompletions( - credentialsProvider, - `${protocol}://${host}`, - `https://storage.googleapis.com/${host}`, - path, - abortSignal, - ); - } - if (protocol === "gs" && path.length > 0) { - return await getGcsPathCompletions( - credentialsProvider, - `${protocol}://${host}`, - host, - path, - abortSignal, - ); - } - if (protocol === "s3" && path.length > 0) { - return await getS3PathCompletions(host, path, abortSignal); - } - const s3Match = parsedUrl.match( - /^((?:http|https):\/\/(?:storage\.googleapis\.com\/[^/]+|[^/]+\.storage\.googleapis\.com|[^/]+\.s3(?:[^./]+)?\.amazonaws.com))(\/.*)$/, - ); - if (s3Match !== null) { - return await getS3CompatiblePathCompletions( - credentialsProvider, - s3Match[1], - s3Match[1], - s3Match[2], - abortSignal, - ); - } - if ((protocol === "http" || protocol === "https") && path.length > 0) { - return await getHtmlPathCompletions( - parsedUrl, - abortSignal, - credentialsProvider, - ); - } - throw null; - })(); - return { - offset: offset + completions.offset, - completions: completions.completions, - }; -} diff --git a/src/util/http_request.ts b/src/util/http_request.ts index 330757fbe0..293cc68119 100644 --- a/src/util/http_request.ts +++ b/src/util/http_request.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Uint64 } from "#src/util/uint64.js"; +import type { ProgressListener } from "#src/util/progress_listener.js"; export class HttpError extends Error { url: string; @@ -27,6 +27,7 @@ export class HttpError extends Error { status: number, statusText: string, response?: Response, + options?: { cause: any }, ) { let message = `Fetching ${JSON.stringify( url, @@ -35,7 +36,7 @@ export class HttpError extends Error { message += `: ${statusText}`; } message += "."; - super(message); + super(message, options); this.name = "HttpError"; this.message = message; this.url = url; @@ -63,7 +64,9 @@ export class HttpError extends Error { } else { url = input.url; } - return new HttpError(url, 0, "Network or CORS error"); + return new HttpError(url, 0, "Network or CORS error", undefined, { + cause: error, + }); } return error; } @@ -97,7 +100,7 @@ export function pickDelay(attemptNumber: number): number { */ export async function fetchOk( input: RequestInfo, - init?: RequestInit, + init?: RequestInitWithProgress, ): Promise { for (let requestAttempt = 0; ; ) { init?.signal?.throwIfAborted(); @@ -126,34 +129,14 @@ export async function fetchOk( } } -const tempUint64 = new Uint64(); - -export function getByteRangeHeader( - startOffset: Uint64 | number, - endOffset: Uint64 | number, -) { - let endOffsetStr: string; - if (typeof endOffset === "number") { - endOffsetStr = `${endOffset - 1}`; - } else { - Uint64.decrement(tempUint64, endOffset); - endOffsetStr = tempUint64.toString(); - } - return { Range: `bytes=${startOffset}-${endOffsetStr}` }; +export interface RequestInitWithProgress extends RequestInit { + progressListener?: ProgressListener; } -export function parseUrl(url: string): { - protocol: string; - host: string; - path: string; -} { - const urlProtocolPattern = /^([^:/]+):\/\/([^/]+)((?:\/.*)?)$/; - const match = url.match(urlProtocolPattern); - if (match === null) { - throw new Error(`Invalid URL: ${JSON.stringify(url)}`); - } - return { protocol: match[1], host: match[2], path: match[3] }; -} +export type FetchOk = ( + input: RequestInfo, + init?: RequestInitWithProgress, +) => Promise; export function isNotFoundError(e: any) { if (!(e instanceof HttpError)) return false; diff --git a/src/util/memoize.ts b/src/util/memoize.ts index c9da1cfbf4..fd2826b78e 100644 --- a/src/util/memoize.ts +++ b/src/util/memoize.ts @@ -14,9 +14,12 @@ * limitations under the License. */ +import { raceWithAbort, SharedAbortController } from "#src/util/abort.js"; import type { RefCounted } from "#src/util/disposable.js"; import { RefCountedValue } from "#src/util/disposable.js"; import { stableStringify } from "#src/util/json.js"; +import type { ProgressOptions } from "#src/util/progress_listener.js"; +import { MultiConsumerProgressListener } from "#src/util/progress_listener.js"; export class Memoize { private map = new Map(); @@ -51,4 +54,107 @@ export class StringMemoize extends Memoize { getUncounted(x: any, getter: () => T) { return this.get(x, () => new RefCountedValue(getter())).value; } + + getAsync( + x: any, + options: Partial, + getter: (options: ProgressOptions) => Promise, + ) { + return this.getUncounted(x, () => asyncMemoizeWithProgress(getter))( + options, + ); + } +} + +export interface AsyncMemoize { + (options: { signal?: AbortSignal }): Promise; +} + +export interface AsyncMemoizeWithProgress { + (options: Partial): Promise; +} + +export function asyncMemoize( + getter: (options: { signal: AbortSignal }) => Promise, +): AsyncMemoize { + let abortController: SharedAbortController | undefined; + let promise: Promise | undefined; + let completed: boolean = false; + + return (options: { signal?: AbortSignal }): Promise => { + if (completed) { + return promise!; + } + if (promise === undefined || abortController!.signal.aborted) { + abortController = new SharedAbortController(); + promise = (async () => { + try { + return await getter({ + signal: abortController!.signal, + }); + } catch (e) { + if (abortController!.signal.aborted) { + promise = undefined; + } + throw e; + } finally { + if (promise !== undefined) { + completed = true; + } + abortController![Symbol.dispose](); + abortController = undefined; + } + })(); + } + abortController!.addConsumer(options.signal); + return raceWithAbort(promise, options.signal); + }; +} + +export function asyncMemoizeWithProgress( + getter: (options: ProgressOptions) => Promise, +): AsyncMemoizeWithProgress { + let progressListener: MultiConsumerProgressListener | undefined; + let abortController: SharedAbortController | undefined; + let promise: Promise | undefined; + let completed: boolean = false; + + return async (options: Partial): Promise => { + if (completed) { + return promise!; + } + if (promise === undefined || abortController!.signal.aborted) { + progressListener = new MultiConsumerProgressListener(); + abortController = new SharedAbortController(); + promise = (async () => { + try { + return await getter({ + signal: abortController!.signal, + progressListener: progressListener!, + }); + } catch (e) { + if (abortController!.signal.aborted) { + promise = undefined; + } + throw e; + } finally { + if (promise !== undefined) { + completed = true; + } + progressListener = undefined; + abortController![Symbol.dispose](); + abortController = undefined; + } + })(); + } + abortController!.addConsumer(options.signal); + const curProgressListener = progressListener!; + curProgressListener.addListener(options.progressListener); + + try { + return await raceWithAbort(promise, options.signal); + } finally { + curProgressListener.removeListener(options.progressListener); + } + }; } diff --git a/src/util/npy.spec.ts b/src/util/npy.spec.ts index f5c55557cd..cc61084bf2 100644 --- a/src/util/npy.spec.ts +++ b/src/util/npy.spec.ts @@ -48,6 +48,8 @@ describe("parseNpy", () => { "..", "..", "testdata", + "codec", + "npy", ); const example = JSON.parse( await fs.readFile(`${testDataDir}/npy_test.${json}.json`, { diff --git a/src/util/progress_listener.ts b/src/util/progress_listener.ts new file mode 100644 index 0000000000..6c95858faf --- /dev/null +++ b/src/util/progress_listener.ts @@ -0,0 +1,177 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type ProgressSpanId = number; + +export interface ProgressSpanOptions { + message: string; + startTime?: number; + id?: ProgressSpanId; +} + +export class ProgressSpan implements Disposable { + id: ProgressSpanId; + startTime: number; + message: string; + + constructor( + public listener: ProgressListener, + options: ProgressSpanOptions, + ) { + const { id = Math.random(), startTime = Date.now(), message } = options; + this.id = id; + this.startTime = startTime; + this.message = message; + listener.addSpan(this); + } + + [Symbol.dispose]() { + this.listener.removeSpan(this.id); + } +} + +export interface ProgressListener { + addSpan(span: ProgressSpan): void; + removeSpan(spanId: ProgressSpanId): void; +} + +export class MultiSet { + private items = new Map(); + add(item: T): number { + const { items } = this; + const count = (items.get(item) ?? 0) + 1; + items.set(item, count); + return count; + } + + delete(item: T): number { + const { items } = this; + let count = items.get(item)!; + if (count > 1) { + count -= 1; + items.set(item, count); + return count; + } + items.delete(item); + return 0; + } + + has(item: T): boolean { + return this.items.has(item); + } + + keys() { + return this.items.keys(); + } + + entries() { + return this.items.entries(); + } + + [Symbol.iterator]() { + return this.items.keys(); + } +} + +export class KeyedMultiSet { + private items = new Map(); + constructor(private getKey: (value: T) => Key) {} + + add(item: T): number { + const { items } = this; + const key = this.getKey(item); + const obj = items.get(key); + if (obj === undefined) { + items.set(key, { value: item, count: 1 }); + return 1; + } else { + return (obj.count += 1); + } + } + + delete(item: T): number { + return this.deleteKey(this.getKey(item)); + } + + deleteKey(key: Key): number { + const { items } = this; + const obj = items.get(key); + if (obj !== undefined && obj.count > 1) { + return (obj.count -= 1); + } + items.delete(key); + return 0; + } + + has(item: T): boolean { + return this.items.has(this.getKey(item)); + } + + *[Symbol.iterator]() { + for (const obj of this.items.values()) { + yield obj.value; + } + } +} + +function getId(span: ProgressSpan) { + return span.id; +} + +export class ProgressSpanSet extends KeyedMultiSet { + constructor() { + super(getId); + } +} + +export class MultiConsumerProgressListener implements ProgressListener { + private spans = new ProgressSpanSet(); + private listeners = new MultiSet(); + addSpan(span: ProgressSpan) { + if (this.spans.add(span) !== 1) return; + for (const listener of this.listeners) { + listener.addSpan(span); + } + } + + removeSpan(spanId: ProgressSpanId) { + if (this.spans.deleteKey(spanId) !== 0) return; + for (const listener of this.listeners) { + listener.removeSpan(spanId); + } + } + + addListener(listener: ProgressListener | undefined) { + if (listener === undefined) return; + if (this.listeners.add(listener) !== 1) return; + for (const span of this.spans) { + listener.addSpan(span); + } + } + + removeListener(listener: ProgressListener | undefined) { + if (listener === undefined) return; + if (this.listeners.delete(listener) !== 0) return; + for (const span of this.spans) { + listener.removeSpan(span.id); + } + } +} + +export interface ProgressOptions { + signal: AbortSignal; + progressListener: ProgressListener; +} diff --git a/src/util/s3.ts b/src/util/s3.ts deleted file mode 100644 index 4088f280a9..0000000000 --- a/src/util/s3.ts +++ /dev/null @@ -1,42 +0,0 @@ -/** - * @license - * Copyright 2021 Google Inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { fetchOk } from "#src/util/http_request.js"; -import { getS3CompatiblePathCompletions } from "#src/util/s3_bucket_listing.js"; - -// Support for s3:// special protocol. - -export function fetchS3Ok( - bucket: string, - path: string, - requestInit: RequestInit, -) { - return fetchOk(`https://${bucket}.s3.amazonaws.com${path}`, requestInit); -} - -export async function getS3PathCompletions( - bucket: string, - path: string, - abortSignal: AbortSignal, -) { - return await getS3CompatiblePathCompletions( - undefined, - `s3://${bucket}`, - `https://${bucket}.s3.amazonaws.com`, - path, - abortSignal, - ); -} diff --git a/src/util/s3_bucket_listing.ts b/src/util/s3_bucket_listing.ts deleted file mode 100644 index a93ed197ef..0000000000 --- a/src/util/s3_bucket_listing.ts +++ /dev/null @@ -1,88 +0,0 @@ -/** - * @license - * Copyright 2019 Google Inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @file Provides file listing and completion for storage systems supporting the S3 XML API (e.g. S3 - * and GCS). - */ - -import { fetchWithOAuth2Credentials } from "#src/credentials_provider/oauth2.js"; -import type { BasicCompletionResult } from "#src/util/completion.js"; -import type { SpecialProtocolCredentialsProvider } from "#src/util/special_protocol_request.js"; - -export async function getS3BucketListing( - credentialsProvider: SpecialProtocolCredentialsProvider, - bucketUrl: string, - prefix: string, - delimiter: string, - abortSignal: AbortSignal, -): Promise { - const response = await fetchWithOAuth2Credentials( - credentialsProvider, - `${bucketUrl}?prefix=${encodeURIComponent(prefix)}` + - `&delimiter=${encodeURIComponent(delimiter)}`, - /*init=*/ { signal: abortSignal }, - ); - const doc = new DOMParser().parseFromString( - await response.text(), - "application/xml", - ); - const commonPrefixNodes = doc.evaluate( - '//*[name()="CommonPrefixes"]/*[name()="Prefix"]', - doc, - null, - XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE, - null, - ); - const results: string[] = []; - for (let i = 0, n = commonPrefixNodes.snapshotLength; i < n; ++i) { - results.push(commonPrefixNodes.snapshotItem(i)!.textContent || ""); - } - const contents = doc.evaluate( - '//*[name()="Contents"]/*[name()="Key"]', - doc, - null, - XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE, - null, - ); - for (let i = 0, n = contents.snapshotLength; i < n; ++i) { - results.push(contents.snapshotItem(i)!.textContent || ""); - } - return results; -} - -export async function getS3CompatiblePathCompletions( - credentialsProvider: SpecialProtocolCredentialsProvider, - enteredBucketUrl: string, - bucketUrl: string, - path: string, - abortSignal: AbortSignal, -): Promise { - const prefix = path; - if (!prefix.startsWith("/")) throw null; - const paths = await getS3BucketListing( - credentialsProvider, - bucketUrl, - path.substring(1), - "/", - abortSignal, - ); - const offset = path.lastIndexOf("/"); - return { - offset: offset + enteredBucketUrl.length + 1, - completions: paths.map((x) => ({ value: x.substring(offset) })), - }; -} diff --git a/src/util/special_protocol_request.ts b/src/util/special_protocol_request.ts deleted file mode 100644 index da6297d2ed..0000000000 --- a/src/util/special_protocol_request.ts +++ /dev/null @@ -1,176 +0,0 @@ -/** - * @license - * Copyright 2020 Google Inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import pythonIntegration from "#python_integration_build"; -import type { - CredentialsManager, - MaybeOptionalCredentialsProvider, -} from "#src/credentials_provider/index.js"; -import { fetchWithOAuth2Credentials } from "#src/credentials_provider/oauth2.js"; -import { parseUrl } from "#src/util/http_request.js"; -import { getRandomHexString } from "#src/util/random.js"; -import { fetchS3Ok } from "#src/util/s3.js"; - -export type SpecialProtocolCredentials = any; -export type SpecialProtocolCredentialsProvider = - MaybeOptionalCredentialsProvider; - -function getMiddleAuthCredentialsProvider( - credentialsManager: CredentialsManager, - url: string, -): SpecialProtocolCredentialsProvider { - return credentialsManager.getCredentialsProvider( - "middleauthapp", - new URL(url).origin, - ); -} - -function getNgauthCredentialsProvider( - credentialsManager: CredentialsManager, - serverUrl: string, - path: string, -): SpecialProtocolCredentialsProvider { - const bucketPattern = /^\/([^/]+)/; - const m = path.match(bucketPattern); - if (m === null) return undefined; - return pythonIntegration - ? credentialsManager.getCredentialsProvider("gcs", { bucket: m[1] }) - : credentialsManager.getCredentialsProvider("ngauth_gcs", { - authServer: serverUrl, - bucket: m[1], - }); -} - -export function parseSpecialUrl( - url: string, - credentialsManager: CredentialsManager, -): { url: string; credentialsProvider: SpecialProtocolCredentialsProvider } { - const u = parseUrl(url); - switch (u.protocol) { - case "gs": - case "gs+xml": - return { - credentialsProvider: pythonIntegration - ? credentialsManager.getCredentialsProvider("gcs", { - bucket: u.host, - }) - : undefined, - url, - }; - case "gs+ngauth+http": - return { - credentialsProvider: getNgauthCredentialsProvider( - credentialsManager, - `http://${u.host}`, - u.path, - ), - url: "gs:/" + u.path, - }; - case "gs+ngauth+https": - return { - credentialsProvider: getNgauthCredentialsProvider( - credentialsManager, - `https://${u.host}`, - u.path, - ), - url: "gs:/" + u.path, - }; - case "gs+xml+ngauth+http": - return { - credentialsProvider: getNgauthCredentialsProvider( - credentialsManager, - `http://${u.host}`, - u.path, - ), - url: "gs+xml:/" + u.path, - }; - case "gs+xml+ngauth+https": - return { - credentialsProvider: getNgauthCredentialsProvider( - credentialsManager, - `https://${u.host}`, - u.path, - ), - url: "gs+xml:/" + u.path, - }; - case "middleauth+https": - url = url.substr("middleauth+".length); - return { - credentialsProvider: getMiddleAuthCredentialsProvider( - credentialsManager, - url, - ), - url: url, - }; - case "s3": - return { - credentialsProvider: undefined, - url, - }; - default: - return { - credentialsProvider: undefined, - url, - }; - } -} - -export async function fetchSpecialOk( - credentialsProvider: SpecialProtocolCredentialsProvider, - url: string, - init: RequestInit, -): Promise { - const u = parseUrl(url); - switch (u.protocol) { - case "gs": - // Include random query string parameter (ignored by GCS) to bypass GCS cache and ensure a - // cached response is never used. - // - // This addresses two issues related to GCS: - // - // 1. GCS fails to send an updated `Access-Control-Allow-Origin` header in 304 responses to - // cache revalidation requests. - // - // https://bugs.chromium.org/p/chromium/issues/detail?id=1214563#c2 - // - // The random query string parameter ensures cached responses are never used. - // - // Note: This issue does not apply to gs+xml because with the XML API, the - // Access-Control-Allow-Origin response header does not vary with the Origin. - // - // 2. If the object does not prohibit caching (e.g. public bucket and default `cache-control` - // metadata value), GCS may return stale responses. - // - return fetchWithOAuth2Credentials( - credentialsProvider, - `https://www.googleapis.com/storage/v1/b/${u.host}/o/` + - `${encodeURIComponent(u.path.substring(1))}?alt=media` + - `&neuroglancer=${getRandomHexString()}`, - init, - ); - case "gs+xml": - return fetchWithOAuth2Credentials( - credentialsProvider, - `https://storage.googleapis.com/${u.host}${u.path}` + - `?neuroglancer=${getRandomHexString()}`, - init, - ); - case "s3": - return fetchS3Ok(u.host, u.path, init); - default: - return fetchWithOAuth2Credentials(credentialsProvider, url, init); - } -} diff --git a/src/util/uint64.ts b/src/util/uint64.ts index 7a05caddc3..fd9fb355f1 100644 --- a/src/util/uint64.ts +++ b/src/util/uint64.ts @@ -338,6 +338,10 @@ export class Uint64 { return this.low + this.high * 0x100000000; } + toBigInt() { + return BigInt(this.low) | (BigInt(this.high) << BigInt(32)); + } + setFromNumber(value: number) { value = Math.round(value); if (value < 0) { diff --git a/src/util/zorder.spec.ts b/src/util/zorder.spec.ts index 16486518c5..c5d5b810b1 100644 --- a/src/util/zorder.spec.ts +++ b/src/util/zorder.spec.ts @@ -15,14 +15,13 @@ */ import { describe, it, expect } from "vitest"; -import { Uint64 } from "#src/util/uint64.js"; import { decodeZIndexCompressed, zorder3LessThan } from "#src/util/zorder.js"; describe("decodeZIndexCompressed", () => { it("works for repetitive pattern 21,21,21", () => { expect( decodeZIndexCompressed( - Uint64.parseString("111000100010001111000100010001111000100010001", 2), + 0b111000100010001111000100010001111000100010001n, 21, 21, 21, @@ -35,10 +34,7 @@ describe("decodeZIndexCompressed", () => { it("works for repetitive pattern 18,15,17", () => { expect( decodeZIndexCompressed( - Uint64.parseString( - "11101111000100010001111000100010001111000100010001", - 2, - ), + 0b11101111000100010001111000100010001111000100010001n, 18, 15, 17, diff --git a/src/util/zorder.ts b/src/util/zorder.ts index 7f8bb9dd5a..323646305b 100644 --- a/src/util/zorder.ts +++ b/src/util/zorder.ts @@ -15,7 +15,6 @@ */ import type { TypedArray } from "#src/util/array.js"; -import type { Uint64 } from "#src/util/uint64.js"; export function getOctreeChildIndex(x: number, y: number, z: number) { return (x & 1) | ((y << 1) & 2) | ((z << 2) & 4); @@ -29,73 +28,46 @@ export function getOctreeChildIndex(x: number, y: number, z: number) { * respectively, for `i` in `[0, xBits)`, `[0, yBits)`, `[0, zBits)`, respectively. */ export function decodeZIndexCompressed( - zindex: Uint64, + zindex: bigint, xBits: number, yBits: number, zBits: number, ): Uint32Array { const maxCoordBits = Math.max(xBits, yBits, zBits); let inputBit = 0; - let inputValue = zindex.low; let x = 0; let y = 0; let z = 0; for (let coordBit = 0; coordBit < maxCoordBits; ++coordBit) { if (coordBit < xBits) { - const bit = (inputValue >>> inputBit) & 1; + const bit = Number((zindex >> BigInt(inputBit++)) & BigInt(1)); x |= bit << coordBit; - if (inputBit === 31) { - inputBit = 0; - inputValue = zindex.high; - } else { - ++inputBit; - } } if (coordBit < yBits) { - const bit = (inputValue >>> inputBit) & 1; + const bit = Number((zindex >> BigInt(inputBit++)) & BigInt(1)); y |= bit << coordBit; - if (inputBit === 31) { - inputBit = 0; - inputValue = zindex.high; - } else { - ++inputBit; - } } if (coordBit < zBits) { - const bit = (inputValue >>> inputBit) & 1; + const bit = Number((zindex >> BigInt(inputBit++)) & BigInt(1)); z |= bit << coordBit; - if (inputBit === 31) { - inputBit = 0; - inputValue = zindex.high; - } else { - ++inputBit; - } } } return Uint32Array.of(x, y, z); } export function encodeZIndexCompressed3d( - zindex: Uint64, xBits: number, yBits: number, zBits: number, x: number, y: number, z: number, -): Uint64 { +): bigint { const maxBits = Math.max(xBits, yBits, zBits); let outputBit = 0; - let outputNum = 0; - let isHigh = false; + let zIndex = 0n; function writeBit(b: number): void { - outputNum |= (b & 1) << outputBit; - if (++outputBit === 32) { - zindex.low = outputNum >>> 0; - outputNum = 0; - outputBit = 0; - isHigh = true; - } + zIndex |= BigInt(b) << BigInt(outputBit++); } for (let bit = 0; bit < maxBits; ++bit) { if (bit < xBits) { @@ -108,32 +80,18 @@ export function encodeZIndexCompressed3d( writeBit((z >> bit) & 1); } } - if (isHigh) { - zindex.high = outputNum >>> 0; - } else { - zindex.high = 0; - zindex.low = outputNum >>> 0; - } - return zindex; + return zIndex; } export function encodeZIndexCompressed( - zindex: Uint64, position: TypedArray, shape: TypedArray, -): Uint64 { +): bigint { + let zIndex = 0n; let outputBit = 0; const rank = position.length; - let outputNum = 0; - let isHigh = false; function writeBit(b: number): void { - outputNum |= (b & 1) << outputBit; - if (++outputBit === 32) { - zindex.low = outputNum >>> 0; - outputNum = 0; - outputBit = 0; - isHigh = true; - } + zIndex |= BigInt(b & 1) << BigInt(outputBit++); } for (let bit = 0; bit < 32; ++bit) { @@ -143,13 +101,7 @@ export function encodeZIndexCompressed( } } } - if (isHigh) { - zindex.high = outputNum >>> 0; - } else { - zindex.high = 0; - zindex.low = outputNum >>> 0; - } - return zindex; + return zIndex; } function lessMsb(a: number, b: number) { diff --git a/src/viewer.ts b/src/viewer.ts index 3a6ebf1f8d..e921402388 100644 --- a/src/viewer.ts +++ b/src/viewer.ts @@ -22,20 +22,17 @@ import svg_layers from "ikonate/icons/layers.svg?raw"; import svg_list from "ikonate/icons/list.svg?raw"; import svg_settings from "ikonate/icons/settings.svg?raw"; import { debounce } from "lodash-es"; -import type { FrameNumberCounter } from "#src/chunk_manager/frontend.js"; -import { - CapacitySpecification, - ChunkManager, - ChunkQueueManager, -} from "#src/chunk_manager/frontend.js"; import { makeCoordinateSpace, TrackableCoordinateSpace, } from "#src/coordinate_transform.js"; -import { defaultCredentialsManager } from "#src/credentials_provider/default_manager.js"; +import { getDefaultCredentialsManager } from "#src/credentials_provider/default_manager.js"; +import type { CredentialsManager } from "#src/credentials_provider/index.js"; +import { SharedCredentialsManager } from "#src/credentials_provider/shared.js"; +import { DataManagementContext } from "#src/data_management_context.js"; import { InputEventBindings as DataPanelInputEventBindings } from "#src/data_panel_layout.js"; import { getDefaultDataSourceProvider } from "#src/datasource/default_provider.js"; -import type { DataSourceProviderRegistry } from "#src/datasource/index.js"; +import type { DataSourceRegistry } from "#src/datasource/index.js"; import { StateShare, stateShareEnabled } from "#src/datasource/state_share.js"; import type { DisplayContext } from "#src/display_context.js"; import { TrackableWindowedViewport } from "#src/display_context.js"; @@ -43,6 +40,7 @@ import { HelpPanelState, InputEventBindingHelpDialog, } from "#src/help/input_event_bindings.js"; +import { SharedKvStoreContext } from "#src/kvstore/frontend.js"; import { addNewLayer, LayerManager, @@ -132,7 +130,6 @@ import type { VisibilityPrioritySpecification, } from "#src/viewer_state.js"; import { WatchableVisibilityPriority } from "#src/visibility_priority/frontend.js"; -import type { GL } from "#src/webgl/context.js"; import { AnnotationToolStatusWidget } from "#src/widget/annotation_tool_status.js"; import { CheckboxIcon } from "#src/widget/checkbox_icon.js"; import { makeIcon } from "#src/widget/icon.js"; @@ -144,7 +141,6 @@ import { registerDimensionToolForViewer, } from "#src/widget/position_widget.js"; import { TrackableScaleBarOptions } from "#src/widget/scale_bar.js"; -import { RPC } from "#src/worker_rpc.js"; declare let NEUROGLANCER_OVERRIDE_DEFAULT_VIEWER_OPTIONS: any; @@ -155,60 +151,6 @@ interface CreditLink { declare let NEUROGLANCER_CREDIT_LINK: CreditLink | CreditLink[] | undefined; -export class DataManagementContext extends RefCounted { - worker: Worker; - chunkQueueManager: ChunkQueueManager; - chunkManager: ChunkManager; - - get rpc(): RPC { - return this.chunkQueueManager.rpc!; - } - - constructor( - public gl: GL, - public frameNumberCounter: FrameNumberCounter, - ) { - super(); - // Note: For compatibility with multiple bundlers, a browser-compatible URL - // must be used with `new URL`, which means a Node.js subpath import like - // "#src/chunk_worker.bundle.js" cannot be used. - this.worker = new Worker( - /* webpackChunkName: "neuroglancer_chunk_worker" */ - new URL("./chunk_worker.bundle.js", import.meta.url), - { type: "module" }, - ); - this.chunkQueueManager = this.registerDisposer( - new ChunkQueueManager( - new RPC(this.worker, /*waitUntilReady=*/ true), - this.gl, - this.frameNumberCounter, - { - gpuMemory: new CapacitySpecification({ - defaultItemLimit: 1e6, - defaultSizeLimit: 1e9, - }), - systemMemory: new CapacitySpecification({ - defaultItemLimit: 1e7, - defaultSizeLimit: 2e9, - }), - download: new CapacitySpecification({ - defaultItemLimit: 100, - defaultSizeLimit: Number.POSITIVE_INFINITY, - }), - compute: new CapacitySpecification({ - defaultItemLimit: 128, - defaultSizeLimit: 5e8, - }), - }, - ), - ); - this.chunkQueueManager.registerDisposer(() => this.worker.terminate()); - this.chunkManager = this.registerDisposer( - new ChunkManager(this.chunkQueueManager), - ); - } -} - export class InputEventBindings extends DataPanelInputEventBindings { global = new EventActionMap(); } @@ -268,7 +210,8 @@ export interface ViewerOptions VisibilityPrioritySpecification { dataContext: Owned; element: HTMLElement; - dataSourceProvider: Borrowed; + credentialsManager: CredentialsManager; + dataSourceProvider: Borrowed; uiConfiguration: ViewerUIConfiguration; showLayerDialog: boolean; inputEventBindings: InputEventBindings; @@ -511,7 +454,7 @@ export class Viewer extends RefCounted implements ViewerState { visibility: WatchableVisibilityPriority; inputEventBindings: InputEventBindings; element: HTMLElement; - dataSourceProvider: Borrowed; + dataSourceProvider: Borrowed; uiConfiguration: ViewerUIConfiguration; @@ -556,14 +499,29 @@ export class Viewer extends RefCounted implements ViewerState { perspectiveView: new EventActionMap(), }, element = display.makeCanvasOverlayElement(), - dataSourceProvider = getDefaultDataSourceProvider({ - credentialsManager: defaultCredentialsManager, - }), + uiConfiguration = makeViewerUIConfiguration(), + dataSourceProvider = (() => { + const { credentialsManager = getDefaultCredentialsManager() } = options; + const sharedCredentialsManager = this.registerDisposer( + new SharedCredentialsManager(credentialsManager, dataContext.rpc), + ); + const kvStoreContext = this.registerDisposer( + new SharedKvStoreContext( + dataContext.chunkManager, + sharedCredentialsManager, + ), + ); + return getDefaultDataSourceProvider({ + credentialsManager: sharedCredentialsManager, + kvStoreContext, + }); + })(), } = options; this.visibility = visibility; this.inputEventBindings = inputEventBindings; this.element = element; + this.dataSourceProvider = dataSourceProvider; this.uiConfiguration = uiConfiguration; diff --git a/src/webgl/context.ts b/src/webgl/context.ts index 5241781a6e..3da15d46e2 100644 --- a/src/webgl/context.ts +++ b/src/webgl/context.ts @@ -41,9 +41,13 @@ export function initializeWebGL(canvas: HTMLCanvasElement) { throw new Error("WebGL not supported."); } gl.memoize = new Memoize(); - gl.maxTextureSize = gl.getParameter(gl.MAX_TEXTURE_SIZE); - gl.max3dTextureSize = gl.getParameter(gl.MAX_3D_TEXTURE_SIZE); - gl.maxTextureImageUnits = gl.getParameter(gl.MAX_TEXTURE_IMAGE_UNITS); + gl.maxTextureSize = gl.getParameter(WebGL2RenderingContext.MAX_TEXTURE_SIZE); + gl.max3dTextureSize = gl.getParameter( + WebGL2RenderingContext.MAX_3D_TEXTURE_SIZE, + ); + gl.maxTextureImageUnits = gl.getParameter( + WebGL2RenderingContext.MAX_TEXTURE_IMAGE_UNITS, + ); gl.tempTextureUnit = gl.maxTextureImageUnits - 1; // FIXME: verify that we received a stencil buffer diff --git a/src/widget/multiline_autocomplete.css b/src/widget/multiline_autocomplete.css index 8ff86179fd..86ad6b91ae 100644 --- a/src/widget/multiline_autocomplete.css +++ b/src/widget/multiline_autocomplete.css @@ -48,8 +48,9 @@ font-size: medium; } -.neuroglancer-multiline-autocomplete-dropdown { - color: #fff; +.neuroglancer-multiline-autocomplete-dropdown, +.neuroglancer-multiline-autocomplete-error, +.neuroglancer-multiline-autocomplete-progress { background-color: #181818; position: fixed; display: block; @@ -67,6 +68,19 @@ word-wrap: break-word; } +.neuroglancer-multiline-autocomplete-dropdown { + color: #fff; +} + +.neuroglancer-multiline-autocomplete-error { + color: red; +} + +.neuroglancer-multiline-autocomplete-progress { + font-style: italic; + color: #ccc; +} + .neuroglancer-multiline-autocomplete-completion:nth-child(even):not( .neuroglancer-multiline-autocomplete-completion-active ) { @@ -88,3 +102,8 @@ font-style: italic; color: #f9f; } + +.neuroglancer-multiline-autocomplete-linebreak::after { + content: "\a"; + white-space: pre; +} diff --git a/src/widget/multiline_autocomplete.ts b/src/widget/multiline_autocomplete.ts index 1b412098eb..e2ce2b5a66 100644 --- a/src/widget/multiline_autocomplete.ts +++ b/src/widget/multiline_autocomplete.ts @@ -25,13 +25,16 @@ import type { import { RefCounted } from "#src/util/disposable.js"; import { removeChildren, removeFromParent } from "#src/util/dom.js"; import { positionDropdown } from "#src/util/dropdown.js"; +import { formatErrorMessage } from "#src/util/error.js"; import { EventActionMap, KeyboardEventBinder, registerActionListener, } from "#src/util/keyboard_bindings.js"; import { longestCommonPrefix } from "#src/util/longest_common_prefix.js"; +import type { ProgressListener } from "#src/util/progress_listener.js"; import { Signal } from "#src/util/signal.js"; +import { ProgressListenerWidget } from "#src/widget/progress_listener.js"; import { VirtualList } from "#src/widget/virtual_list.js"; export type { @@ -56,19 +59,16 @@ export function makeDefaultCompletionElement(completion: Completion) { return element; } -function* splitByWordBreaks(value: string) { - while (value.length > 0) { - const m = value.match(/[:/_]+/); - if (m === null) { - yield value; - return; - } - const endOffset = m.index! + m[0].length; - yield value.substring(0, endOffset); - value = value.substring(endOffset); - } +export interface SyntaxHighlighter { + splitPattern: RegExp; + getSeparatorNode: (text: string) => HTMLElement; } +const DEFAULT_SYNTAX_HIGHLIGHTER: SyntaxHighlighter = { + splitPattern: /.*/gs, + getSeparatorNode: () => document.createElement("wbr"), +}; + export function makeCompletionElementWithDescription( completion: CompletionWithDescription, ) { @@ -100,7 +100,8 @@ export interface CompletionRequest { } export type Completer = ( request: CompletionRequest, - abortSignal: AbortSignal, + signal: AbortSignal, + progressListener: ProgressListener, ) => Promise | null; const DEFAULT_COMPLETION_DELAY = 200; // milliseconds @@ -117,6 +118,9 @@ export class AutocompleteTextInput extends RefCounted { private activeCompletionPromise: Promise | null = null; private activeCompletionAbortController: AbortController | undefined = undefined; + private activeCompletionProgressListener: ProgressListenerWidget | undefined = + undefined; + private completionErrorElement: HTMLElement | undefined; private hasFocus = false; private completionResult: CompletionResult | null = null; private dropdownContentsStale = true; @@ -161,14 +165,15 @@ export class AutocompleteTextInput extends RefCounted { ) { const completionDisabled = this.completionDisabled !== -1; this.onInput.dispatch(value); - const { inputElement } = this; + const { inputElement, syntaxHighlighter } = this; removeChildren(inputElement); let outputOffset = 0; const r = selection !== undefined ? document.createRange() : undefined; let isFirst = true; - for (const text of splitByWordBreaks(value)) { + for (const text of value.match(syntaxHighlighter.splitPattern)!) { + if (!text) continue; if (!isFirst) { - inputElement.appendChild(document.createElement("wbr")); + inputElement.appendChild(syntaxHighlighter.getSeparatorNode(text)); } isFirst = false; const newOutputOffset = outputOffset + text.length; @@ -216,21 +221,31 @@ export class AutocompleteTextInput extends RefCounted { completer: Completer; private resizeHandler = () => { - if (!this.completionsVisible) return; this.updateDropdownStyle(); }; private resizeObserver = new ResizeObserver(this.resizeHandler); - constructor(options: { completer: Completer; delay?: number }) { + private syntaxHighlighter: SyntaxHighlighter; + + constructor(options: { + completer: Completer; + syntaxHighlighter?: SyntaxHighlighter; + delay?: number; + }) { super(); this.completer = options.completer; - const { delay = DEFAULT_COMPLETION_DELAY } = options; + const { + delay = DEFAULT_COMPLETION_DELAY, + syntaxHighlighter = DEFAULT_SYNTAX_HIGHLIGHTER, + } = options; + this.syntaxHighlighter = syntaxHighlighter; const debouncedCompleter = (this.scheduleUpdateCompletions = debounce( () => { const abortController = (this.activeCompletionAbortController = new AbortController()); + const progressListener = this.makeCompletionProgressListener(); const activeCompletionPromise = (this.activeCompletionPromise = this.completer( { @@ -238,14 +253,24 @@ export class AutocompleteTextInput extends RefCounted { selectionRange: this.getSelectionRange(), }, abortController.signal, + progressListener, )); if (activeCompletionPromise !== null) { - activeCompletionPromise.then((completionResult) => { - if (this.activeCompletionPromise === activeCompletionPromise) { - this.setCompletions(completionResult); - this.activeCompletionPromise = null; - } - }); + activeCompletionPromise.then( + (completionResult) => { + if (this.activeCompletionPromise === activeCompletionPromise) { + this.setCompletions(completionResult); + this.activeCompletionPromise = null; + this.removeProgressListener(); + } + }, + (reason) => { + if (this.activeCompletionPromise === activeCompletionPromise) { + this.removeProgressListener(); + this.showCompletionError(reason); + } + }, + ); } }, delay, @@ -263,9 +288,9 @@ export class AutocompleteTextInput extends RefCounted { inputElement.contentEditable = "true"; inputElement.spellcheck = false; - element.appendChild(document.createTextNode("\u200b")); // Prevent input area from collapsing element.appendChild(inputElement); element.appendChild(hintElement); + element.appendChild(document.createTextNode("\u200b")); // Prevent input area from collapsing inputElement.classList.add("neuroglancer-multiline-autocomplete-input"); hintElement.classList.add("neuroglancer-multiline-autocomplete-hint"); inputElement.addEventListener("input", () => { @@ -421,6 +446,34 @@ export class AutocompleteTextInput extends RefCounted { }); } + private removeProgressListener() { + if (this.activeCompletionProgressListener !== undefined) { + this.element.removeChild(this.activeCompletionProgressListener.element); + this.activeCompletionProgressListener = undefined; + } + } + + private makeCompletionProgressListener(): ProgressListenerWidget { + const widget = new ProgressListenerWidget(); + this.element.appendChild(widget.element); + widget.element.classList.add( + "neuroglancer-multiline-autocomplete-progress", + ); + this.activeCompletionProgressListener = widget; + this.updateDropdownStyle(); + return widget; + } + + private showCompletionError(reason: unknown) { + if (reason === null) return; + const element = document.createElement("div"); + this.completionErrorElement = element; + element.classList.add("neuroglancer-multiline-autocomplete-error"); + element.textContent = formatErrorMessage(reason); + this.element.appendChild(element); + this.updateDropdownStyle(); + } + private shouldAttemptCompletion() { const { inputElement } = this; if (document.activeElement !== inputElement) return false; @@ -503,9 +556,12 @@ export class AutocompleteTextInput extends RefCounted { } private updateDropdownStyle() { - const { completionsVirtualList, element } = this; - if (completionsVirtualList !== undefined) { - positionDropdown(completionsVirtualList.element, element, { + const widget = + this.completionsVirtualList?.element ?? + this.activeCompletionProgressListener?.element ?? + this.completionErrorElement; + if (widget !== undefined) { + positionDropdown(widget, this.element, { horizontal: false, }); } @@ -610,13 +666,15 @@ export class AutocompleteTextInput extends RefCounted { } else { this.hasResultForDropdown = true; // Check for a common prefix. - const commonResultPrefix = longestCommonPrefix( - (function* () { - for (const completion of completionResult.completions) { - yield completion.value; - } - })(), - ); + const commonResultPrefix = + completionResult.defaultCompletion ?? + longestCommonPrefix( + (function* () { + for (const completion of completionResult.completions) { + yield completion.value; + } + })(), + ); const commonPrefix = this.getCompletedValue(commonResultPrefix); if (commonPrefix.startsWith(value)) { this.commonPrefix = commonPrefix; @@ -635,12 +693,13 @@ export class AutocompleteTextInput extends RefCounted { hintValue = ""; } hintValue = hintValue.substring(value.length); - const { hintElement } = this; + const { hintElement, syntaxHighlighter } = this; removeChildren(hintElement); let isFirst = true; - for (const text of splitByWordBreaks(hintValue)) { + for (const text of hintValue.match(syntaxHighlighter.splitPattern)!) { + if (!text) continue; if (!isFirst) { - hintElement.appendChild(document.createElement("wbr")); + hintElement.appendChild(syntaxHighlighter.getSeparatorNode(text)); } isFirst = false; const node = document.createTextNode(text); @@ -774,14 +833,15 @@ export class AutocompleteTextInput extends RefCounted { this.activeCompletionAbortController?.abort(); this.activeCompletionAbortController = undefined; this.activeCompletionPromise = null; + this.removeProgressListener(); } private clearCompletions() { + this.dropdownContentsStale = true; + this.dropdownStyleStale = true; if (this.completionResult !== null) { this.activeIndex = -1; this.completionResult = null; - this.dropdownContentsStale = true; - this.dropdownStyleStale = true; this.commonPrefix = ""; const { completionsVirtualList } = this; if (completionsVirtualList !== undefined) { @@ -790,6 +850,12 @@ export class AutocompleteTextInput extends RefCounted { } this.updateDropdown(); } + + const { completionErrorElement } = this; + if (completionErrorElement !== undefined) { + this.element.removeChild(completionErrorElement); + this.completionErrorElement = undefined; + } } get value() { diff --git a/src/widget/progress_listener.ts b/src/widget/progress_listener.ts new file mode 100644 index 0000000000..970f8c91b5 --- /dev/null +++ b/src/widget/progress_listener.ts @@ -0,0 +1,47 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { + ProgressListener, + ProgressSpan, + ProgressSpanId, +} from "#src/util/progress_listener.js"; +import { ProgressSpanSet } from "#src/util/progress_listener.js"; + +export class ProgressListenerWidget implements ProgressListener { + element = document.createElement("ul"); + constructor() { + this.element.classList.add("neuroglancer-progress"); + } + private spanElements = new Map(); + private spans = new ProgressSpanSet(); + + addSpan(span: ProgressSpan) { + if (this.spans.add(span) !== 1) return; + const spanElement = document.createElement("li"); + spanElement.textContent = span.message; + this.spanElements.set(span.id, spanElement); + this.element.appendChild(spanElement); + } + + removeSpan(spanId: ProgressSpanId) { + if (this.spans.deleteKey(spanId) !== 0) return; + const { spanElements } = this; + const spanElement = spanElements.get(spanId)!; + spanElements.delete(spanId); + this.element.removeChild(spanElement); + } +} diff --git a/src/worker_rpc.ts b/src/worker_rpc.ts index 25f5cf3223..d80804a38b 100644 --- a/src/worker_rpc.ts +++ b/src/worker_rpc.ts @@ -16,6 +16,12 @@ import { promiseWithResolversAndAbortCallback } from "#src/util/abort.js"; import { RefCounted } from "#src/util/disposable.js"; +import type { + ProgressListener, + ProgressOptions, + ProgressSpanId, +} from "#src/util/progress_listener.js"; +import { ProgressSpan } from "#src/util/progress_listener.js"; export type RPCHandler = (this: RPC, x: any) => void; @@ -29,6 +35,8 @@ const DEBUG_MESSAGES = false; const PROMISE_RESPONSE_ID = "rpc.promise.response"; const PROMISE_CANCEL_ID = "rpc.promise.cancel"; +const PROMISE_PROGRESS_ADD_SPAN_ID = "rpc.promise.addProgressSpan"; +const PROMISE_PROGRESS_REMOVE_SPAN_ID = "rpc.promise.removeProgressSpan"; const READY_ID = "rpc.ready"; const handlers = new Map(); @@ -39,27 +47,49 @@ export function registerRPC(key: string, handler: RPCHandler) { export type RPCPromise = Promise<{ value: T; transfers?: any[] }>; -export class RPCError extends Error { +class ProxyProgressListener implements ProgressListener { constructor( - public name: string, - public message: string, - ) { - super(message); + private rpc: RPC, + private id: number, + ) {} + + addSpan(span: ProgressSpan) { + this.rpc.invoke(PROMISE_PROGRESS_ADD_SPAN_ID, { + id: this.id, + span: { + id: span.id, + message: span.message, + startTime: span.startTime, + }, + }); + } + removeSpan(spanId: ProgressSpanId) { + this.rpc.invoke(PROMISE_PROGRESS_REMOVE_SPAN_ID, { + id: this.id, + spanId, + }); } } export function registerPromiseRPC( key: string, - handler: (this: RPC, x: any, abortSignal: AbortSignal) => RPCPromise, + handler: ( + this: RPC, + x: any, + progressOptions: Partial, + ) => RPCPromise, ) { registerRPC(key, function (this: RPC, x: any) { const id = x.id; const abortController = new AbortController(); - const promise = handler.call( - this, - x, - abortController.signal, - ) as RPCPromise; + let progressListener: ProgressListener | undefined; + if (x.progressListener === true) { + progressListener = new ProxyProgressListener(this, id); + } + const promise = handler.call(this, x, { + signal: abortController.signal, + progressListener, + }) as RPCPromise; this.set(id, { promise, abortController }); promise.then( ({ value, transfers }) => { @@ -70,8 +100,7 @@ export function registerPromiseRPC( this.delete(id); this.invoke(PROMISE_RESPONSE_ID, { id: id, - error: error.message, - errorName: error.name, + error: error, }); }, ); @@ -94,10 +123,22 @@ registerRPC(PROMISE_RESPONSE_ID, function (this: RPC, x: any) { if (Object.prototype.hasOwnProperty.call(x, "value")) { resolve(x.value); } else { - reject(new RPCError(x.errorName, x.error)); + reject(x.error); } }); +registerRPC(PROMISE_PROGRESS_ADD_SPAN_ID, function (this: RPC, x: any) { + const id = x.id; + const { progressListener } = this.get(id); + new ProgressSpan(progressListener, x.span); +}); + +registerRPC(PROMISE_PROGRESS_REMOVE_SPAN_ID, function (this: RPC, x: any) { + const id = x.id; + const { progressListener } = this.get(id); + progressListener.removeSpan(x.spanId); +}); + registerRPC(READY_ID, function (this: RPC, x: any) { x; this.onPeerReady(); @@ -126,6 +167,10 @@ export class RPC { if (DEBUG_MESSAGES) { console.log("Received message", data); } + const handler = handlers.get(data.functionName); + if (handler === undefined) { + throw new Error(`Missing RPC function: ${data.functionName}`); + } handlers.get(data.functionName)!.call(this, data); }; } @@ -193,21 +238,33 @@ export class RPC { promiseInvoke( name: string, x: any, - abortSignal?: AbortSignal | undefined, - transfers?: any[], + options?: { + signal?: AbortSignal; + progressListener?: ProgressListener; + transfers?: any[]; + }, ): Promise { - if (abortSignal?.aborted) { - return Promise.reject(abortSignal.reason); + let signal: AbortSignal | undefined; + let progressListener: ProgressListener | undefined; + let transfers: any[] | undefined; + if (options !== undefined) { + ({ signal, progressListener, transfers } = options); + } + if (signal?.aborted) { + return Promise.reject(signal.reason); + } + if (progressListener !== undefined) { + x.progressListener = true; } const id = (x.id = this.newId()); this.invoke(name, x, transfers); const { promise, resolve, reject } = - abortSignal === undefined + signal === undefined ? Promise.withResolvers() - : promiseWithResolversAndAbortCallback(abortSignal, () => { + : promiseWithResolversAndAbortCallback(signal, () => { this.invoke(PROMISE_CANCEL_ID, { id: id }); }); - this.set(id, { resolve, reject }); + this.set(id, { resolve, reject, progressListener }); return promise; } diff --git a/testdata/64x64x64-raw-uint64-segmentation.dat b/testdata/codec/compressed_segmentation/64x64x64-raw-uint64-segmentation.dat similarity index 100% rename from testdata/64x64x64-raw-uint64-segmentation.dat rename to testdata/codec/compressed_segmentation/64x64x64-raw-uint64-segmentation.dat diff --git a/testdata/README.md b/testdata/codec/compressed_segmentation/README.md similarity index 100% rename from testdata/README.md rename to testdata/codec/compressed_segmentation/README.md diff --git a/testdata/generate_npy_examples.py b/testdata/codec/npy/generate_npy_examples.py similarity index 100% rename from testdata/generate_npy_examples.py rename to testdata/codec/npy/generate_npy_examples.py diff --git a/testdata/npy_test.float32-be.npy b/testdata/codec/npy/npy_test.float32-be.npy similarity index 100% rename from testdata/npy_test.float32-be.npy rename to testdata/codec/npy/npy_test.float32-be.npy diff --git a/testdata/npy_test.float32-le.npy b/testdata/codec/npy/npy_test.float32-le.npy similarity index 100% rename from testdata/npy_test.float32-le.npy rename to testdata/codec/npy/npy_test.float32-le.npy diff --git a/testdata/npy_test.float32.json b/testdata/codec/npy/npy_test.float32.json similarity index 100% rename from testdata/npy_test.float32.json rename to testdata/codec/npy/npy_test.float32.json diff --git a/testdata/npy_test.uint16-be.npy b/testdata/codec/npy/npy_test.uint16-be.npy similarity index 100% rename from testdata/npy_test.uint16-be.npy rename to testdata/codec/npy/npy_test.uint16-be.npy diff --git a/testdata/npy_test.uint16-le.npy b/testdata/codec/npy/npy_test.uint16-le.npy similarity index 100% rename from testdata/npy_test.uint16-le.npy rename to testdata/codec/npy/npy_test.uint16-le.npy diff --git a/testdata/npy_test.uint16.json b/testdata/codec/npy/npy_test.uint16.json similarity index 100% rename from testdata/npy_test.uint16.json rename to testdata/codec/npy/npy_test.uint16.json diff --git a/testdata/npy_test.uint32-be.npy b/testdata/codec/npy/npy_test.uint32-be.npy similarity index 100% rename from testdata/npy_test.uint32-be.npy rename to testdata/codec/npy/npy_test.uint32-be.npy diff --git a/testdata/npy_test.uint32-le.npy b/testdata/codec/npy/npy_test.uint32-le.npy similarity index 100% rename from testdata/npy_test.uint32-le.npy rename to testdata/codec/npy/npy_test.uint32-le.npy diff --git a/testdata/npy_test.uint32.json b/testdata/codec/npy/npy_test.uint32.json similarity index 100% rename from testdata/npy_test.uint32.json rename to testdata/codec/npy/npy_test.uint32.json diff --git a/testdata/npy_test.uint64-be.npy b/testdata/codec/npy/npy_test.uint64-be.npy similarity index 100% rename from testdata/npy_test.uint64-be.npy rename to testdata/codec/npy/npy_test.uint64-be.npy diff --git a/testdata/npy_test.uint64-le.npy b/testdata/codec/npy/npy_test.uint64-le.npy similarity index 100% rename from testdata/npy_test.uint64-le.npy rename to testdata/codec/npy/npy_test.uint64-le.npy diff --git a/testdata/npy_test.uint64.json b/testdata/codec/npy/npy_test.uint64.json similarity index 100% rename from testdata/npy_test.uint64.json rename to testdata/codec/npy/npy_test.uint64.json diff --git a/testdata/npy_test.uint8.json b/testdata/codec/npy/npy_test.uint8.json similarity index 100% rename from testdata/npy_test.uint8.json rename to testdata/codec/npy/npy_test.uint8.json diff --git a/testdata/npy_test.uint8.npy b/testdata/codec/npy/npy_test.uint8.npy similarity index 100% rename from testdata/npy_test.uint8.npy rename to testdata/codec/npy/npy_test.uint8.npy diff --git a/testdata/datasource/deepzoom/14122_mPPC_BDA_s186.dzi b/testdata/datasource/deepzoom/14122_mPPC_BDA_s186.dzi new file mode 100644 index 0000000000..10bb89f11b --- /dev/null +++ b/testdata/datasource/deepzoom/14122_mPPC_BDA_s186.dzi @@ -0,0 +1,4 @@ + + + + diff --git a/testdata/datasource/n5/generate_n5.py b/testdata/datasource/n5/generate_n5.py new file mode 100755 index 0000000000..7bd8ff79c6 --- /dev/null +++ b/testdata/datasource/n5/generate_n5.py @@ -0,0 +1,69 @@ +#!/usr/bin/env -S uv run +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "tensorstore", +# ] +# /// +import json +import os +import pathlib +import shutil + +import tensorstore as ts + + +def write_multiscale( + path, include_top_level_scales: bool, include_per_scale_downsampling_factors: bool +): + shutil.rmtree(path, ignore_errors=True) + shape = [10, 20] + scales = [] + for scale in [0, 1, 2]: + downsample_factors = [2**scale, 2**scale] + scales.append(downsample_factors) + scale_path = os.path.join(path, f"s{scale}") + scale_attributes = {} + if include_per_scale_downsampling_factors: + scale_attributes["downsamplingFactors"] = downsample_factors + ts.open( + { + "driver": "n5", + "kvstore": {"driver": "file", "path": scale_path}, + "metadata": scale_attributes, + }, + create=True, + delete_existing=True, + dtype=ts.uint16, + shape=[ + -(-shape[0] // downsample_factors[0]), + -(-shape[1] // downsample_factors[1]), + ], + ).result() + top_level_attributes = { + "axes": ["x", "y"], + "units": ["nm", "s"], + "resolution": [10, 20], + } + if include_top_level_scales: + top_level_attributes["scales"] = scales + pathlib.Path(os.path.join(path, "attributes.json")).write_text( + json.dumps(top_level_attributes) + ) + + +write_multiscale( + os.path.abspath( + os.path.join(os.path.dirname(__file__), "n5_viewer_multiscale_deprecated") + ), + include_top_level_scales=True, + include_per_scale_downsampling_factors=False, +) + +write_multiscale( + os.path.abspath( + os.path.join(os.path.dirname(__file__), "n5_viewer_multiscale_modern") + ), + include_top_level_scales=False, + include_per_scale_downsampling_factors=True, +) diff --git a/testdata/datasource/n5/n5_viewer_multiscale_deprecated/attributes.json b/testdata/datasource/n5/n5_viewer_multiscale_deprecated/attributes.json new file mode 100644 index 0000000000..7311cde9a8 --- /dev/null +++ b/testdata/datasource/n5/n5_viewer_multiscale_deprecated/attributes.json @@ -0,0 +1 @@ +{"axes": ["x", "y"], "units": ["nm", "s"], "resolution": [10, 20], "scales": [[1, 1], [2, 2], [4, 4]]} \ No newline at end of file diff --git a/testdata/datasource/n5/n5_viewer_multiscale_deprecated/s0/attributes.json b/testdata/datasource/n5/n5_viewer_multiscale_deprecated/s0/attributes.json new file mode 100644 index 0000000000..7254f6e6fa --- /dev/null +++ b/testdata/datasource/n5/n5_viewer_multiscale_deprecated/s0/attributes.json @@ -0,0 +1 @@ +{"blockSize":[10,20],"compression":{"blocksize":0,"clevel":5,"cname":"lz4","shuffle":1,"type":"blosc"},"dataType":"uint16","dimensions":[10,20]} \ No newline at end of file diff --git a/testdata/datasource/n5/n5_viewer_multiscale_deprecated/s1/attributes.json b/testdata/datasource/n5/n5_viewer_multiscale_deprecated/s1/attributes.json new file mode 100644 index 0000000000..1a935f8806 --- /dev/null +++ b/testdata/datasource/n5/n5_viewer_multiscale_deprecated/s1/attributes.json @@ -0,0 +1 @@ +{"blockSize":[5,10],"compression":{"blocksize":0,"clevel":5,"cname":"lz4","shuffle":1,"type":"blosc"},"dataType":"uint16","dimensions":[5,10]} \ No newline at end of file diff --git a/testdata/datasource/n5/n5_viewer_multiscale_deprecated/s2/attributes.json b/testdata/datasource/n5/n5_viewer_multiscale_deprecated/s2/attributes.json new file mode 100644 index 0000000000..2c2d38eab1 --- /dev/null +++ b/testdata/datasource/n5/n5_viewer_multiscale_deprecated/s2/attributes.json @@ -0,0 +1 @@ +{"blockSize":[3,5],"compression":{"blocksize":0,"clevel":5,"cname":"lz4","shuffle":1,"type":"blosc"},"dataType":"uint16","dimensions":[3,5]} \ No newline at end of file diff --git a/testdata/datasource/n5/n5_viewer_multiscale_modern/attributes.json b/testdata/datasource/n5/n5_viewer_multiscale_modern/attributes.json new file mode 100644 index 0000000000..2b49d435e7 --- /dev/null +++ b/testdata/datasource/n5/n5_viewer_multiscale_modern/attributes.json @@ -0,0 +1 @@ +{"axes": ["x", "y"], "units": ["nm", "s"], "resolution": [10, 20]} \ No newline at end of file diff --git a/testdata/datasource/n5/n5_viewer_multiscale_modern/s0/attributes.json b/testdata/datasource/n5/n5_viewer_multiscale_modern/s0/attributes.json new file mode 100644 index 0000000000..1d0eb09bc9 --- /dev/null +++ b/testdata/datasource/n5/n5_viewer_multiscale_modern/s0/attributes.json @@ -0,0 +1 @@ +{"blockSize":[10,20],"compression":{"blocksize":0,"clevel":5,"cname":"lz4","shuffle":1,"type":"blosc"},"dataType":"uint16","dimensions":[10,20],"downsamplingFactors":[1,1]} \ No newline at end of file diff --git a/testdata/datasource/n5/n5_viewer_multiscale_modern/s1/attributes.json b/testdata/datasource/n5/n5_viewer_multiscale_modern/s1/attributes.json new file mode 100644 index 0000000000..08ca0551a5 --- /dev/null +++ b/testdata/datasource/n5/n5_viewer_multiscale_modern/s1/attributes.json @@ -0,0 +1 @@ +{"blockSize":[5,10],"compression":{"blocksize":0,"clevel":5,"cname":"lz4","shuffle":1,"type":"blosc"},"dataType":"uint16","dimensions":[5,10],"downsamplingFactors":[2,2]} \ No newline at end of file diff --git a/testdata/datasource/n5/n5_viewer_multiscale_modern/s2/attributes.json b/testdata/datasource/n5/n5_viewer_multiscale_modern/s2/attributes.json new file mode 100644 index 0000000000..8f77ef7ea7 --- /dev/null +++ b/testdata/datasource/n5/n5_viewer_multiscale_modern/s2/attributes.json @@ -0,0 +1 @@ +{"blockSize":[3,5],"compression":{"blocksize":0,"clevel":5,"cname":"lz4","shuffle":1,"type":"blosc"},"dataType":"uint16","dimensions":[3,5],"downsamplingFactors":[4,4]} \ No newline at end of file diff --git a/testdata/datasource/nifti/README.md b/testdata/datasource/nifti/README.md new file mode 100644 index 0000000000..0aaf544ebd --- /dev/null +++ b/testdata/datasource/nifti/README.md @@ -0,0 +1,35 @@ +The following test files: + +- example_nifti2.nii.gz +- standard.nii.gz +- standard.nii (derived from standard.nii.gz) + +were copied from nibabel, and are subject to the following license: + +The MIT License + +Copyright (c) 2009-2019 Matthew Brett +Copyright (c) 2010-2013 Stephan Gerhard +Copyright (c) 2006-2014 Michael Hanke +Copyright (c) 2011 Christian Haselgrove +Copyright (c) 2010-2011 Jarrod Millman +Copyright (c) 2011-2019 Yaroslav Halchenko +Copyright (c) 2015-2019 Chris Markiewicz + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/testdata/datasource/nifti/example_nifti2.nii.gz b/testdata/datasource/nifti/example_nifti2.nii.gz new file mode 100644 index 0000000000..a0d9e408f4 Binary files /dev/null and b/testdata/datasource/nifti/example_nifti2.nii.gz differ diff --git a/testdata/datasource/nifti/standard.nii b/testdata/datasource/nifti/standard.nii new file mode 100644 index 0000000000..d685a25135 Binary files /dev/null and b/testdata/datasource/nifti/standard.nii differ diff --git a/testdata/datasource/nifti/standard.nii.gz b/testdata/datasource/nifti/standard.nii.gz new file mode 100644 index 0000000000..98bb31a778 Binary files /dev/null and b/testdata/datasource/nifti/standard.nii.gz differ diff --git a/testdata/datasource/precomputed/generate_precomputed.py b/testdata/datasource/precomputed/generate_precomputed.py new file mode 100755 index 0000000000..24e72e36e8 --- /dev/null +++ b/testdata/datasource/precomputed/generate_precomputed.py @@ -0,0 +1,46 @@ +#!/usr/bin/env -S uv run +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "tensorstore", +# "numpy", +# ] +# /// +import os +import shutil + +import numpy as np +import tensorstore as ts + + +def write_multiscale(path: str, num_channels: int, num_scales: int): + shutil.rmtree(path, ignore_errors=True) + shape = np.array([10, 20, 30, num_channels]) + base_resolution = np.array([3, 4, 5]) + for scale in range(num_scales): + downsample_factors = [2**scale, 2**scale, 2**scale, 1] + ts.open( + { + "driver": "neuroglancer_precomputed", + "kvstore": {"driver": "file", "path": path}, + "scale_metadata": { + "resolution": base_resolution * downsample_factors[:-1] + }, + }, + create=True, + dtype=ts.uint16, + shape=-(-shape // downsample_factors), + ).result() + + +write_multiscale( + os.path.abspath(os.path.join(os.path.dirname(__file__), "one_channel")), + num_channels=1, + num_scales=3, +) + +write_multiscale( + os.path.abspath(os.path.join(os.path.dirname(__file__), "two_channels")), + num_channels=2, + num_scales=3, +) diff --git a/testdata/datasource/precomputed/one_channel/info b/testdata/datasource/precomputed/one_channel/info new file mode 100644 index 0000000000..1408773d9f --- /dev/null +++ b/testdata/datasource/precomputed/one_channel/info @@ -0,0 +1 @@ +{"@type":"neuroglancer_multiscale_volume","data_type":"uint16","num_channels":1,"scales":[{"chunk_sizes":[[10,20,30]],"encoding":"raw","key":"3_4_5","resolution":[3.0,4.0,5.0],"size":[10,20,30],"voxel_offset":[0,0,0]},{"chunk_sizes":[[5,10,15]],"encoding":"raw","key":"6_8_10","resolution":[6.0,8.0,10.0],"size":[5,10,15],"voxel_offset":[0,0,0]},{"chunk_sizes":[[3,5,8]],"encoding":"raw","key":"12_16_20","resolution":[12.0,16.0,20.0],"size":[3,5,8],"voxel_offset":[0,0,0]}],"type":"image"} \ No newline at end of file diff --git a/testdata/datasource/precomputed/two_channels/info b/testdata/datasource/precomputed/two_channels/info new file mode 100644 index 0000000000..58769bce6f --- /dev/null +++ b/testdata/datasource/precomputed/two_channels/info @@ -0,0 +1 @@ +{"@type":"neuroglancer_multiscale_volume","data_type":"uint16","num_channels":2,"scales":[{"chunk_sizes":[[10,20,30]],"encoding":"raw","key":"3_4_5","resolution":[3.0,4.0,5.0],"size":[10,20,30],"voxel_offset":[0,0,0]},{"chunk_sizes":[[5,10,15]],"encoding":"raw","key":"6_8_10","resolution":[6.0,8.0,10.0],"size":[5,10,15],"voxel_offset":[0,0,0]},{"chunk_sizes":[[3,5,8]],"encoding":"raw","key":"12_16_20","resolution":[12.0,16.0,20.0],"size":[3,5,8],"voxel_offset":[0,0,0]}],"type":"image"} \ No newline at end of file diff --git a/testdata/datasource/zarr/ome_zarr/generate_ome_zarr.py b/testdata/datasource/zarr/ome_zarr/generate_ome_zarr.py new file mode 100755 index 0000000000..97ffac099e --- /dev/null +++ b/testdata/datasource/zarr/ome_zarr/generate_ome_zarr.py @@ -0,0 +1,57 @@ +#!/usr/bin/env -S uv run +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "ngff-zarr[tensorstore]", +# "numpy", +# ] +# /// + +import os +import shutil +import zipfile + +import ngff_zarr as nz +import numpy as np + +THIS_DIR = os.path.abspath(os.path.dirname(__file__)) + + +def write_data(path: str, version: str): + shape = [10, 10] + + data = np.arange(np.prod(shape), dtype=np.uint16).reshape(shape) + + image = nz.to_ngff_image( + data, + dims=["y", "x"], + scale={"y": 4, "x": 30}, + axes_units={"y": "nanometer", "x": "nanometer"}, + ) + + multiscales = nz.to_multiscales( + image, scale_factors=[{"x": 2, "y": 2}, {"x": 4, "y": 4}] + ) + + full_path = os.path.join(THIS_DIR, path) + shutil.rmtree(full_path, ignore_errors=True) + + nz.to_ngff_zarr( + full_path, + multiscales, + use_tensorstore=True, + version=version, + ) + + +def create_zip(directory_path: str, zip_path: str): + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_STORED) as zipf: + for root, dirs, files in os.walk(directory_path): + for filename in files: + entry_path = os.path.join(root, filename) + zipf.write(entry_path, os.path.relpath(entry_path, directory_path)) + + +write_data("simple_0.4", version="0.4") +write_data("simple_0.5", version="0.5") +create_zip(os.path.join(THIS_DIR, "simple_0.5"), "simple_0.4.zip") diff --git a/testdata/datasource/zarr/ome_zarr/simple_0.4.zip b/testdata/datasource/zarr/ome_zarr/simple_0.4.zip new file mode 100644 index 0000000000..3766297dfa Binary files /dev/null and b/testdata/datasource/zarr/ome_zarr/simple_0.4.zip differ diff --git a/testdata/datasource/zarr/ome_zarr/simple_0.4/.zattrs b/testdata/datasource/zarr/ome_zarr/simple_0.4/.zattrs new file mode 100644 index 0000000000..8881cf31db --- /dev/null +++ b/testdata/datasource/zarr/ome_zarr/simple_0.4/.zattrs @@ -0,0 +1,80 @@ +{ + "multiscales": [ + { + "axes": [ + { + "name": "y", + "type": "space", + "unit": "nanometer" + }, + { + "name": "x", + "type": "space", + "unit": "nanometer" + } + ], + "datasets": [ + { + "path": "scale0/image", + "coordinateTransformations": [ + { + "scale": [ + 4, + 30 + ], + "type": "scale" + }, + { + "translation": [ + 0.0, + 0.0 + ], + "type": "translation" + } + ] + }, + { + "path": "scale1/image", + "coordinateTransformations": [ + { + "scale": [ + 8, + 60 + ], + "type": "scale" + }, + { + "translation": [ + 2, + 15 + ], + "type": "translation" + } + ] + }, + { + "path": "scale2/image", + "coordinateTransformations": [ + { + "scale": [ + 16, + 120 + ], + "type": "scale" + }, + { + "translation": [ + 6, + 45 + ], + "type": "translation" + } + ] + } + ], + "name": "image", + "version": "0.4", + "@type": "ngff:Image" + } + ] +} \ No newline at end of file diff --git a/testdata/datasource/zarr/ome_zarr/simple_0.4/.zgroup b/testdata/datasource/zarr/ome_zarr/simple_0.4/.zgroup new file mode 100644 index 0000000000..cab13da6ee --- /dev/null +++ b/testdata/datasource/zarr/ome_zarr/simple_0.4/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} \ No newline at end of file diff --git a/testdata/datasource/zarr/ome_zarr/simple_0.4/.zmetadata b/testdata/datasource/zarr/ome_zarr/simple_0.4/.zmetadata new file mode 100644 index 0000000000..18b2fe0a54 --- /dev/null +++ b/testdata/datasource/zarr/ome_zarr/simple_0.4/.zmetadata @@ -0,0 +1,202 @@ +{ + "metadata": { + ".zgroup": { + "zarr_format": 2 + }, + ".zattrs": { + "multiscales": [ + { + "axes": [ + { + "name": "y", + "type": "space", + "unit": "nanometer" + }, + { + "name": "x", + "type": "space", + "unit": "nanometer" + } + ], + "datasets": [ + { + "path": "scale0/image", + "coordinateTransformations": [ + { + "scale": [ + 4, + 30 + ], + "type": "scale" + }, + { + "translation": [ + 0.0, + 0.0 + ], + "type": "translation" + } + ] + }, + { + "path": "scale1/image", + "coordinateTransformations": [ + { + "scale": [ + 8, + 60 + ], + "type": "scale" + }, + { + "translation": [ + 2, + 15 + ], + "type": "translation" + } + ] + }, + { + "path": "scale2/image", + "coordinateTransformations": [ + { + "scale": [ + 16, + 120 + ], + "type": "scale" + }, + { + "translation": [ + 6, + 45 + ], + "type": "translation" + } + ] + } + ], + "name": "image", + "version": "0.4", + "@type": "ngff:Image" + } + ] + }, + "scale2/.zattrs": { + "_ARRAY_DIMENSIONS": [ + "y", + "x" + ] + }, + "scale2/.zgroup": { + "zarr_format": 2, + "consolidated_metadata": { + "metadata": {}, + "must_understand": false, + "kind": "inline" + } + }, + "scale2/image/.zattrs": {}, + "scale2/image/.zarray": { + "shape": [ + 2, + 2 + ], + "chunks": [ + 2, + 2 + ], + "fill_value": null, + "order": "C", + "filters": null, + "dimension_separator": "/", + "compressor": { + "id": "blosc", + "cname": "lz4", + "clevel": 5, + "shuffle": -1, + "blocksize": 0 + }, + "zarr_format": 2, + "dtype": " { + const serverFixture = httpServerFixture(constantFixture(TEST_FILES_DIR)); + const kvStoreContextFixture = sharedKvStoreContextFixture(); + test("get completions", async () => { + const serverUrl = await serverFixture.serverUrl(); + expect( + await getKvStoreCompletions(await kvStoreContextFixture(), { + url: serverUrl, + }), + ).toMatchInlineSnapshot(` + { + "completions": [ + { + "value": "baz/", + }, + { + "value": "#|", + }, + { + "value": "a|", + }, + { + "value": "b|", + }, + { + "value": "c|", + }, + { + "value": "empty|", + }, + ], + "defaultCompletion": undefined, + "offset": 23, + } + `); + }); + + test("get completions with partial name", async () => { + const serverUrl = await serverFixture.serverUrl(); + expect( + await getKvStoreCompletions(await kvStoreContextFixture(), { + url: serverUrl + "b", + }), + ).toMatchInlineSnapshot(` + { + "completions": [ + { + "value": "baz/", + }, + { + "value": "b|", + }, + ], + "defaultCompletion": undefined, + "offset": 23, + } + `); + }); + + test("get completions with subdirectory", async () => { + const serverUrl = await serverFixture.serverUrl(); + expect( + await getKvStoreCompletions(await kvStoreContextFixture(), { + url: serverUrl + "baz/", + }), + ).toMatchInlineSnapshot(` + { + "completions": [ + { + "value": "x|", + }, + ], + "defaultCompletion": undefined, + "offset": 27, + } + `); + }); +}); + +describe("datasource completion", () => { + const datasourceProviderFixture = dataSourceProviderFixture(); + + test("get empty completions", async () => { + expect(await (await datasourceProviderFixture()).completeUrl({ url: "" })) + .toMatchInlineSnapshot(` + { + "completions": [ + { + "description": "http (unauthenticated)", + "value": "http://", + }, + { + "description": "https (unauthenticated)", + "value": "https://", + }, + { + "description": "Local in-memory", + "value": "local://", + }, + ], + "offset": 0, + } + `); + }); +}); diff --git a/tests/datasource/deepzoom.browser_test.ts b/tests/datasource/deepzoom.browser_test.ts new file mode 100644 index 0000000000..924ccf3547 --- /dev/null +++ b/tests/datasource/deepzoom.browser_test.ts @@ -0,0 +1,21 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "#src/datasource/deepzoom/register_default"; +import "#src/sliceview/uncompressed_chunk_format.js"; +import { datasourceMetadataSnapshotTests } from "#tests/datasource/metadata_snapshot_test_util.js"; + +datasourceMetadataSnapshotTests("deepzoom", ["14122_mPPC_BDA_s186.dzi"]); diff --git a/tests/datasource/metadata_snapshot_test_util.ts b/tests/datasource/metadata_snapshot_test_util.ts new file mode 100644 index 0000000000..e03580b874 --- /dev/null +++ b/tests/datasource/metadata_snapshot_test_util.ts @@ -0,0 +1,42 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "#src/kvstore/http/register.js"; +import { describe, test } from "vitest"; +import { getDatasourceMetadata } from "#tests/datasource/test_util.js"; +import { dataSourceProviderFixture } from "#tests/fixtures/datasource_provider.js"; + +declare const TEST_DATA_SERVER: string; + +export const dataSourceProvider = dataSourceProviderFixture(); + +export function datasourceMetadataSnapshotTests( + datasourceName: string, + names: string[], +) { + describe("metadata tests", () => { + test.for(names)("metadata %s", async (name, { expect }) => { + await expect( + await getDatasourceMetadata( + dataSourceProvider, + `${TEST_DATA_SERVER}datasource/${datasourceName}/${name}`, + ), + ).toMatchFileSnapshot( + `./metadata_snapshots/${datasourceName}/${name.replaceAll("/", "_")}.snapshot`, + ); + }); + }); +} diff --git a/tests/datasource/metadata_snapshots/deepzoom/14122_mPPC_BDA_s186.dzi.snapshot b/tests/datasource/metadata_snapshots/deepzoom/14122_mPPC_BDA_s186.dzi.snapshot new file mode 100644 index 0000000000..9556efadaf --- /dev/null +++ b/tests/datasource/metadata_snapshots/deepzoom/14122_mPPC_BDA_s186.dzi.snapshot @@ -0,0 +1,1181 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186.dzi|deepzoom:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + 0, + 0, + 0, + ], + "upperBounds": Float64Array [ + 48023, + 34907, + 3, + ], + "voxelCenterAtIntegerCoordinates": [ + false, + false, + false, + ], + }, + "coordinateArrays": [ + , + , + , + ], + "names": [ + "x", + "y", + "c^", + ], + "scales": Float64Array [ + 1e-9, + 1e-9, + 1, + ], + "units": [ + "m", + "m", + "", + ], + "valid": true, + }, + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "UINT8", + "rank": 3, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/16/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 48023, + 34907, + 3, + ], + }, + }, + "upperClipBound": Float32Array [ + 48023, + 34907, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/15/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 24012, + 17454, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 2, + 0, + 0, + 0, + ], + [ + 0, + 2, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 24011.5, + 17453.5, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/14/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 12006, + 8727, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 4, + 0, + 0, + 0, + ], + [ + 0, + 4, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 12005.75, + 8726.75, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/13/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 6003, + 4364, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 8, + 0, + 0, + 0, + ], + [ + 0, + 8, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 6002.875, + 4363.375, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/12/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 3002, + 2182, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 16, + 0, + 0, + 0, + ], + [ + 0, + 16, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 3001.4375, + 2181.6875, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/11/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 1501, + 1091, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 32, + 0, + 0, + 0, + ], + [ + 0, + 32, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 1500.71875, + 1090.84375, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/10/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 751, + 546, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 64, + 0, + 0, + 0, + ], + [ + 0, + 64, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 750.359375, + 545.421875, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/9/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 376, + 273, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 128, + 0, + 0, + 0, + ], + [ + 0, + 128, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 375.1796875, + 272.7109375, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/8/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 188, + 137, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 256, + 0, + 0, + 0, + ], + [ + 0, + 256, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 187.58984375, + 136.35546875, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/7/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 94, + 69, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 512, + 0, + 0, + 0, + ], + [ + 0, + 512, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 93.794921875, + 68.177734375, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/6/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 47, + 35, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 1024, + 0, + 0, + 0, + ], + [ + 0, + 1024, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 46.8974609375, + 34.0888671875, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/5/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 24, + 18, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 2048, + 0, + 0, + 0, + ], + [ + 0, + 2048, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 23.44873046875, + 17.04443359375, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/4/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 12, + 9, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 4096, + 0, + 0, + 0, + ], + [ + 0, + 4096, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 11.724365234375, + 8.522216796875, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/3/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 6, + 5, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 8192, + 0, + 0, + 0, + ], + [ + 0, + 8192, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 5.8621826171875, + 4.2611083984375, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/2/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 3, + 3, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 16384, + 0, + 0, + 0, + ], + [ + 0, + 16384, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 2.93109130859375, + 2.13055419921875, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/1/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 2, + 2, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 32768, + 0, + 0, + 0, + ], + [ + 0, + 32768, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 1.465545654296875, + 1.065277099609375, + 3, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "format": "jpg", + "overlap": 1, + "tilesize": 254, + "url": "http://localhost:*/datasource/deepzoom/14122_mPPC_BDA_s186_files/0/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 254, + 254, + 3, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 1, + 1, + 3, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 65536, + 0, + 0, + 0, + ], + [ + 0, + 65536, + 0, + 0, + ], + [ + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "upperClipBound": Float32Array [ + 0.7327728271484375, + 0.5326385498046875, + 3, + ], + }, + ], + ], + "volumeType": "IMAGE", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + 0, + 0, + 0, + ], + "pointB": [ + 48023, + 34907, + 3, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/metadata_snapshots/n5/n5_viewer_multiscale_deprecated.snapshot b/tests/datasource/metadata_snapshots/n5/n5_viewer_multiscale_deprecated.snapshot new file mode 100644 index 0000000000..f391dcfe50 --- /dev/null +++ b/tests/datasource/metadata_snapshots/n5/n5_viewer_multiscale_deprecated.snapshot @@ -0,0 +1,194 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/n5/n5_viewer_multiscale_deprecated/|n5:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + 0, + 0, + ], + "upperBounds": Float64Array [ + 10, + 20, + ], + "voxelCenterAtIntegerCoordinates": [ + false, + false, + ], + }, + "coordinateArrays": [ + , + , + ], + "names": [ + "x", + "y", + ], + "scales": Float64Array [ + 1e-8, + 20, + ], + "units": [ + "m", + "s", + ], + "valid": true, + }, + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "UINT16", + "rank": 2, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "encoding": 3, + "url": "http://localhost:*/datasource/n5/n5_viewer_multiscale_deprecated/s0/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 10, + 20, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 10, + 20, + ], + }, + }, + }, + { + "chunkSource": { + "parameters": { + "encoding": 3, + "url": "http://localhost:*/datasource/n5/n5_viewer_multiscale_deprecated/s1/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 5, + 10, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 5, + 10, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 2, + 0, + 0, + ], + [ + 0, + 2, + 0, + ], + [ + 0, + 0, + 1, + ], + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 3, + "url": "http://localhost:*/datasource/n5/n5_viewer_multiscale_deprecated/s2/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 3, + 5, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 3, + 5, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 4, + 0, + 0, + ], + [ + 0, + 4, + 0, + ], + [ + 0, + 0, + 1, + ], + ], + }, + ], + ], + "volumeType": "IMAGE", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + 0, + 0, + ], + "pointB": [ + 10, + 20, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/metadata_snapshots/n5/n5_viewer_multiscale_modern.snapshot b/tests/datasource/metadata_snapshots/n5/n5_viewer_multiscale_modern.snapshot new file mode 100644 index 0000000000..aca3867952 --- /dev/null +++ b/tests/datasource/metadata_snapshots/n5/n5_viewer_multiscale_modern.snapshot @@ -0,0 +1,194 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/n5/n5_viewer_multiscale_modern/|n5:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + 0, + 0, + ], + "upperBounds": Float64Array [ + 10, + 20, + ], + "voxelCenterAtIntegerCoordinates": [ + false, + false, + ], + }, + "coordinateArrays": [ + , + , + ], + "names": [ + "x", + "y", + ], + "scales": Float64Array [ + 1e-8, + 20, + ], + "units": [ + "m", + "s", + ], + "valid": true, + }, + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "UINT16", + "rank": 2, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "encoding": 3, + "url": "http://localhost:*/datasource/n5/n5_viewer_multiscale_modern/s0/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 10, + 20, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 10, + 20, + ], + }, + }, + }, + { + "chunkSource": { + "parameters": { + "encoding": 3, + "url": "http://localhost:*/datasource/n5/n5_viewer_multiscale_modern/s1/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 5, + 10, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 5, + 10, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 2, + 0, + 0, + ], + [ + 0, + 2, + 0, + ], + [ + 0, + 0, + 1, + ], + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 3, + "url": "http://localhost:*/datasource/n5/n5_viewer_multiscale_modern/s2/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 3, + 5, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 3, + 5, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 4, + 0, + 0, + ], + [ + 0, + 4, + 0, + ], + [ + 0, + 0, + 1, + ], + ], + }, + ], + ], + "volumeType": "IMAGE", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + 0, + 0, + ], + "pointB": [ + 10, + 20, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/metadata_snapshots/nifti/example_nifti2.nii.gz.snapshot b/tests/datasource/metadata_snapshots/nifti/example_nifti2.nii.gz.snapshot new file mode 100644 index 0000000000..165e4f1203 --- /dev/null +++ b/tests/datasource/metadata_snapshots/nifti/example_nifti2.nii.gz.snapshot @@ -0,0 +1,213 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/nifti/example_nifti2.nii.gz|gzip:|nifti:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + 0, + 0, + 0, + 0, + ], + "upperBounds": Float64Array [ + 32, + 20, + 12, + 2, + ], + "voxelCenterAtIntegerCoordinates": [ + false, + false, + false, + false, + ], + }, + "coordinateArrays": [ + , + , + , + , + ], + "names": [ + "i", + "j", + "k", + "m", + ], + "scales": Float64Array [ + 0.002, + 0.002, + 0.0021999990940093994, + 2000, + ], + "units": [ + "m", + "m", + "m", + "s", + ], + "valid": true, + }, + "outputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + -Infinity, + -Infinity, + -Infinity, + -Infinity, + ], + "upperBounds": Float64Array [ + Infinity, + Infinity, + Infinity, + Infinity, + ], + "voxelCenterAtIntegerCoordinates": [ + false, + false, + false, + false, + ], + }, + "coordinateArrays": [ + , + , + , + , + ], + "names": [ + "x", + "y", + "z", + "t", + ], + "scales": Float64Array [ + 0.001, + 0.001, + 0.001, + 1, + ], + "units": [ + "m", + "m", + "m", + "s", + ], + "valid": true, + }, + "transform": [ + [ + -1, + 0.000005141198016644921, + 0.00006320902321022004, + 0, + 117.8551025390625, + ], + [ + -0.000005141198016644921, + 0.9868557453155518, + -0.16160380840301514, + 0, + -35.72294235229492, + ], + [ + 0.00006320902321022004, + 0.16160380840301514, + 0.9868557453155518, + 0, + -7.248798370361328, + ], + [ + 0, + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 0, + 1, + ], + ], + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "INT16", + "rank": 4, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/nifti/example_nifti2.nii.gz|gzip:", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + 0, + ], + "chunkDataSize": [ + 32, + 20, + 12, + 2, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "INT16", + "lowerVoxelBound": [ + 0, + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 32, + 20, + 12, + 2, + ], + }, + }, + }, + ], + ], + "volumeType": "UNKNOWN", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + 0, + 0, + 0, + 0, + ], + "pointB": [ + 32, + 20, + 12, + 2, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/metadata_snapshots/nifti/standard.nii.gz.snapshot b/tests/datasource/metadata_snapshots/nifti/standard.nii.gz.snapshot new file mode 100644 index 0000000000..ae3792a1a4 --- /dev/null +++ b/tests/datasource/metadata_snapshots/nifti/standard.nii.gz.snapshot @@ -0,0 +1,156 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/nifti/standard.nii.gz|gzip:|nifti:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + 0, + 0, + 0, + ], + "upperBounds": Float64Array [ + 4, + 5, + 7, + ], + "voxelCenterAtIntegerCoordinates": [ + false, + false, + false, + ], + }, + "coordinateArrays": [ + , + , + , + ], + "names": [ + "i", + "j", + "k", + ], + "scales": Float64Array [ + 1, + 3, + 2, + ], + "units": [ + "", + "", + "", + ], + "valid": true, + }, + "outputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + -Infinity, + -Infinity, + -Infinity, + ], + "upperBounds": Float64Array [ + Infinity, + Infinity, + Infinity, + ], + "voxelCenterAtIntegerCoordinates": [ + false, + false, + false, + ], + }, + "coordinateArrays": [ + , + , + , + ], + "names": [ + "x", + "y", + "z", + ], + "scales": Float64Array [ + 1, + 1, + 1, + ], + "units": [ + "", + "", + "", + ], + "valid": true, + }, + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "UINT8", + "rank": 3, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/nifti/standard.nii.gz|gzip:", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 4, + 5, + 7, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 4, + 5, + 7, + ], + }, + }, + }, + ], + ], + "volumeType": "UNKNOWN", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + 0, + 0, + 0, + ], + "pointB": [ + 4, + 5, + 7, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/metadata_snapshots/nifti/standard.nii.snapshot b/tests/datasource/metadata_snapshots/nifti/standard.nii.snapshot new file mode 100644 index 0000000000..3c57057f7b --- /dev/null +++ b/tests/datasource/metadata_snapshots/nifti/standard.nii.snapshot @@ -0,0 +1,156 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/nifti/standard.nii|nifti:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + 0, + 0, + 0, + ], + "upperBounds": Float64Array [ + 4, + 5, + 7, + ], + "voxelCenterAtIntegerCoordinates": [ + false, + false, + false, + ], + }, + "coordinateArrays": [ + , + , + , + ], + "names": [ + "i", + "j", + "k", + ], + "scales": Float64Array [ + 1, + 3, + 2, + ], + "units": [ + "", + "", + "", + ], + "valid": true, + }, + "outputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + -Infinity, + -Infinity, + -Infinity, + ], + "upperBounds": Float64Array [ + Infinity, + Infinity, + Infinity, + ], + "voxelCenterAtIntegerCoordinates": [ + false, + false, + false, + ], + }, + "coordinateArrays": [ + , + , + , + ], + "names": [ + "x", + "y", + "z", + ], + "scales": Float64Array [ + 1, + 1, + 1, + ], + "units": [ + "", + "", + "", + ], + "valid": true, + }, + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "UINT8", + "rank": 3, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/nifti/standard.nii", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 4, + 5, + 7, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT8", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 4, + 5, + 7, + ], + }, + }, + }, + ], + ], + "volumeType": "UNKNOWN", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + 0, + 0, + 0, + ], + "pointB": [ + 4, + 5, + 7, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/metadata_snapshots/precomputed/one_channel.snapshot b/tests/datasource/metadata_snapshots/precomputed/one_channel.snapshot new file mode 100644 index 0000000000..df3b27612c --- /dev/null +++ b/tests/datasource/metadata_snapshots/precomputed/one_channel.snapshot @@ -0,0 +1,266 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/precomputed/one_channel/|neuroglancer-precomputed:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + 0, + 0, + 0, + ], + "upperBounds": Float64Array [ + 10, + 20, + 30, + ], + "voxelCenterAtIntegerCoordinates": [ + false, + false, + false, + ], + }, + "coordinateArrays": [ + , + , + , + ], + "names": [ + "x", + "y", + "z", + ], + "scales": Float64Array [ + 3e-9, + 4e-9, + 5e-9, + ], + "units": [ + "m", + "m", + "m", + ], + "valid": true, + }, + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "UINT16", + "rank": 3, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "encoding": 0, + "sharding": undefined, + "url": "http://localhost:*/datasource/precomputed/one_channel/3_4_5/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 10, + 20, + 30, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 10, + 20, + 30, + ], + }, + }, + "lowerClipBound": Float32Array [ + 0, + 0, + 0, + ], + "upperClipBound": Float32Array [ + 10, + 20, + 30, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "sharding": undefined, + "url": "http://localhost:*/datasource/precomputed/one_channel/6_8_10/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 5, + 10, + 15, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 5, + 10, + 15, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 2, + 0, + 0, + 0, + ], + [ + 0, + 2, + 0, + 0, + ], + [ + 0, + 0, + 2, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "lowerClipBound": Float32Array [ + 0, + 0, + 0, + ], + "upperClipBound": Float32Array [ + 5, + 10, + 15, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "sharding": undefined, + "url": "http://localhost:*/datasource/precomputed/one_channel/12_16_20/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + ], + "chunkDataSize": [ + 3, + 5, + 8, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 3, + 5, + 8, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 4, + 0, + 0, + 0, + ], + [ + 0, + 4, + 0, + 0, + ], + [ + 0, + 0, + 4, + 0, + ], + [ + 0, + 0, + 0, + 1, + ], + ], + "lowerClipBound": Float32Array [ + 0, + 0, + 0, + ], + "upperClipBound": Float32Array [ + 2.5, + 5, + 7.5, + ], + }, + ], + ], + "volumeType": "IMAGE", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + 0, + 0, + 0, + ], + "pointB": [ + 10, + 20, + 30, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/metadata_snapshots/precomputed/two_channels.snapshot b/tests/datasource/metadata_snapshots/precomputed/two_channels.snapshot new file mode 100644 index 0000000000..1b252e1674 --- /dev/null +++ b/tests/datasource/metadata_snapshots/precomputed/two_channels.snapshot @@ -0,0 +1,315 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/precomputed/two_channels/|neuroglancer-precomputed:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + 0, + 0, + 0, + 0, + ], + "upperBounds": Float64Array [ + 10, + 20, + 30, + 2, + ], + "voxelCenterAtIntegerCoordinates": [ + false, + false, + false, + false, + ], + }, + "coordinateArrays": [ + , + , + , + , + ], + "names": [ + "x", + "y", + "z", + "c^", + ], + "scales": Float64Array [ + 3e-9, + 4e-9, + 5e-9, + 1, + ], + "units": [ + "m", + "m", + "m", + "", + ], + "valid": true, + }, + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "UINT16", + "rank": 4, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "encoding": 0, + "sharding": undefined, + "url": "http://localhost:*/datasource/precomputed/two_channels/3_4_5/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + 0, + ], + "chunkDataSize": [ + 10, + 20, + 30, + 2, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 10, + 20, + 30, + 2, + ], + }, + }, + "lowerClipBound": Float32Array [ + 0, + 0, + 0, + 0, + ], + "upperClipBound": Float32Array [ + 10, + 20, + 30, + 2, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "sharding": undefined, + "url": "http://localhost:*/datasource/precomputed/two_channels/6_8_10/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + 0, + ], + "chunkDataSize": [ + 5, + 10, + 15, + 2, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 5, + 10, + 15, + 2, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 2, + 0, + 0, + 0, + 0, + ], + [ + 0, + 2, + 0, + 0, + 0, + ], + [ + 0, + 0, + 2, + 0, + 0, + ], + [ + 0, + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 0, + 1, + ], + ], + "lowerClipBound": Float32Array [ + 0, + 0, + 0, + 0, + ], + "upperClipBound": Float32Array [ + 5, + 10, + 15, + 2, + ], + }, + { + "chunkSource": { + "parameters": { + "encoding": 0, + "sharding": undefined, + "url": "http://localhost:*/datasource/precomputed/two_channels/12_16_20/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + 0, + 0, + ], + "chunkDataSize": [ + 3, + 5, + 8, + 2, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + 0, + 0, + ], + "upperVoxelBound": [ + 3, + 5, + 8, + 2, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 4, + 0, + 0, + 0, + 0, + ], + [ + 0, + 4, + 0, + 0, + 0, + ], + [ + 0, + 0, + 4, + 0, + 0, + ], + [ + 0, + 0, + 0, + 1, + 0, + ], + [ + 0, + 0, + 0, + 0, + 1, + ], + ], + "lowerClipBound": Float32Array [ + 0, + 0, + 0, + 0, + ], + "upperClipBound": Float32Array [ + 2.5, + 5, + 7.5, + 2, + ], + }, + ], + ], + "volumeType": "IMAGE", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + 0, + 0, + 0, + 0, + ], + "pointB": [ + 10, + 20, + 30, + 2, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/metadata_snapshots/zarr/ome_zarr_simple_0.4.snapshot b/tests/datasource/metadata_snapshots/zarr/ome_zarr_simple_0.4.snapshot new file mode 100644 index 0000000000..2a04b75837 --- /dev/null +++ b/tests/datasource/metadata_snapshots/zarr/ome_zarr_simple_0.4.snapshot @@ -0,0 +1,208 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/zarr/ome_zarr/simple_0.4/|zarr2:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + -0.5, + -0.5, + ], + "upperBounds": Float64Array [ + 9.5, + 9.5, + ], + "voxelCenterAtIntegerCoordinates": [ + true, + true, + ], + }, + "coordinateArrays": [ + , + , + ], + "names": [ + "y", + "x", + ], + "scales": Float64Array [ + 4e-9, + 3.0000000000000004e-8, + ], + "units": [ + "m", + "m", + ], + "valid": true, + }, + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "UINT16", + "rank": 2, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/zarr/ome_zarr/simple_0.4/scale0/image/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 10, + 10, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 10, + 10, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 0, + 1, + -0.5, + ], + [ + 1, + 0, + -0.5, + ], + [ + 0, + 0, + 1, + ], + ], + }, + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/zarr/ome_zarr/simple_0.4/scale1/image/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 5, + 5, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 5, + 5, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 0, + 2, + -0.5, + ], + [ + 2, + 0, + -0.5, + ], + [ + 0, + 0, + 1, + ], + ], + }, + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/zarr/ome_zarr/simple_0.4/scale2/image/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 2, + 2, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 2, + 2, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 0, + 4, + -0.5, + ], + [ + 4, + 0, + -0.5, + ], + [ + 0, + 0, + 1, + ], + ], + }, + ], + ], + "volumeType": "IMAGE", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + -0.5, + -0.5, + ], + "pointB": [ + 9.5, + 9.5, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/metadata_snapshots/zarr/ome_zarr_simple_0.4.zip.snapshot b/tests/datasource/metadata_snapshots/zarr/ome_zarr_simple_0.4.zip.snapshot new file mode 100644 index 0000000000..2f91f77c3c --- /dev/null +++ b/tests/datasource/metadata_snapshots/zarr/ome_zarr_simple_0.4.zip.snapshot @@ -0,0 +1,208 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/zarr/ome_zarr/simple_0.4.zip|zip:|zarr3:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + -0.5, + -0.5, + ], + "upperBounds": Float64Array [ + 9.5, + 9.5, + ], + "voxelCenterAtIntegerCoordinates": [ + true, + true, + ], + }, + "coordinateArrays": [ + , + , + ], + "names": [ + "y", + "x", + ], + "scales": Float64Array [ + 4e-9, + 3.0000000000000004e-8, + ], + "units": [ + "m", + "m", + ], + "valid": true, + }, + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "UINT16", + "rank": 2, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/zarr/ome_zarr/simple_0.4.zip|zip:scale0/image/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 10, + 10, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 10, + 10, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 0, + 1, + -0.5, + ], + [ + 1, + 0, + -0.5, + ], + [ + 0, + 0, + 1, + ], + ], + }, + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/zarr/ome_zarr/simple_0.4.zip|zip:scale1/image/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 5, + 5, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 5, + 5, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 0, + 2, + -0.5, + ], + [ + 2, + 0, + -0.5, + ], + [ + 0, + 0, + 1, + ], + ], + }, + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/zarr/ome_zarr/simple_0.4.zip|zip:scale2/image/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 2, + 2, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 2, + 2, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 0, + 4, + -0.5, + ], + [ + 4, + 0, + -0.5, + ], + [ + 0, + 0, + 1, + ], + ], + }, + ], + ], + "volumeType": "IMAGE", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + -0.5, + -0.5, + ], + "pointB": [ + 9.5, + 9.5, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/metadata_snapshots/zarr/ome_zarr_simple_0.5.snapshot b/tests/datasource/metadata_snapshots/zarr/ome_zarr_simple_0.5.snapshot new file mode 100644 index 0000000000..fe94094414 --- /dev/null +++ b/tests/datasource/metadata_snapshots/zarr/ome_zarr_simple_0.5.snapshot @@ -0,0 +1,208 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/zarr/ome_zarr/simple_0.5/|zarr3:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + -0.5, + -0.5, + ], + "upperBounds": Float64Array [ + 9.5, + 9.5, + ], + "voxelCenterAtIntegerCoordinates": [ + true, + true, + ], + }, + "coordinateArrays": [ + , + , + ], + "names": [ + "y", + "x", + ], + "scales": Float64Array [ + 4e-9, + 3.0000000000000004e-8, + ], + "units": [ + "m", + "m", + ], + "valid": true, + }, + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "UINT16", + "rank": 2, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/zarr/ome_zarr/simple_0.5/scale0/image/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 10, + 10, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 10, + 10, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 0, + 1, + -0.5, + ], + [ + 1, + 0, + -0.5, + ], + [ + 0, + 0, + 1, + ], + ], + }, + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/zarr/ome_zarr/simple_0.5/scale1/image/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 5, + 5, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 5, + 5, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 0, + 2, + -0.5, + ], + [ + 2, + 0, + -0.5, + ], + [ + 0, + 0, + 1, + ], + ], + }, + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/zarr/ome_zarr/simple_0.5/scale2/image/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 2, + 2, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 2, + 2, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 0, + 4, + -0.5, + ], + [ + 4, + 0, + -0.5, + ], + [ + 0, + 0, + 1, + ], + ], + }, + ], + ], + "volumeType": "IMAGE", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + -0.5, + -0.5, + ], + "pointB": [ + 9.5, + 9.5, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_single_res.snapshot b/tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_single_res.snapshot new file mode 100644 index 0000000000..61d179dcc5 --- /dev/null +++ b/tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_single_res.snapshot @@ -0,0 +1,120 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/zarr/zarr_v3/examples/single_res/|zarr3:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + 0, + 0, + ], + "upperBounds": Float64Array [ + 6, + 7, + ], + "voxelCenterAtIntegerCoordinates": [ + false, + false, + ], + }, + "coordinateArrays": [ + , + , + ], + "names": [ + "dim_0", + "dim_1", + ], + "scales": Float64Array [ + 1, + 1, + ], + "units": [ + "", + "", + ], + "valid": true, + }, + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "UINT16", + "rank": 2, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/zarr/zarr_v3/examples/single_res/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 5, + 4, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 7, + 6, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 0, + 1, + 0, + ], + [ + 1, + 0, + 0, + ], + [ + 0, + 0, + 1, + ], + ], + }, + ], + ], + "volumeType": "IMAGE", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + 0, + 0, + ], + "pointB": [ + 6, + 7, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/n5.browser_test.ts b/tests/datasource/n5.browser_test.ts new file mode 100644 index 0000000000..1de2f252a4 --- /dev/null +++ b/tests/datasource/n5.browser_test.ts @@ -0,0 +1,24 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "#src/datasource/n5/register_default"; +import "#src/sliceview/uncompressed_chunk_format.js"; +import { datasourceMetadataSnapshotTests } from "#tests/datasource/metadata_snapshot_test_util.js"; + +datasourceMetadataSnapshotTests("n5", [ + "n5_viewer_multiscale_deprecated", + "n5_viewer_multiscale_modern", +]); diff --git a/tests/datasource/nifti.browser_test.ts b/tests/datasource/nifti.browser_test.ts new file mode 100644 index 0000000000..430f745b46 --- /dev/null +++ b/tests/datasource/nifti.browser_test.ts @@ -0,0 +1,26 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "#src/datasource/nifti/register_default.js"; +import "#src/kvstore/gzip/register.js"; +import "#src/sliceview/uncompressed_chunk_format.js"; +import { datasourceMetadataSnapshotTests } from "#tests/datasource/metadata_snapshot_test_util.js"; + +datasourceMetadataSnapshotTests("nifti", [ + "standard.nii", + "standard.nii.gz", + "example_nifti2.nii.gz", +]); diff --git a/tests/datasource/precomputed.browser_test.ts b/tests/datasource/precomputed.browser_test.ts new file mode 100644 index 0000000000..735bbe16da --- /dev/null +++ b/tests/datasource/precomputed.browser_test.ts @@ -0,0 +1,21 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "#src/datasource/precomputed/register_default"; +import "#src/sliceview/uncompressed_chunk_format.js"; +import { datasourceMetadataSnapshotTests } from "#tests/datasource/metadata_snapshot_test_util.js"; + +datasourceMetadataSnapshotTests("precomputed", ["one_channel", "two_channels"]); diff --git a/tests/datasource/suggest_name.spec.ts b/tests/datasource/suggest_name.spec.ts new file mode 100644 index 0000000000..a9128e05f4 --- /dev/null +++ b/tests/datasource/suggest_name.spec.ts @@ -0,0 +1,31 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect, test } from "vitest"; +import { dataSourceProviderFixture } from "#tests/fixtures/datasource_provider.js"; + +const datasourceProviderFixture = dataSourceProviderFixture(); + +test.for([ + ["http://localhost/path/to/array/|zarr3:", "array"], + ["http://localhost/path/to/array/", "array"], + ["http://localhost/path/to/array", "array"], + ["http://localhost/path/to/group/|zarr3:path/to/array/", "array"], + ["brainmaps://foo:bar:baz", "baz"], +])("%s -> %s", async ([url, expectedName]) => { + const registry = await datasourceProviderFixture(); + expect(registry.suggestLayerName(url)).toEqual(expectedName); +}); diff --git a/tests/datasource/test_util.ts b/tests/datasource/test_util.ts new file mode 100644 index 0000000000..66b804ada5 --- /dev/null +++ b/tests/datasource/test_util.ts @@ -0,0 +1,184 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { + CoordinateSpace, + CoordinateSpaceTransform, +} from "#src/coordinate_transform.js"; +import { coordinateSpacesEqual } from "#src/coordinate_transform.js"; +import type { + DataSource, + DataSourceRegistry, + DataSubsource, + DataSubsourceEntry, +} from "#src/datasource/index.js"; +import type { SliceViewSingleResolutionSource } from "#src/sliceview/frontend.js"; +import { VolumeType } from "#src/sliceview/volume/base.js"; +import type { MultiscaleVolumeChunkSource } from "#src/sliceview/volume/frontend.js"; +import { VolumeChunkSource } from "#src/sliceview/volume/frontend.js"; +import { DataType } from "#src/util/data_type.js"; +import * as matrix from "#src/util/matrix.js"; +import type { ProgressListener } from "#src/util/progress_listener.js"; +import type { Fixture } from "#tests/fixtures/fixture.js"; + +export function getDatasourceSnapshot(datasource: DataSource) { + return { + subsources: Array.from(datasource.subsources, getSubsourceEntrySnapshot), + modelTransform: getCoordinateSpaceTransformSnapshot( + datasource.modelTransform, + ), + canonicalUrl: redactUrl(datasource.canonicalUrl), + ...getKeys(datasource, ["canChangeModelSpaceRank"]), + }; +} + +function getKeys(x: T, keys: Array) { + return Object.fromEntries( + keys + .map((key) => [key, x[key]]) + .filter(([_key, value]) => value !== undefined), + ); +} + +function getCoordinateSpaceSnapshot(x: CoordinateSpace) { + return getKeys(x, [ + "valid", + "names", + "units", + "scales", + "bounds", + "coordinateArrays", + ]); +} + +function getCoordinateSpaceTransformSnapshot(x: CoordinateSpaceTransform) { + const { rank } = x; + const result: any = {}; + result.inputSpace = getCoordinateSpaceSnapshot(x.inputSpace); + if (!coordinateSpacesEqual(x.inputSpace, x.outputSpace)) { + result.outputSpace = getCoordinateSpaceSnapshot(x.outputSpace); + } + if (!matrix.isIdentity(x.transform, rank + 1, rank + 1)) { + result.transform = getMatrixSnapshot(x.transform, rank + 1); + } + return result; +} + +function getMatrixSnapshot(matrix: ArrayLike, rows: number) { + const result: number[][] = []; + const cols = matrix.length / rows; + for (let row = 0; row < rows; ++row) { + const rowElements: number[] = []; + for (let col = 0; col < cols; ++col) { + rowElements[col] = matrix[col * rows + row]; + } + result[row] = rowElements; + } + return result; +} + +function getSubsourceEntrySnapshot(subsource: DataSubsourceEntry) { + return { + subsource: getSubsourceSnapshot(subsource.subsource), + ...getKeys(subsource, [ + "default", + "id", + "modelSubspaceDimensionIndices", + "subsourceToModelSubspaceTransform", + ]), + }; +} + +function getSubsourceSnapshot(subsource: DataSubsource) { + if (subsource.volume) { + return { volume: getVolumeSnapshot(subsource.volume) }; + } + if (subsource.staticAnnotations) { + return { staticAnnotations: subsource.staticAnnotations.toJSON() }; + } + return {}; +} + +function getVolumeSnapshot(volume: MultiscaleVolumeChunkSource) { + const sources = volume.getSources({ + multiscaleToViewTransform: matrix.createIdentity(Float32Array, volume.rank), + displayRank: volume.rank, + modelChannelDimensionIndices: [], + }); + return { + dataType: DataType[volume.dataType], + volumeType: VolumeType[volume.volumeType], + rank: volume.rank, + sources: sources.map((sourceList) => sourceList.map(getSourceSnapshot)), + }; +} + +function getSourceSnapshot( + source: SliceViewSingleResolutionSource, +) { + const rank = source.chunkSource.spec.rank; + let spec: any = VolumeChunkSource.encodeSpec(source.chunkSource.spec); + spec = { ...spec, dataType: DataType[spec.dataType] }; + const parameters = (source.chunkSource as any).parameters; + delete parameters.metadata; + if (parameters.url) { + parameters.url = redactUrl(parameters.url as string); + } + const result: any = { + ...getKeys(source, ["lowerClipBound", "upperClipBound"]), + chunkSource: { + parameters: (source.chunkSource as any).parameters, + spec, + }, + }; + if ( + !matrix.isIdentity(source.chunkToMultiscaleTransform, rank + 1, rank + 1) + ) { + result.chunkToMultiscaleTransform = getMatrixSnapshot( + source.chunkToMultiscaleTransform, + rank + 1, + ); + } + return result; +} + +function redactUrl(x: string | undefined): string | undefined { + if (x === undefined) return undefined; + return x.replaceAll(/(?<=http:\/\/localhost:)[0-9]+/g, "*"); +} + +export function loggingProgressListener(): ProgressListener { + return { + addSpan(span) { + console.log(`[progress] ${span.message}`); + }, + removeSpan(_span) {}, + }; +} + +export async function getDatasourceMetadata( + dataSourceProvider: Fixture, + url: string, +) { + const provider = await dataSourceProvider(); + const dataSource = await provider.get({ + url, + globalCoordinateSpace: undefined as any, + transform: undefined, + progressListener: loggingProgressListener(), + }); + return getDatasourceSnapshot(dataSource); +} diff --git a/tests/datasource/zarr.browser_test.ts b/tests/datasource/zarr.browser_test.ts new file mode 100644 index 0000000000..dbf8b6b8fb --- /dev/null +++ b/tests/datasource/zarr.browser_test.ts @@ -0,0 +1,27 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "#src/datasource/zarr/register_default.js"; +import "#src/kvstore/zip/register_frontend.js"; +import "#src/sliceview/uncompressed_chunk_format.js"; +import { datasourceMetadataSnapshotTests } from "#tests/datasource/metadata_snapshot_test_util.js"; + +datasourceMetadataSnapshotTests("zarr", [ + "zarr_v3/examples/single_res", + "ome_zarr/simple_0.4", + "ome_zarr/simple_0.4.zip", + "ome_zarr/simple_0.5", +]); diff --git a/tests/fixtures/clear_cookies.ts b/tests/fixtures/clear_cookies.ts new file mode 100644 index 0000000000..f8c05947eb --- /dev/null +++ b/tests/fixtures/clear_cookies.ts @@ -0,0 +1,31 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { afterEach } from "vitest"; +import type { Fixture } from "#tests/fixtures/fixture.js"; + +export function clearCookiesFixture(): Fixture { + afterEach(() => clearCookies()); + return async () => {}; +} + +export function clearCookies(): void { + document.cookie.split(";").forEach((cookie) => { + document.cookie = cookie + .replace(/^ +/, "") + .replace(/=.*/, `=;expires=${new Date(0).toUTCString()};path=/`); + }); +} diff --git a/tests/fixtures/datasource_provider.ts b/tests/fixtures/datasource_provider.ts new file mode 100644 index 0000000000..8d39316eda --- /dev/null +++ b/tests/fixtures/datasource_provider.ts @@ -0,0 +1,35 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getDefaultDataSourceProvider } from "#src/datasource/default_provider.js"; +import { type DataSourceRegistry } from "#src/datasource/index.js"; +import type { SharedKvStoreContext } from "#src/kvstore/frontend.js"; +import { fixture, type Fixture } from "#tests/fixtures/fixture.js"; +import { sharedKvStoreContextFixture } from "./shared_kvstore_context"; + +export function dataSourceProviderFixture( + sharedKvStoreContext: Fixture = sharedKvStoreContextFixture(), +): Fixture { + return fixture(async (stack) => { + const kvStoreContext = await sharedKvStoreContext(); + const provider = getDefaultDataSourceProvider({ + kvStoreContext, + credentialsManager: kvStoreContext.credentialsManager, + }); + stack.defer(() => provider.dispose()); + return provider; + }); +} diff --git a/tests/fixtures/fake_gcs_server.ts b/tests/fixtures/fake_gcs_server.ts new file mode 100644 index 0000000000..f225d6c6ae --- /dev/null +++ b/tests/fixtures/fake_gcs_server.ts @@ -0,0 +1,120 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { spawn } from "node:child_process"; +import readline from "node:readline"; +import { bypass, http } from "msw"; +import { beforeEach } from "vitest"; +import { fetchOk } from "#src/util/http_request.js"; +import { fixture, type Fixture } from "#tests/fixtures/fixture.js"; +import type { mswFixture } from "#tests/fixtures/msw"; + +function pickRandomPort() { + const minPort = 1024; + const maxPort = 65535; + return Math.round(Math.random() * (maxPort - minPort) + minPort); +} + +declare const FAKE_GCS_SERVER_BIN: string; + +export function fakeGcsServerFixture( + msw?: ReturnType, +): Fixture { + const gcsServer = fixture(async (stack) => { + const port = pickRandomPort(); + const proc = stack.use( + spawn( + FAKE_GCS_SERVER_BIN, + [ + "-backend", + "memory", + "-scheme", + "http", + "-host", + "localhost", + "-port", + `${port}`, + ], + { stdio: ["ignore", "ignore", "pipe"] }, + ), + ); + + const { resolve, reject, promise } = Promise.withResolvers(); + + (async () => { + for await (const line of readline.createInterface({ + input: proc.stderr, + })) { + console.log(`fake_gcs_server: ${line}`); + if (line.match(/server started at/)) { + resolve(); + } + } + reject(new Error("Failed to start server")); + })(); + await promise; + return `http://localhost:${port}`; + }); + if (msw !== undefined) { + beforeEach(async () => { + const serverUrl = await gcsServer(); + const PUBLIC_SERVER = "https://storage.googleapis.com"; + (await msw()).use( + http.all(`${PUBLIC_SERVER}/storage/*`, ({ request }) => { + const adjustedUrl = + serverUrl + request.url.substring(PUBLIC_SERVER.length); + return fetch(bypass(adjustedUrl, request)); + }), + ); + }); + } + return gcsServer; +} + +const DEFAULT_PROJECT = "myproject"; + +export async function createBucket( + gcs: Fixture, + bucket: string, + project: string = DEFAULT_PROJECT, +) { + await fetchOk( + bypass( + `${await gcs()}/storage/v1/b?project=${encodeURIComponent(project)}`, + { + method: "POST", + body: JSON.stringify({ name: bucket }), + }, + ), + ); +} + +export async function writeObject( + gcs: Fixture, + bucket: string, + path: string, + body: RequestInit["body"], +) { + await fetchOk( + bypass( + `${await gcs()}/upload/storage/v1/b/${encodeURIComponent(bucket)}/o?name=${encodeURIComponent(path)}&uploadType=media`, + { + method: "POST", + body: body, + }, + ), + ); +} diff --git a/tests/fixtures/fake_s3_server.ts b/tests/fixtures/fake_s3_server.ts new file mode 100644 index 0000000000..36c480718f --- /dev/null +++ b/tests/fixtures/fake_s3_server.ts @@ -0,0 +1,137 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import nodeHttp from "node:http"; +import nodeStream from "node:stream"; +import { bypass, http } from "msw"; +import S3rver from "s3rver"; +import { beforeEach } from "vitest"; +import { fetchOk } from "#src/util/http_request.js"; +import { fixture, type Fixture } from "#tests/fixtures/fixture.js"; +import type { mswFixture } from "#tests/fixtures/msw"; +import { tempDirectoryFixture } from "#tests/fixtures/temp_directory.js"; + +export function fakeS3ServerFixture( + options: { + directory?: Fixture; + msw?: ReturnType; + } = {}, +): Fixture { + const { directory = tempDirectoryFixture(), msw } = options; + const s3Server = fixture(async (stack) => { + const server = new S3rver({ + address: "localhost", + port: 0, + directory: await directory(), + }); + stack.defer(async () => server.close()); + const address = await server.run(); + return `http://localhost:${address.port}`; + }); + if (msw !== undefined) { + beforeEach(async () => { + const serverUrl = await s3Server(); + (await msw()).use( + http.all( + /^https:\/\/([^/]+\.)?s3\.amazonaws\.com\/.*/, + ({ request }) => { + const parsedServerUrl = new URL(serverUrl); + const requestUrl = new URL(request.url); + const requestHost = requestUrl.host; + requestUrl.protocol = parsedServerUrl.protocol; + requestUrl.host = parsedServerUrl.host; + const headers = request.headers; + headers.set("host", requestHost); + const modifiedRequest = bypass(requestUrl.toString(), request); + const { promise, resolve, reject } = + Promise.withResolvers(); + const req = nodeHttp.request(modifiedRequest.url, { + setHost: false, + signal: modifiedRequest.signal, + headers: { + ...Object.fromEntries(modifiedRequest.headers), + host: requestHost, + }, + method: modifiedRequest.method, + }); + const requestBody = modifiedRequest.body; + req.on("error", (reason) => reject(reason)); + req.on("response", (res) => { + // Convert nodeHttp.IncomingHttpHeaders to Fetch Headers object. + const headers = new Headers(); + for (let [key, value] of Object.entries(res.headers)) { + if (!Array.isArray(value)) value = [value!]; + for (const v of value) { + headers.append(key, v); + } + } + // Convert node stream.Readable to ReadableStream. + const responseBody = new ReadableStream({ + start(controller) { + res.on("data", (chunk) => { + controller.enqueue(chunk); + }); + res.on("end", () => { + controller.close(); + }); + res.on("error", (err) => { + controller.error(err); + }); + }, + }); + resolve( + new Response(responseBody, { + status: res.statusCode, + statusText: res.statusMessage, + headers, + }), + ); + }); + if (requestBody === null) { + req.end(); + } else { + nodeStream.Readable.fromWeb(requestBody as any).pipe(req); + } + return promise; + }, + ), + ); + }); + } + return s3Server; +} + +export async function createBucket(s3: Fixture, bucket: string) { + await fetchOk( + bypass(`${await s3()}/${bucket}/`, { + method: "PUT", + }), + ); +} + +export async function writeObject( + s3: Fixture, + bucket: string, + path: string, + body: RequestInit["body"], +) { + await fetchOk( + bypass(`${await s3()}/${bucket}/${encodeURIComponent(path)}`, { + method: "PUT", + body: body, + }), + ); +} diff --git a/tests/fixtures/fixture.ts b/tests/fixtures/fixture.ts new file mode 100644 index 0000000000..2471c8e91f --- /dev/null +++ b/tests/fixtures/fixture.ts @@ -0,0 +1,54 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "core-js/proposals/explicit-resource-management.js"; +import { beforeAll, afterAll } from "vitest"; + +export interface Fixture { + (): Promise; +} + +export function fixture( + setup: (disposableStack: AsyncDisposableStack) => Promise, +): Fixture { + let setupPromise: Promise | undefined; + const stack = new AsyncDisposableStack(); + + afterAll(async () => { + setupPromise = undefined; + await stack[Symbol.asyncDispose](); + }); + + const asyncGetter = () => { + if (setupPromise === undefined) { + setupPromise = (async () => { + return setup(stack); + })(); + } + return setupPromise; + }; + + beforeAll(async () => { + await asyncGetter(); + }); + + return asyncGetter; +} + +export function constantFixture(value: T): Fixture { + const promise = Promise.resolve(value); + return () => promise; +} diff --git a/tests/fixtures/gl_browser.ts b/tests/fixtures/gl_browser.ts new file mode 100644 index 0000000000..d7f208c42a --- /dev/null +++ b/tests/fixtures/gl_browser.ts @@ -0,0 +1,26 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { GL } from "#src/webgl/context.js"; +import { initializeWebGL } from "#src/webgl/context.js"; +import type { Fixture } from "#tests/fixtures/fixture.js"; +import { fixture } from "#tests/fixtures/fixture.js"; + +export function glFixture(): Fixture { + return fixture(async () => { + return initializeWebGL(document.createElement("canvas")); + }); +} diff --git a/tests/fixtures/gl_node.ts b/tests/fixtures/gl_node.ts new file mode 100644 index 0000000000..6da2ff7275 --- /dev/null +++ b/tests/fixtures/gl_node.ts @@ -0,0 +1,27 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { RefCounted } from "#src/util/disposable.js"; +import { Memoize } from "#src/util/memoize.js"; +import type { GL } from "#src/webgl/context.js"; +import type { Fixture } from "#tests/fixtures/fixture.js"; +import { fixture } from "#tests/fixtures/fixture.js"; + +export function glFixture(): Fixture { + return fixture(async () => { + return { memoize: new Memoize() } as any; + }); +} diff --git a/tests/fixtures/http_server.ts b/tests/fixtures/http_server.ts new file mode 100644 index 0000000000..800c546838 --- /dev/null +++ b/tests/fixtures/http_server.ts @@ -0,0 +1,50 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type * as http from "node:http"; +import type { AddressInfo } from "node:net"; +import { createServer } from "http-server"; +import { fixture, type Fixture } from "#tests/fixtures/fixture.js"; +import { tempDirectoryFixture } from "#tests/fixtures/temp_directory.js"; + +export function httpServerFixture( + rootDirectory: Fixture = tempDirectoryFixture(), +): { serverUrl: Fixture; rootDirectory: Fixture } { + return { + serverUrl: fixture(async (stack) => { + const resolvedRootDirectory = await rootDirectory(); + const server: http.Server = ( + createServer({ + root: resolvedRootDirectory, + cache: -1, + }) as any + ).server; + stack.defer(async () => { + await new Promise((resolve) => server.close(resolve)); + }); + const serverUrl = await new Promise((resolve, reject) => { + server.on("error", reject); + server.listen(0, "localhost", () => { + const port = (server.address() as AddressInfo).port; + resolve(`http://localhost:${port}/`); + }); + }); + console.log(`Serving ${resolvedRootDirectory} at ${serverUrl}`); + return serverUrl; + }), + rootDirectory, + }; +} diff --git a/tests/fixtures/msw_browser.ts b/tests/fixtures/msw_browser.ts new file mode 100644 index 0000000000..a282bbf53f --- /dev/null +++ b/tests/fixtures/msw_browser.ts @@ -0,0 +1,21 @@ +import type { LifeCycleEventsMap, SetupApi } from "msw"; +import { http, passthrough } from "msw"; +import { setupWorker, type SetupWorkerApi } from "msw/browser"; +import { afterEach } from "vitest"; +import type { Fixture } from "#tests/fixtures/fixture.js"; +import { fixture } from "#tests/fixtures/fixture.js"; + +export function mswFixture(): Fixture> { + const mswServer = fixture(async (stack) => { + const server = setupWorker( + http.get("/*", () => passthrough()), + ) as SetupWorkerApi; + stack.defer(() => server.stop()); + await server.start(); + return server; + }); + + afterEach(async () => (await mswServer()).resetHandlers()); + + return mswServer; +} diff --git a/tests/fixtures/msw_node.ts b/tests/fixtures/msw_node.ts new file mode 100644 index 0000000000..a3749e6222 --- /dev/null +++ b/tests/fixtures/msw_node.ts @@ -0,0 +1,18 @@ +import type { LifeCycleEventsMap, SetupApi } from "msw"; +import { setupServer } from "msw/node"; +import { afterEach } from "vitest"; +import type { Fixture } from "#tests/fixtures/fixture.js"; +import { fixture } from "#tests/fixtures/fixture.js"; + +export function mswFixture(): Fixture> { + const mswServer = fixture(async (stack) => { + const server = setupServer(); + stack.defer(() => server.close()); + server.listen(); + return server; + }); + + afterEach(async () => (await mswServer()).resetHandlers()); + + return mswServer; +} diff --git a/tests/fixtures/shared_kvstore_context.ts b/tests/fixtures/shared_kvstore_context.ts new file mode 100644 index 0000000000..7fa6538db0 --- /dev/null +++ b/tests/fixtures/shared_kvstore_context.ts @@ -0,0 +1,50 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getDefaultCredentialsManager } from "#src/credentials_provider/default_manager.js"; +import { SharedCredentialsManager } from "#src/credentials_provider/shared.js"; +import { DataManagementContext } from "#src/data_management_context.js"; +import { SharedKvStoreContext } from "#src/kvstore/frontend.js"; +import type { GL } from "#src/webgl/context.js"; +import { fixture, type Fixture } from "#tests/fixtures/fixture.js"; +import { glFixture } from "#tests/fixtures/gl"; + +export function sharedKvStoreContextFixture( + gl: Fixture = glFixture(), +): Fixture { + return fixture(async (stack) => { + const dataContext = new DataManagementContext( + /*gl=*/ await gl(), + /*frameNumberCounter=*/ undefined as any, + ); + stack.defer(() => dataContext.dispose()); + + const sharedCredentialsManager = dataContext.registerDisposer( + new SharedCredentialsManager( + getDefaultCredentialsManager(), + dataContext.rpc, + ), + ); + + const sharedKvStoreContext = new SharedKvStoreContext( + dataContext.chunkManager, + sharedCredentialsManager, + ); + stack.defer(() => sharedKvStoreContext.dispose()); + + return sharedKvStoreContext; + }); +} diff --git a/tests/fixtures/status_message_observer.ts b/tests/fixtures/status_message_observer.ts new file mode 100644 index 0000000000..78476b70ec --- /dev/null +++ b/tests/fixtures/status_message_observer.ts @@ -0,0 +1,89 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { afterEach } from "vitest"; +import type { StatusMessage } from "#src/status.js"; +import { statusMessages, getStatusMessageContainers } from "#src/status.js"; +import type { Fixture } from "#tests/fixtures/fixture.js"; +import { fixture } from "#tests/fixtures/fixture.js"; + +export interface StatusMessageHandler { + (message: StatusMessage): void; +} +export class StatusMessageObserver { + private handlers: Set = new Set(); + private observer: MutationObserver; + constructor() { + this.observer = new MutationObserver(() => { + for (const handler of this.handlers) { + for (const message of statusMessages) { + handler(message); + } + } + }); + for (const element of getStatusMessageContainers()) { + this.observer.observe(element, { + subtree: true, + childList: true, + attributes: true, + characterData: true, + }); + } + } + registerHandler(handler: StatusMessageHandler): Disposable { + const wrappedHandler: StatusMessageHandler = (status) => handler(status); + this.handlers.add(wrappedHandler); + return { + [Symbol.dispose]: () => { + this.handlers.delete(wrappedHandler); + }, + }; + } + async waitForButton(pattern: RegExp): Promise { + const { promise, resolve } = Promise.withResolvers(); + using _handler = this.registerHandler((status) => { + const result = document.evaluate( + `.//button`, + status.element, + null, + XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, + null, + ); + for (let i = 0, length = result.snapshotLength; i < length; ++i) { + const button = result.snapshotItem(i) as HTMLButtonElement; + if ((button.textContent ?? "").match(pattern)) { + resolve(button); + } + } + }); + return await promise; + } + reset() { + this.handlers.clear(); + } + [Symbol.dispose]() { + this.observer.disconnect(); + } +} + +export function statusMessageObserverFixture(): Fixture { + const f = fixture(async () => new StatusMessageObserver()); + afterEach(async () => { + const handler = await f(); + handler.reset(); + }); + return f; +} diff --git a/tests/fixtures/temp_directory.ts b/tests/fixtures/temp_directory.ts new file mode 100644 index 0000000000..248ab191e7 --- /dev/null +++ b/tests/fixtures/temp_directory.ts @@ -0,0 +1,34 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Vitest fixture that creates a temporary directory. + +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { type Fixture, fixture } from "#tests/fixtures/fixture.js"; + +export function tempDirectoryFixture(prefix: string = ""): Fixture { + return fixture(async (stack) => { + const tempDir = await fs.mkdtemp( + `${os.tmpdir()}${path.sep}neuroglancer-vitest-${prefix}`, + ); + stack.defer(async () => { + await fs.rm(tempDir, { recursive: true, force: true }); + }); + return tempDir; + }); +} diff --git a/tests/kvstore/gcs.spec.ts b/tests/kvstore/gcs.spec.ts new file mode 100644 index 0000000000..5bdfa62fe5 --- /dev/null +++ b/tests/kvstore/gcs.spec.ts @@ -0,0 +1,47 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "#src/kvstore/gcs/register.js"; +import { beforeAll } from "vitest"; +import { + createBucket, + fakeGcsServerFixture, + writeObject, +} from "#tests/fixtures/fake_gcs_server.js"; +import { constantFixture } from "#tests/fixtures/fixture.js"; +import { mswFixture } from "#tests/fixtures/msw"; +import { getTestFiles } from "#tests/kvstore/test_data.js"; +import { testKvStore } from "#tests/kvstore/test_util.js"; + +const msw = mswFixture(); +const fakeGcsServer = fakeGcsServerFixture(msw); + +const BUCKET = "mybucket"; + +beforeAll(async () => { + // Add data to GCS. + await createBucket(fakeGcsServer, BUCKET); + for (const [relativePath, content] of await getTestFiles()) { + await writeObject( + fakeGcsServer, + BUCKET, + relativePath, + new Uint8Array(content), + ); + } +}); + +testKvStore(constantFixture(`gs://${BUCKET}/`)); diff --git a/tests/kvstore/gzip.spec.ts b/tests/kvstore/gzip.spec.ts new file mode 100644 index 0000000000..1f23d1f974 --- /dev/null +++ b/tests/kvstore/gzip.spec.ts @@ -0,0 +1,36 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "#src/kvstore/http/register.js"; +import "#src/kvstore/gzip/register.js"; +import { expect, test } from "vitest"; +import { constantFixture } from "#tests/fixtures/fixture.js"; +import { httpServerFixture } from "#tests/fixtures/http_server.js"; +import { sharedKvStoreContextFixture } from "#tests/fixtures/shared_kvstore_context.js"; +import { TEST_DATA_DIR } from "#tests/kvstore/test_data.js"; + +const serverFixture = httpServerFixture(constantFixture(TEST_DATA_DIR)); +const sharedKvStoreContext = sharedKvStoreContextFixture(); + +test("can read", async () => { + const { response } = await ( + await sharedKvStoreContext() + ).kvStoreContext.read( + `${await serverFixture.serverUrl()}gzip/simple.txt.gz|gzip`, + { throwIfMissing: true }, + ); + expect(await response.text()).toEqual("Hello"); +}); diff --git a/tests/kvstore/http.spec.ts b/tests/kvstore/http.spec.ts new file mode 100644 index 0000000000..9f5d862481 --- /dev/null +++ b/tests/kvstore/http.spec.ts @@ -0,0 +1,25 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "#src/kvstore/http/register.js"; +import { constantFixture } from "#tests/fixtures/fixture.js"; +import { httpServerFixture } from "#tests/fixtures/http_server.js"; +import { TEST_FILES_DIR } from "#tests/kvstore/test_data.js"; +import { testKvStore } from "#tests/kvstore/test_util.js"; + +const serverFixture = httpServerFixture(constantFixture(TEST_FILES_DIR)); + +testKvStore(serverFixture.serverUrl); diff --git a/tests/kvstore/ngauth.browser_test.ts b/tests/kvstore/ngauth.browser_test.ts new file mode 100644 index 0000000000..152250e233 --- /dev/null +++ b/tests/kvstore/ngauth.browser_test.ts @@ -0,0 +1,142 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "#src/kvstore/ngauth/register.js"; +import "#src/kvstore/ngauth/register_credentials_provider.js"; +import { http, passthrough } from "msw"; +import { expect, test, afterEach } from "vitest"; +import { mswFixture } from "#tests/fixtures/msw"; +import { sharedKvStoreContextFixture } from "#tests/fixtures/shared_kvstore_context.js"; +import { statusMessageObserverFixture } from "#tests/fixtures/status_message_observer.js"; +import { clearCookies } from "#tests/util/clear_cookies.js"; +import { mswRequestLog } from "#tests/util/msw_request_log.js"; + +const msw = mswFixture(); + +const statusMessageObserver = statusMessageObserverFixture(); + +declare const FAKE_NGAUTH_SERVER: string; + +const BUCKET = "mybucket"; + +const sharedKvStoreContext = sharedKvStoreContextFixture(); +const sharedKvStoreContext2 = sharedKvStoreContextFixture(); +afterEach(() => { + clearCookies(); +}); + +test("login", async () => { + using requestLog = mswRequestLog(await msw(), { + redact: ["(?<=neuroglancer=)[a-f0-9]+"], + }); + (await msw()).use( + http.get( + `https://storage.googleapis.com/storage/v1/b/mybucket/o/missing`, + async () => { + return new Response(null, { status: 404 }); + }, + ), + http.all(`${FAKE_NGAUTH_SERVER}/*`, () => passthrough()), + ); + { + const readPromise = (await sharedKvStoreContext()).kvStoreContext.read( + `gs+ngauth+${FAKE_NGAUTH_SERVER}/${BUCKET}/missing`, + ); + console.log("Waiting for login"); + const loginButton = await ( + await statusMessageObserver() + ).waitForButton(/\blogin\b/); + console.log("Clicking login"); + loginButton.click(); + console.log("Waiting for read to complete"); + expect(await readPromise).toBe(undefined); + expect.soft(await requestLog.popAll()).toMatchInlineSnapshot(` + [ + { + "request": { + "url": "http://localhost:*/token", + }, + "response": { + "status": 401, + }, + }, + { + "request": { + "body": "{"token":"fake_token","bucket":"mybucket"}", + "url": "http://localhost:*/gcs_token", + }, + "response": { + "body": "{"token":"fake_gcs_token:mybucket"}", + "status": 200, + }, + }, + { + "request": { + "headers": [ + "authorization: Bearer fake_gcs_token:mybucket", + ], + "url": "https://storage.googleapis.com/storage/v1/b/mybucket/o/missing?alt=media&neuroglancer=*", + }, + "response": { + "status": 404, + }, + }, + ] + `); + } + + // Now that cookies has been set, login is not required. + { + const readPromise = (await sharedKvStoreContext2()).kvStoreContext.read( + `gs+ngauth+${FAKE_NGAUTH_SERVER}/${BUCKET}/missing`, + ); + expect(await readPromise).toBe(undefined); + expect(await requestLog.popAll()).toMatchInlineSnapshot(` + [ + { + "request": { + "url": "http://localhost:*/token", + }, + "response": { + "body": "fake_token", + "status": 200, + }, + }, + { + "request": { + "body": "{"token":"fake_token","bucket":"mybucket"}", + "url": "http://localhost:*/gcs_token", + }, + "response": { + "body": "{"token":"fake_gcs_token:mybucket"}", + "status": 200, + }, + }, + { + "request": { + "headers": [ + "authorization: Bearer fake_gcs_token:mybucket", + ], + "url": "https://storage.googleapis.com/storage/v1/b/mybucket/o/missing?alt=media&neuroglancer=*", + }, + "response": { + "status": 404, + }, + }, + ] + `); + } +}); diff --git a/tests/kvstore/s3.spec.ts b/tests/kvstore/s3.spec.ts new file mode 100644 index 0000000000..2f64b292db --- /dev/null +++ b/tests/kvstore/s3.spec.ts @@ -0,0 +1,47 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "#src/kvstore/s3/register.js"; +import { beforeAll } from "vitest"; +import { + createBucket, + fakeS3ServerFixture, + writeObject, +} from "#tests/fixtures/fake_s3_server.js"; +import { constantFixture } from "#tests/fixtures/fixture.js"; +import { mswFixture } from "#tests/fixtures/msw"; +import { getTestFiles } from "#tests/kvstore/test_data.js"; +import { testKvStore } from "#tests/kvstore/test_util.js"; + +const msw = mswFixture(); +const fakeS3Server = fakeS3ServerFixture({ msw }); + +const BUCKET = "mybucket"; + +beforeAll(async () => { + // Add data to S3. + await createBucket(fakeS3Server, BUCKET); + for (const [relativePath, content] of await getTestFiles()) { + await writeObject( + fakeS3Server, + BUCKET, + relativePath, + new Uint8Array(content), + ); + } +}); + +testKvStore(constantFixture(`s3://${BUCKET}/`)); diff --git a/tests/kvstore/test_data.ts b/tests/kvstore/test_data.ts new file mode 100644 index 0000000000..89f87f30e7 --- /dev/null +++ b/tests/kvstore/test_data.ts @@ -0,0 +1,56 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from "node:fs/promises"; +import path from "node:path"; + +export const TEST_DATA_DIR = path.join( + import.meta.dirname, + "..", + "..", + "testdata", + "kvstore", +); +export const TEST_FILES_DIR = path.join(TEST_DATA_DIR, "files"); + +export async function findFilesRecursively(rootDir: string): Promise { + const relativePaths: string[] = []; + for (const entry of await fs.readdir(rootDir, { + recursive: true, + withFileTypes: true, + })) { + if (!entry.isFile()) { + continue; + } + const fullPath = path.join(entry.parentPath, entry.name); + const relativePath = path + .relative(rootDir, fullPath) + .replaceAll(path.sep, "/"); + relativePaths.push(relativePath); + } + return relativePaths; +} + +export async function getTestFiles( + rootDir: string = TEST_FILES_DIR, +): Promise> { + const map = new Map(); + for (const relativePath of await findFilesRecursively(rootDir)) { + const content = await fs.readFile(path.join(rootDir, relativePath)); + map.set(relativePath, content); + } + return map; +} diff --git a/tests/kvstore/test_util.ts b/tests/kvstore/test_util.ts new file mode 100644 index 0000000000..14a3cb71a7 --- /dev/null +++ b/tests/kvstore/test_util.ts @@ -0,0 +1,213 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, expect, test } from "vitest"; +import type { KvStore } from "#src/kvstore/index.js"; +import { listKvStoreRecursively, readKvStore } from "#src/kvstore/index.js"; +import type { Fixture } from "#tests/fixtures/fixture.js"; +import { sharedKvStoreContextFixture } from "#tests/fixtures/shared_kvstore_context.js"; + +export const sharedKvStoreContext = sharedKvStoreContextFixture(); + +export function testRead(url: Fixture) { + test("read not found", async () => { + expect( + await ( + await sharedKvStoreContext() + ).kvStoreContext.read(`${await url()}missing`), + ).toBe(undefined); + }); + test("read full", async () => { + const response = await ( + await sharedKvStoreContext() + ).kvStoreContext.read(`${await url()}a`); + expect(response).toBeTruthy(); + expect.soft(response!.totalSize).toEqual(3); + expect.soft(response!.offset).toEqual(0); + expect.soft(response!.length).toEqual(3); + expect.soft(await response!.response.text()).toEqual("abc"); + }); + test("read byte range zero length", async () => { + const response = await ( + await sharedKvStoreContext() + ).kvStoreContext.read(`${await url()}a`, { + byteRange: { offset: 1, length: 0 }, + }); + expect(response).toBeTruthy(); + expect.soft(response!.totalSize).toEqual(3); + expect.soft(response!.offset).toEqual(1); + expect.soft(response!.length).toEqual(0); + expect.soft(await response!.response.text()).toEqual(""); + }); + test("read byte range zero length empty file", async () => { + const response = await ( + await sharedKvStoreContext() + ).kvStoreContext.read(`${await url()}empty`, { + byteRange: { offset: 0, length: 0 }, + }); + expect(response).toBeTruthy(); + expect.soft(response!.totalSize).toEqual(0); + expect.soft(response!.offset).toEqual(0); + expect.soft(response!.length).toEqual(0); + expect.soft(await response!.response.text()).toEqual(""); + }); + test("read byte range offset+length", async () => { + const response = await ( + await sharedKvStoreContext() + ).kvStoreContext.read(`${await url()}a`, { + byteRange: { offset: 1, length: 1 }, + }); + expect(response).toBeTruthy(); + expect.soft(response!.totalSize).toEqual(3); + expect.soft(response!.offset).toEqual(1); + expect.soft(response!.length).toEqual(1); + expect.soft(await response!.response.text()).toEqual("b"); + }); + test("read byte range suffixLength", async () => { + const response = await ( + await sharedKvStoreContext() + ).kvStoreContext.read(`${await url()}a`, { + byteRange: { suffixLength: 1 }, + }); + expect(response).toBeTruthy(); + expect.soft(response!.totalSize).toEqual(3); + expect.soft(response!.offset).toEqual(2); + expect.soft(response!.length).toEqual(1); + expect.soft(await response!.response.text()).toEqual("c"); + }); + test("read byte range suffixLength=0", async () => { + const response = await ( + await sharedKvStoreContext() + ).kvStoreContext.read(`${await url()}a`, { + byteRange: { suffixLength: 0 }, + }); + expect(response).toBeTruthy(); + expect.soft(response!.totalSize).toEqual(3); + expect.soft(response!.offset).toEqual(3); + expect.soft(response!.length).toEqual(0); + expect.soft(await response!.response.text()).toEqual(""); + }); + test("stat on directory returns not found", async () => { + const response = await ( + await sharedKvStoreContext() + ).kvStoreContext.stat(`${await url()}baz`); + expect(response).toEqual(undefined); + }); + test("read on directory returns not found", async () => { + const response = await ( + await sharedKvStoreContext() + ).kvStoreContext.read(`${await url()}baz`); + expect(response).toEqual(undefined); + }); +} + +export function testList(url: Fixture) { + test("list with empty prefix", async () => { + expect( + await ( + await sharedKvStoreContext() + ).kvStoreContext.list(await url(), { + responseKeys: "suffix", + }), + ).toMatchInlineSnapshot(` + { + "directories": [ + "baz", + ], + "entries": [ + { + "key": "#", + }, + { + "key": "a", + }, + { + "key": "b", + }, + { + "key": "c", + }, + { + "key": "empty", + }, + ], + } + `); + }); + + test("list with file prefix", async () => { + expect( + await ( + await sharedKvStoreContext() + ).kvStoreContext.list(`${await url()}e`, { + responseKeys: "suffix", + }), + ).toMatchInlineSnapshot(` + { + "directories": [], + "entries": [ + { + "key": "mpty", + }, + ], + } + `); + }); + + test("list with directory prefix", async () => { + expect( + await ( + await sharedKvStoreContext() + ).kvStoreContext.list(`${await url()}baz/`, { + responseKeys: "suffix", + }), + ).toMatchInlineSnapshot(` + { + "directories": [], + "entries": [ + { + "key": "x", + }, + ], + } + `); + }); +} + +export function testKvStore(url: Fixture) { + describe("kvstore", () => { + testRead(url); + testList(url); + }); +} + +export async function readAllFromKvStore( + kvStore: KvStore, + prefix: string, +): Promise> { + const keys = await listKvStoreRecursively(kvStore, prefix, { + responseKeys: "suffix", + }); + const values = await Promise.all( + keys.map(async ({ key }) => { + const readResponse = await readKvStore(kvStore, prefix + key, { + throwIfMissing: true, + }); + return Buffer.from(await readResponse.response.arrayBuffer()); + }), + ); + return new Map(Array.from(keys, ({ key }, i) => [key, values[i]])); +} diff --git a/tests/kvstore/url.spec.ts b/tests/kvstore/url.spec.ts new file mode 100644 index 0000000000..db1981963f --- /dev/null +++ b/tests/kvstore/url.spec.ts @@ -0,0 +1,182 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, expect, test } from "vitest"; +import { + finalPipelineUrlComponent, + kvstoreEnsureDirectoryPipelineUrl, + parsePipelineUrlComponent, + parseUrlSuffix, + pipelineUrlJoin, +} from "#src/kvstore/url.js"; + +describe("kvstoreEnsureDirectoryPipelineUrl", () => { + test("single pipeline component", () => { + expect(kvstoreEnsureDirectoryPipelineUrl("http://foo")).toEqual( + "http://foo/", + ); + expect(kvstoreEnsureDirectoryPipelineUrl("http://foo/")).toEqual( + "http://foo/", + ); + }); + + test("with query parameters", () => { + expect(kvstoreEnsureDirectoryPipelineUrl("http://foo?a=b")).toEqual( + "http://foo/?a=b", + ); + }); + + test("with fragment parameters", () => { + expect(kvstoreEnsureDirectoryPipelineUrl("http://foo#a=b")).toEqual( + "http://foo/#a=b", + ); + }); + + test("multiple pipeline component", () => { + expect(kvstoreEnsureDirectoryPipelineUrl("http://foo|zarr")).toEqual( + "http://foo|zarr:", + ); + expect(kvstoreEnsureDirectoryPipelineUrl("http://foo|zarr:")).toEqual( + "http://foo|zarr:", + ); + }); +}); + +describe("pipelineUrlJoin", () => { + test("simple", () => { + expect(pipelineUrlJoin("gs://foo", "a", "b")).toEqual("gs://foo/a/b"); + }); + test("query parameter", () => { + expect(pipelineUrlJoin("gs://foo?a=b", "a", "b")).toEqual( + "gs://foo/a/b?a=b", + ); + expect(pipelineUrlJoin("gs://foo?a=b|zarr", "a", "b")).toEqual( + "gs://foo?a=b|zarr:a/b", + ); + }); +}); + +describe("finalPipelineUrlComponent", () => { + test("single component", () => { + expect(finalPipelineUrlComponent("")).toEqual(""); + expect(finalPipelineUrlComponent("gs://a")).toEqual("gs://a"); + }); + test("multiple components", () => { + expect(finalPipelineUrlComponent("gs://a|zarr")).toEqual("zarr"); + expect(finalPipelineUrlComponent("gs://a|zip:foo|zarr:")).toEqual("zarr:"); + }); +}); + +describe("parsePipelineUrlComponent", () => { + test("scheme only", () => { + expect(parsePipelineUrlComponent("zarr")).toEqual({ + scheme: "zarr", + suffix: undefined, + url: "zarr", + }); + }); + test("empty suffix", () => { + expect(parsePipelineUrlComponent("zarr:")).toEqual({ + scheme: "zarr", + suffix: "", + url: "zarr:", + }); + }); +}); + +describe("parseUrlSuffix", () => { + test("no suffix", () => { + expect(parseUrlSuffix(undefined)).toMatchInlineSnapshot(` + { + "authorityAndPath": undefined, + "fragment": undefined, + "query": undefined, + } + `); + }); + test("empty suffix", () => { + expect(parseUrlSuffix("")).toMatchInlineSnapshot(` + { + "authorityAndPath": "", + "fragment": undefined, + "query": undefined, + } + `); + }); + test("path only", () => { + expect(parseUrlSuffix("a/b/")).toMatchInlineSnapshot(` + { + "authorityAndPath": "a/b/", + "fragment": undefined, + "query": undefined, + } + `); + }); + test("query only", () => { + expect(parseUrlSuffix("?query")).toMatchInlineSnapshot(` + { + "authorityAndPath": "", + "fragment": undefined, + "query": "query", + } + `); + }); + test("fragment only", () => { + expect(parseUrlSuffix("#fragment")).toMatchInlineSnapshot(` + { + "authorityAndPath": "", + "fragment": "fragment", + "query": undefined, + } + `); + }); + test("path and query", () => { + expect(parseUrlSuffix("//host/path?query")).toMatchInlineSnapshot(` + { + "authorityAndPath": "//host/path", + "fragment": undefined, + "query": "query", + } + `); + }); + test("path and fragment", () => { + expect(parseUrlSuffix("//host/path#fragment")).toMatchInlineSnapshot(` + { + "authorityAndPath": "//host/path", + "fragment": "fragment", + "query": undefined, + } + `); + }); + test("path and query and fragment", () => { + expect(parseUrlSuffix("//host/path?query#fragment")).toMatchInlineSnapshot(` + { + "authorityAndPath": "//host/path", + "fragment": "fragment", + "query": "query", + } + `); + }); + test("path and fragment with fake query", () => { + expect(parseUrlSuffix("//host/path#fragment?query")).toMatchInlineSnapshot(` + { + "authorityAndPath": "//host/path", + "fragment": "fragment?query", + "query": undefined, + } + `); + }); +}); diff --git a/tests/kvstore/zip.spec.ts b/tests/kvstore/zip.spec.ts new file mode 100644 index 0000000000..ad20d07e10 --- /dev/null +++ b/tests/kvstore/zip.spec.ts @@ -0,0 +1,177 @@ +/** + * @license + * Copyright 2025 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "#src/kvstore/http/register.js"; +import "#src/kvstore/zip/register_backend.js"; +import "#src/kvstore/zip/register_frontend.js"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import yauzl from "yauzl"; +import { getKvStoreCompletions } from "#src/datasource/kvstore_completions.js"; +import { constantFixture } from "#tests/fixtures/fixture.js"; +import { httpServerFixture } from "#tests/fixtures/http_server.js"; +import { TEST_DATA_DIR } from "#tests/kvstore/test_data.js"; +import { + testKvStore, + sharedKvStoreContext, + readAllFromKvStore, +} from "#tests/kvstore/test_util.js"; + +const serverFixture = httpServerFixture(constantFixture(TEST_DATA_DIR)); + +function readAllUsingYauzl(zipPath: string) { + return new Promise((resolve, reject) => { + const map = new Map(); + yauzl.open(zipPath, { lazyEntries: true }, function (err, zipfile) { + if (err) { + reject(err); + return; + } + zipfile.on("entry", async (entry) => { + if (/\/$/.test(entry.fileName)) { + zipfile.readEntry(); + return; + } + zipfile.openReadStream(entry, async (err, readStream) => { + if (err) { + reject(err); + return; + } + const parts: Buffer[] = []; + readStream.on("data", (chunk) => { + parts.push(chunk); + }); + readStream.on("end", () => { + map.set(entry.fileName, Buffer.concat(parts)); + zipfile.readEntry(); + }); + }); + }); + zipfile.on("end", () => { + zipfile.close(); + resolve(map); + }); + zipfile.readEntry(); + }); + }); +} + +describe("yauzl success cases", async () => { + const relativePath = "zip/from-yauzl/success"; + const zipFiles = await fs.readdir(path.join(TEST_DATA_DIR, relativePath)); + test.for(zipFiles)("%s", async (zipFile) => { + const url = + (await serverFixture.serverUrl()) + `${relativePath}/${zipFile}|zip:`; + const { kvStoreContext } = await sharedKvStoreContext(); + const kvStore = kvStoreContext.getKvStore(url); + const contentFromZip = await readAllFromKvStore( + kvStore.store, + kvStore.path, + ); + const expectedFiles = await readAllUsingYauzl( + path.join(TEST_DATA_DIR, relativePath, zipFile), + ); + expect(contentFromZip).toEqual(expectedFiles); + }); +}); + +describe("yauzl failure cases", async () => { + const relativePath = "zip/from-yauzl/failure"; + const zipFiles = await fs.readdir(path.join(TEST_DATA_DIR, relativePath)); + test.for(zipFiles)("%s", async (zipFile) => { + const url = + (await serverFixture.serverUrl()) + `${relativePath}/${zipFile}|zip:`; + const { kvStoreContext } = await sharedKvStoreContext(); + const kvStore = kvStoreContext.getKvStore(url); + const expectedError = new RegExp( + zipFile + .replace(/(_[0-9]+)?\.zip$/, "") + .split(/\s+/) + .join(".*"), + "i", + ); + await expect( + readAllFromKvStore(kvStore.store, kvStore.path).then(() => null), + ).rejects.toThrowError(expectedError); + }); +}); + +testKvStore( + async () => (await serverFixture.serverUrl()) + "zip/files.zip|zip:", +); + +describe("completion", () => { + test("empty prefix", async () => { + const url = (await serverFixture.serverUrl()) + "zip/files.zip|zip:"; + const completions = await getKvStoreCompletions( + await sharedKvStoreContext(), + { + url, + }, + ); + expect(completions).toMatchInlineSnapshot(` + { + "completions": [ + { + "value": "baz/", + }, + { + "value": "%23|", + }, + { + "value": "a|", + }, + { + "value": "b|", + }, + { + "value": "c|", + }, + { + "value": "empty|", + }, + ], + "defaultCompletion": undefined, + "offset": 41, + } + `); + }); + + test("single letter prefix", async () => { + const url = (await serverFixture.serverUrl()) + "zip/files.zip|zip:b"; + const completions = await getKvStoreCompletions( + await sharedKvStoreContext(), + { + url, + }, + ); + expect(completions).toMatchInlineSnapshot(` + { + "completions": [ + { + "value": "baz/", + }, + { + "value": "b|", + }, + ], + "defaultCompletion": undefined, + "offset": 41, + } + `); + }); +}); diff --git a/tests/util/clear_cookies.ts b/tests/util/clear_cookies.ts new file mode 100644 index 0000000000..54ab46b3dd --- /dev/null +++ b/tests/util/clear_cookies.ts @@ -0,0 +1,26 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Clears all cookies that are accessible to JavaScript. HTTP-only cookies +// cannot be cleared. +export function clearCookies(): void { + for (const cookie of document.cookie.split(";")) { + document.cookie = cookie + .replace(/^ +/, "") + .replace(/=.*/, `=;expires=${new Date(0).toUTCString()};path=/`); + } + window.localStorage.clear(); +} diff --git a/tests/util/msw_request_log.ts b/tests/util/msw_request_log.ts new file mode 100644 index 0000000000..a8e4982888 --- /dev/null +++ b/tests/util/msw_request_log.ts @@ -0,0 +1,128 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type LifeCycleEventsMap, type SetupApi } from "msw"; + +interface RequestLogEntry { + request: { + url: string; + headers?: string[]; + body?: string; + }; + response: { + status: number; + body?: string; + }; +} + +export function mswRequestLog( + msw: SetupApi, + options: { + redact?: string[]; + } = {}, +) { + const redactPatterns = [ + "(?<=http://localhost:)[0-9]+", + ...(options.redact ?? []), + ]; + const redactRegexp = new RegExp( + redactPatterns.map((s) => `(?:${s})`).join("|"), + "g", + ); + const redact = (s: string) => s.replaceAll(redactRegexp, "*"); + const getHeaders = (headers: Headers): { headers?: string[] } => { + const list: string[] = []; + for (const [name, value] of headers.entries()) { + if (name === "accept" || name === "content-type") { + continue; + } + list.push(redact(`${name}: ${value}`)); + } + if (list.length === 0) return {}; + return { headers: list }; + }; + const getBody = async (r: Request | Response): Promise<{ body?: string }> => { + const body = redact(await r.text()); + if (!body) return {}; + return { body }; + }; + const log: Promise[] = []; + const outstandingRequests = new Map< + string, + { resolve: (value: RequestLogEntry) => void } + >(); + const handler = async ({ + request, + response, + requestId, + }: { + request: Request; + response: Response; + requestId: string; + }) => { + const resolvers = outstandingRequests.get(requestId); + if (resolvers === undefined) return; + outstandingRequests.delete(requestId); + request = request.clone(); + response = response.clone(); + const entry: RequestLogEntry = { + request: { + url: redact(request.url), + ...getHeaders(request.headers), + ...(await getBody(request)), + }, + response: { + status: response.status, + ...(await getBody(response)), + }, + }; + resolvers.resolve(entry); + }; + const requestHandler = ({ + request, + requestId, + }: { + request: Request; + requestId: string; + }) => { + if (new URL(request.url).origin === window.origin) { + return; + } + const { promise, resolve } = Promise.withResolvers(); + outstandingRequests.set(requestId, { resolve }); + log.push(promise); + }; + // request:start event is always sequenced before the `fetch` resolves, but + // `response:*` events sometimes don't. To avoid missing events, log entries + // are added on `response:start` and resolved by the corresponding + // `response:*` event. + msw.events.on("request:start", requestHandler); + msw.events.on("response:mocked", handler); + msw.events.on("response:bypass", handler); + return { + [Symbol.dispose]: () => { + msw.events.removeListener("request:start", requestHandler); + msw.events.removeListener("response:mocked", handler); + msw.events.removeListener("response:bypass", handler); + }, + log, + popAll: () => { + const result = log.slice(); + log.length = 0; + return Promise.all(result); + }, + }; +} diff --git a/tsconfig.json b/tsconfig.json index 172a9e26e7..e2185ad26e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -34,6 +34,5 @@ "testdata", "examples", "config", - "tests", ], } diff --git a/vitest.workspace.ts b/vitest.workspace.ts index 00f4e15afb..8b489d1ca7 100644 --- a/vitest.workspace.ts +++ b/vitest.workspace.ts @@ -1,29 +1,60 @@ +import path from "node:path"; +import mswPlugin from "@iodigital/vite-plugin-msw"; import { defineWorkspace } from "vitest/config"; +import { getFakeGcsServerBin } from "build_tools/vitest/build_fake_gcs_server.ts"; +import { startFakeNgauthServer } from "./build_tools/vitest/fake_ngauth_server.ts"; +import { startTestDataServer } from "./build_tools/vitest/test_data_server.ts"; + +const fakeNgauthServer = await startFakeNgauthServer(); +const testDataServer = await startTestDataServer( + path.join(import.meta.dirname, "testdata"), +); +const fakeGcsServerBin = await getFakeGcsServerBin(); export default defineWorkspace([ { test: { name: "node", - environment: "node", - setupFiles: ["./build_tools/vitest/setup-crypto.ts"], - include: ["src/**/*.spec.ts"], + environment: "jsdom", + setupFiles: [ + "./build_tools/vitest/polyfill-browser-globals-in-node.ts", + "@vitest/web-worker", + ], + include: ["src/**/*.spec.ts", "tests/**/*.spec.ts"], benchmark: { include: ["src/**/*.benchmark.ts"], }, + testTimeout: 10000, + }, + define: { + FAKE_GCS_SERVER_BIN: JSON.stringify(fakeGcsServerBin), }, }, { + define: { + FAKE_NGAUTH_SERVER: JSON.stringify(fakeNgauthServer.url), + TEST_DATA_SERVER: JSON.stringify(testDataServer.url), + }, + esbuild: { + target: "es2022", + }, + plugins: [mswPlugin({ mode: "browser", handlers: [] })], + optimizeDeps: { + include: ["nifti-reader-js"], + entries: ["src/util/polyfills.ts"], + }, test: { name: "browser", - include: ["src/**/*.browser_test.ts"], + include: ["src/**/*.browser_test.ts", "tests/**/*.browser_test.ts"], benchmark: { include: [], }, browser: { - provider: "webdriverio", + provider: "playwright", enabled: true, headless: true, - name: "chrome", + instances: [{ browser: "chromium" }], + screenshotFailures: false, }, }, },