From ccd67312b96c70fc1022cdcc6b8a22898e6055d3 Mon Sep 17 00:00:00 2001 From: Naing Linn Khant Date: Fri, 29 Aug 2025 23:34:10 +0700 Subject: [PATCH 1/6] feat: update max of concurrent uploads/deletion to 200 and update default to 100 --- packages/cli/README.md | 6 +++--- packages/cli/src/commands/vector-store/sync.ts | 6 +++--- packages/cli/src/commands/vector-store/upload.ts | 6 +++--- packages/cli/src/utils/config.ts | 3 ++- packages/cli/src/utils/manifest.ts | 2 +- packages/cli/src/utils/sync.ts | 2 +- packages/cli/tests/commands/config/get.test.ts | 2 +- packages/cli/tests/commands/config/set.test.ts | 4 ++-- packages/cli/tests/commands/vector-store/upload.test.ts | 4 ++-- packages/cli/tests/helpers/test-utils.ts | 2 +- packages/cli/tests/utils/config.test.ts | 2 +- 11 files changed, 20 insertions(+), 19 deletions(-) diff --git a/packages/cli/README.md b/packages/cli/README.md index ea8b52a..3fe96e9 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -66,7 +66,7 @@ mxbai vs upload "My Documents" --manifest upload-manifest.yaml ### File Management - `mxbai vs upload ` - Upload files to vector store - - Options: `--strategy fast|high_quality`, `--contextualization`, `--metadata `, `--dry-run`, `--parallel ` (1-20), `--unique`, `--manifest ` + - Options: `--strategy fast|high_quality`, `--contextualization`, `--metadata `, `--dry-run`, `--parallel ` (1-200), `--unique`, `--manifest ` - `mxbai vs files list ` - List files in vector store (alias: `ls`) - Options: `--status ` (pending|in_progress|cancelled|completed|failed), `--limit ` - `mxbai vs files get ` - Get file details @@ -83,7 +83,7 @@ mxbai vs upload "My Documents" --manifest upload-manifest.yaml ### Advanced Features - `mxbai vs sync ` - Sync files with intelligent change detection - - Options: `--strategy `, `--contextualization`, `--from-git `, `--dry-run`, `--yes/-y`, `--force/-f`, `--metadata `, `--parallel ` (1-20) + - Options: `--strategy `, `--contextualization`, `--from-git `, `--dry-run`, `--yes/-y`, `--force/-f`, `--metadata `, `--parallel ` (1-200) ### Configuration @@ -229,7 +229,7 @@ Set defaults for common options: # Upload defaults (apply to both upload and sync commands) mxbai config set defaults.upload.strategy high_quality # or 'fast' (default: fast) mxbai config set defaults.upload.contextualization true # Enable context preservation (default: false) -mxbai config set defaults.upload.parallel 10 # Concurrent operations (1-20, default: 5) +mxbai config set defaults.upload.parallel 10 # Concurrent operations (1-200, default: 100) # Search defaults mxbai config set defaults.search.top_k 20 # Number of results (default: 10) diff --git a/packages/cli/src/commands/vector-store/sync.ts b/packages/cli/src/commands/vector-store/sync.ts index 0788c54..1004633 100644 --- a/packages/cli/src/commands/vector-store/sync.ts +++ b/packages/cli/src/commands/vector-store/sync.ts @@ -41,9 +41,9 @@ const SyncVectorStoreSchema = extendGlobalOptions({ .number({ message: '"parallel" must be a number' }) .int({ message: '"parallel" must be an integer' }) .min(1, { message: '"parallel" must be at least 1' }) - .max(20, { message: '"parallel" must be less than or equal to 20' }) + .max(200, { message: '"parallel" must be less than or equal to 200' }) .optional() - .default(5), + .default(100), }); interface SyncOptions extends GlobalOptions { @@ -81,7 +81,7 @@ export function createSyncCommand(): Command { "Force re-upload all files, ignoring change detection" ) .option("--metadata ", "Additional metadata for files") - .option("--parallel ", "Number of concurrent operations (1-20)") + .option("--parallel ", "Number of concurrent operations (1-200)") ); command.action( diff --git a/packages/cli/src/commands/vector-store/upload.ts b/packages/cli/src/commands/vector-store/upload.ts index 323f4fc..dd6adad 100644 --- a/packages/cli/src/commands/vector-store/upload.ts +++ b/packages/cli/src/commands/vector-store/upload.ts @@ -40,7 +40,7 @@ const UploadVectorStoreSchema = extendGlobalOptions({ .number({ message: '"parallel" must be a number' }) .int({ message: '"parallel" must be an integer' }) .min(1, { message: '"parallel" must be at least 1' }) - .max(20, { message: '"parallel" must be less than or equal to 20' }) + .max(200, { message: '"parallel" must be less than or equal to 200' }) .optional(), unique: z.boolean().optional(), manifest: z.string().optional(), @@ -69,7 +69,7 @@ export function createUploadCommand(): Command { .option("--contextualization", "Enable context preservation") .option("--metadata ", "Additional metadata as JSON string") .option("--dry-run", "Preview what would be uploaded", false) - .option("--parallel ", "Number of concurrent uploads (1-20)") + .option("--parallel ", "Number of concurrent uploads (1-200)") .option( "--unique", "Update existing files instead of creating duplicates", @@ -125,7 +125,7 @@ export function createUploadCommand(): Command { config.defaults?.upload?.contextualization ?? false; const parallel = - parsedOptions.parallel ?? config.defaults?.upload?.parallel ?? 5; + parsedOptions.parallel ?? config.defaults?.upload?.parallel ?? 100; const metadata = validateMetadata(parsedOptions.metadata); diff --git a/packages/cli/src/utils/config.ts b/packages/cli/src/utils/config.ts index be1513b..42b84d3 100644 --- a/packages/cli/src/utils/config.ts +++ b/packages/cli/src/utils/config.ts @@ -24,6 +24,7 @@ export const UploadDefaultsSchema = z.object({ .number({ message: "Must be a number" }) .int({ message: "Must be an integer" }) .positive({ message: "Must be a positive number" }) + .max(200, { message: "Must be less than or equal to 200" }) .optional(), }); @@ -103,7 +104,7 @@ const DEFAULT_CONFIG: CLIConfig = { upload: { strategy: "fast", contextualization: false, - parallel: 5, + parallel: 100, }, search: { top_k: 10, diff --git a/packages/cli/src/utils/manifest.ts b/packages/cli/src/utils/manifest.ts index 16a093a..6cbcb2b 100644 --- a/packages/cli/src/utils/manifest.ts +++ b/packages/cli/src/utils/manifest.ts @@ -206,7 +206,7 @@ export async function uploadFromManifest( await uploadFilesInBatch(client, vectorStoreIdentifier, uniqueFiles, { unique: options.unique || false, existingFiles, - parallel: options.parallel ?? config.defaults.upload.parallel ?? 5, + parallel: options.parallel ?? config.defaults.upload.parallel ?? 100, }); } catch (error) { if (error instanceof z.ZodError) { diff --git a/packages/cli/src/utils/sync.ts b/packages/cli/src/utils/sync.ts index d9fbfd9..9dda4ad 100644 --- a/packages/cli/src/utils/sync.ts +++ b/packages/cli/src/utils/sync.ts @@ -252,7 +252,7 @@ export async function executeSyncChanges( parallel?: number; } ): Promise { - const parallel = options.parallel ?? 5; + const parallel = options.parallel ?? 100; const limit = pLimit(parallel); const totalOperations = analysis.added.length + diff --git a/packages/cli/tests/commands/config/get.test.ts b/packages/cli/tests/commands/config/get.test.ts index cb10bd2..47f4738 100644 --- a/packages/cli/tests/commands/config/get.test.ts +++ b/packages/cli/tests/commands/config/get.test.ts @@ -37,7 +37,7 @@ describe("Config Get Command", () => { defaults: { upload: { strategy: "fast", - parallel: 5, + parallel: 100, }, api_key: "work", }, diff --git a/packages/cli/tests/commands/config/set.test.ts b/packages/cli/tests/commands/config/set.test.ts index 3bb5fa2..fa34e06 100644 --- a/packages/cli/tests/commands/config/set.test.ts +++ b/packages/cli/tests/commands/config/set.test.ts @@ -124,7 +124,7 @@ describe("Config Set Command", () => { defaults: { upload: { strategy: "fast", - parallel: 5, + parallel: 100, }, }, }), @@ -139,7 +139,7 @@ describe("Config Set Command", () => { const config = loadConfig(); expect(config.defaults?.upload?.strategy).toBe("fast"); - expect(config.defaults?.upload?.parallel).toBe(5); + expect(config.defaults?.upload?.parallel).toBe(100); expect(config.defaults?.upload?.contextualization).toBe(true); }); }); diff --git a/packages/cli/tests/commands/vector-store/upload.test.ts b/packages/cli/tests/commands/vector-store/upload.test.ts index 2f33e2f..2c817f2 100644 --- a/packages/cli/tests/commands/vector-store/upload.test.ts +++ b/packages/cli/tests/commands/vector-store/upload.test.ts @@ -299,12 +299,12 @@ describe("Vector Store Upload Command", () => { "test-store", "test.md", "--parallel", - "25", + "201", ]); expect(console.error).toHaveBeenCalledWith( expect.any(String), - expect.stringContaining('"parallel" must be less than or equal to 20') + expect.stringContaining('"parallel" must be less than or equal to 200') ); expect(process.exit).toHaveBeenCalledWith(1); }); diff --git a/packages/cli/tests/helpers/test-utils.ts b/packages/cli/tests/helpers/test-utils.ts index 2400c68..06cae99 100644 --- a/packages/cli/tests/helpers/test-utils.ts +++ b/packages/cli/tests/helpers/test-utils.ts @@ -175,7 +175,7 @@ export function createTestConfig(overrides: Record = {}) { upload: { strategy: "fast", contextualization: false, - parallel: 5, + parallel: 100, }, search: { top_k: 10, diff --git a/packages/cli/tests/utils/config.test.ts b/packages/cli/tests/utils/config.test.ts index c3de2a1..2f83924 100644 --- a/packages/cli/tests/utils/config.test.ts +++ b/packages/cli/tests/utils/config.test.ts @@ -33,7 +33,7 @@ describe("Config Utils", () => { upload: { strategy: "fast", contextualization: false, - parallel: 5, + parallel: 100, }, search: { top_k: 10, From 86e75f49df2a55df4b2985951c933175e95f6e70 Mon Sep 17 00:00:00 2001 From: Naing Linn Khant Date: Fri, 29 Aug 2025 23:38:36 +0700 Subject: [PATCH 2/6] chore: update sdk --- package.json | 2 +- packages/cli/package.json | 2 +- packages/search/package.json | 2 +- pnpm-lock.yaml | 20 +++++++------------- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 60e2527..1e2e0da 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "turbo": "^2.5.4", "typescript": "5.8.2" }, - "packageManager": "pnpm@9.0.0", + "packageManager": "pnpm@10.15.0", "engines": { "node": ">=18" } diff --git a/packages/cli/package.json b/packages/cli/package.json index 9099362..579bd85 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -49,7 +49,7 @@ "setup-cli": "node ./scripts/setup-cli.js" }, "dependencies": { - "@mixedbread/sdk": "^0.19.0", + "@mixedbread/sdk": "^0.26.0", "@pnpm/tabtab": "^0.5.4", "chalk": "^5.3.0", "cli-table3": "^0.6.5", diff --git a/packages/search/package.json b/packages/search/package.json index 025de6e..d49d717 100644 --- a/packages/search/package.json +++ b/packages/search/package.json @@ -11,7 +11,7 @@ "check-types": "tsc --noEmit" }, "dependencies": { - "@mixedbread/sdk": "^0.24.0", + "@mixedbread/sdk": "^0.26.0", "@radix-ui/react-dialog": "^1.1.14", "@radix-ui/react-scroll-area": "^1.2.9", "@radix-ui/react-slot": "^1.2.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eda560c..5e60386 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,8 +24,8 @@ importers: packages/cli: dependencies: '@mixedbread/sdk': - specifier: ^0.19.0 - version: 0.19.2 + specifier: ^0.26.0 + version: 0.26.0 '@pnpm/tabtab': specifier: ^0.5.4 version: 0.5.4 @@ -109,8 +109,8 @@ importers: packages/search: dependencies: '@mixedbread/sdk': - specifier: ^0.24.0 - version: 0.24.0 + specifier: ^0.26.0 + version: 0.26.0 '@radix-ui/react-dialog': specifier: ^1.1.14 version: 1.1.15(@types/react-dom@19.1.7(@types/react@19.1.10))(@types/react@19.1.10)(react-dom@19.1.1(react@19.1.1))(react@19.1.1) @@ -778,12 +778,8 @@ packages: '@manypkg/get-packages@1.1.3': resolution: {integrity: sha512-fo+QhuU3qE/2TQMQmbVMqaQ6EWbMhi4ABWP+O4AM1NqPBuy0OrApV5LO6BrrgnhtAHS2NH6RrVk9OL181tTi8A==} - '@mixedbread/sdk@0.19.2': - resolution: {integrity: sha512-X/jKfpqAnRcFmTfq2eB0TfIlD1rIN6fGtudk4qX+m3Y34HLDsDCdl457ZKv++bvY8TPKlDHvNKf7bRFUSHGtIA==} - hasBin: true - - '@mixedbread/sdk@0.24.0': - resolution: {integrity: sha512-UV9V5vwF1W3mGUaXQUvo/QcKr/r0i+HgspQevYiqhfUa9UsmpuiBSLj2wG5gVONDwVVk6+R+3MEwPFMF1/CLwQ==} + '@mixedbread/sdk@0.26.0': + resolution: {integrity: sha512-B0Un0thpz4UKN69QVmgdVsCIgiz+jR4V+nxuCOgdm/P0Iie1NctqY092VQ/iobY/FSMzhHJofXij086y/9E5rg==} hasBin: true '@next/env@15.3.3': @@ -4262,9 +4258,7 @@ snapshots: globby: 11.1.0 read-yaml-file: 1.1.0 - '@mixedbread/sdk@0.19.2': {} - - '@mixedbread/sdk@0.24.0': {} + '@mixedbread/sdk@0.26.0': {} '@next/env@15.3.3': {} From dc17e8dd42acdfec6fb9ec66ae724e136bfb703c Mon Sep 17 00:00:00 2001 From: Naing Linn Khant Date: Fri, 29 Aug 2025 23:56:21 +0700 Subject: [PATCH 3/6] test: fix test --- .../tests/commands/vector-store/files.test.ts | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/packages/cli/tests/commands/vector-store/files.test.ts b/packages/cli/tests/commands/vector-store/files.test.ts index 77607f3..0348135 100644 --- a/packages/cli/tests/commands/vector-store/files.test.ts +++ b/packages/cli/tests/commands/vector-store/files.test.ts @@ -7,11 +7,8 @@ import { jest, } from "@jest/globals"; import type Mixedbread from "@mixedbread/sdk"; -import type { CursorResponse } from "@mixedbread/sdk/core/pagination"; -import type { - VectorStoreFile, - VectorStoreFilesCursor, -} from "@mixedbread/sdk/resources/vector-stores"; +import type { VectorStoreFile } from "@mixedbread/sdk/resources/vector-stores"; +import type { FileListResponse } from "@mixedbread/sdk/resources/vector-stores/files"; import type { Command } from "commander"; import { createFilesCommand } from "../../../src/commands/vector-store/files"; import * as clientUtils from "../../../src/utils/client"; @@ -38,17 +35,14 @@ const mockFormatOutput = outputUtils.formatOutput as jest.MockedFunction< typeof outputUtils.formatOutput >; -// Helper to create a properly typed mock cursor -// Since the Cursor class has private fields, we need to use type assertion -// but we avoid 'any' by being specific about what we're mocking const createMockCursor = ( data: VectorStoreFile[], - pagination: CursorResponse.Pagination -): VectorStoreFilesCursor => { + pagination: FileListResponse.Pagination +): FileListResponse => { return { data, pagination, - } as unknown as VectorStoreFilesCursor; + }; }; describe("Files Command", () => { @@ -209,8 +203,8 @@ describe("Files Command", () => { it("should handle empty results", async () => { mockClient.vectorStores.files.list.mockResolvedValue( createMockCursor([], { - first_cursor: undefined, - last_cursor: undefined, + first_cursor: null, + last_cursor: null, has_more: false, }) ); From 447857815cbb10bc130a4adb77d12c9ffb0c1599 Mon Sep 17 00:00:00 2001 From: Naing Linn Khant Date: Sat, 30 Aug 2025 00:00:55 +0700 Subject: [PATCH 4/6] fix: fix ci --- .github/workflows/changeset-validation.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/prerelease.yml | 2 +- .github/workflows/release.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/changeset-validation.yml b/.github/workflows/changeset-validation.yml index e2f9da5..103464c 100644 --- a/.github/workflows/changeset-validation.yml +++ b/.github/workflows/changeset-validation.yml @@ -17,7 +17,7 @@ jobs: - name: Install pnpm uses: pnpm/action-setup@v4 with: - version: 9.0.0 + version: 10.15.0 - name: Setup Node.js uses: actions/setup-node@v4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8af3a3a..1539aaf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - name: Install pnpm uses: pnpm/action-setup@v4 with: - version: 9.0.0 + version: 10.15.0 - name: Setup Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index dc2aa75..9aad812 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -31,7 +31,7 @@ jobs: - name: Install pnpm uses: pnpm/action-setup@v4 with: - version: 9.0.0 + version: 10.15.0 - name: Setup Node.js uses: actions/setup-node@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 934281a..b552cbc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: - name: Install pnpm uses: pnpm/action-setup@v4 with: - version: 9.0.0 + version: 10.15.0 - name: Setup Node.js uses: actions/setup-node@v4 From 3aac554b34b167cfec95dfabb1ca06045ed227ec Mon Sep 17 00:00:00 2001 From: Naing Linn Khant Date: Sat, 30 Aug 2025 00:02:16 +0700 Subject: [PATCH 5/6] fix: fix types --- packages/search/src/search/lib/fumadocs.ts | 2 +- packages/search/src/search/lib/search.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/search/src/search/lib/fumadocs.ts b/packages/search/src/search/lib/fumadocs.ts index c29201b..001174b 100644 --- a/packages/search/src/search/lib/fumadocs.ts +++ b/packages/search/src/search/lib/fumadocs.ts @@ -47,7 +47,7 @@ export async function search( const res = await client.vectorStores.search({ query, - vector_store_ids: [vectorStoreId], + vector_store_identifiers: [vectorStoreId], top_k: 10, search_options: { return_metadata: true, diff --git a/packages/search/src/search/lib/search.ts b/packages/search/src/search/lib/search.ts index 35ac009..00dfd4e 100644 --- a/packages/search/src/search/lib/search.ts +++ b/packages/search/src/search/lib/search.ts @@ -24,7 +24,7 @@ export async function search( const res = await mxbai.vectorStores.files.search({ query, - vector_store_ids: [process.env.VECTOR_STORE_ID], + vector_store_identifiers: [process.env.VECTOR_STORE_ID], top_k: topK, search_options: { return_metadata: true, From 37cf096939a60227b3ad2fcc620bb8b46edf7729 Mon Sep 17 00:00:00 2001 From: Naing Linn Khant Date: Sat, 30 Aug 2025 00:16:34 +0700 Subject: [PATCH 6/6] chore(release): add changeset --- .changeset/young-taxis-deny.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/young-taxis-deny.md diff --git a/.changeset/young-taxis-deny.md b/.changeset/young-taxis-deny.md new file mode 100644 index 0000000..161df3f --- /dev/null +++ b/.changeset/young-taxis-deny.md @@ -0,0 +1,6 @@ +--- +"@mixedbread/cli": minor +--- + +- Change max parallel upload/deletion to 200 +- Change default parallel and `parallel` in config to 100