diff --git a/.changeset/cold-bobcats-win.md b/.changeset/cold-bobcats-win.md new file mode 100644 index 0000000..8388c92 --- /dev/null +++ b/.changeset/cold-bobcats-win.md @@ -0,0 +1,5 @@ +--- +"@mixedbread/cli": patch +--- + +Fix metadata types before uploading diff --git a/CLAUDE.md b/CLAUDE.md index 2b6fefa..3e9e8cf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -76,6 +76,7 @@ pnpm release - Tests in `tests/` directories within each package - Mocks for external dependencies (chalk, ora, inquirer, glob, p-limit) - Run `pnpm test` from root or package directory +- We don't need redundant tests in most cases. For example, I've removed handling edge cases like FILE.TS, my.test.ts, no file extension. Ask me follow-up questions if you're not sure if a test is redundant or not. ## Environment Setup @@ -103,4 +104,5 @@ export MXBAI_API_KEY="your-api-key" - File upload with processing strategies (high_quality, fast, auto) - Git-based and hash-based sync capabilities - Manifest-based bulk uploads via YAML configuration -- Support for aliases and configuration management \ No newline at end of file +- Support for aliases and configuration management +``` \ No newline at end of file diff --git a/packages/cli/src/utils/upload.ts b/packages/cli/src/utils/upload.ts index b747616..2cbad0a 100644 --- a/packages/cli/src/utils/upload.ts +++ b/packages/cli/src/utils/upload.ts @@ -28,6 +28,37 @@ export interface UploadResults { successfulSize: number; } +/** + * Fix MIME types for files that are commonly misidentified + */ +function fixMimeTypes(file: File): File { + const fileName = file.name.toLowerCase(); + let correctedType = file.type; + + // Fix .ts files that are detected as video/mp2t + if (fileName.endsWith(".ts") && file.type === "video/mp2t") { + correctedType = "text/typescript"; + } + // Fix .py files that might be detected incorrectly + else if (fileName.endsWith(".py") && file.type !== "text/x-python") { + correctedType = "text/x-python"; + } + // Fix .mdx files that are detected as text/x-markdown + else if (fileName.endsWith(".mdx") && file.type !== "text/mdx") { + correctedType = "text/mdx"; + } + + if (correctedType !== file.type) { + // Only create a new File object if we need to correct the type + return new File([file], file.name, { + type: correctedType, + lastModified: file.lastModified, + }); + } + + return file; +} + /** * Upload a single file to a vector store */ @@ -43,7 +74,9 @@ export async function uploadFile( const fileContent = readFileSync(filePath); const fileName = basename(filePath); const mimeType = lookup(filePath) || "application/octet-stream"; - const file = new File([fileContent], fileName, { type: mimeType }); + const file = fixMimeTypes( + new File([fileContent], fileName, { type: mimeType }) + ); // Upload the file await client.vectorStores.files.upload(vectorStoreIdentifier, file, { @@ -132,9 +165,11 @@ export async function uploadFilesInBatch( const fileContent = readFileSync(file.path); const fileName = basename(file.path); const mimeType = lookup(file.path) || "application/octet-stream"; - const fileToUpload = new File([fileContent], fileName, { - type: mimeType, - }); + const fileToUpload = fixMimeTypes( + new File([fileContent], fileName, { + type: mimeType, + }) + ); await client.vectorStores.files.upload( vectorStoreIdentifier, diff --git a/packages/cli/tests/utils/upload.test.ts b/packages/cli/tests/utils/upload.test.ts index 545261c..2bf5541 100644 --- a/packages/cli/tests/utils/upload.test.ts +++ b/packages/cli/tests/utils/upload.test.ts @@ -122,6 +122,75 @@ describe("Upload Utils", () => { ); expect(mockConsoleWarn).not.toHaveBeenCalled(); }); + + it("should fix TypeScript files with incorrect video/mp2t mime type", async () => { + mockFs({ + "test.ts": "const hello = 'world';", + }); + + mockClient.vectorStores.files.upload.mockResolvedValue({}); + + await uploadFile( + mockClient as unknown as Mixedbread, + "test-store", + "test.ts" + ); + + expect(mockClient.vectorStores.files.upload).toHaveBeenCalledWith( + "test-store", + expect.objectContaining({ + name: "test.ts", + type: "text/typescript", + }), + expect.any(Object) + ); + }); + + it("should fix Python files with incorrect mime type", async () => { + mockFs({ + "script.py": "print('Hello, World!')", + }); + + mockClient.vectorStores.files.upload.mockResolvedValue({}); + + await uploadFile( + mockClient as unknown as Mixedbread, + "test-store", + "script.py" + ); + + expect(mockClient.vectorStores.files.upload).toHaveBeenCalledWith( + "test-store", + expect.objectContaining({ + name: "script.py", + type: "text/x-python", + }), + expect.any(Object) + ); + }); + + it("should fix MDX files with incorrect mime type", async () => { + mockFs({ + "content.mdx": "# Hello MDX\n\n", + }); + + mockClient.vectorStores.files.upload.mockResolvedValue({}); + + await uploadFile( + mockClient as unknown as Mixedbread, + "test-store", + "content.mdx" + ); + + expect(mockClient.vectorStores.files.upload).toHaveBeenCalledWith( + "test-store", + expect.objectContaining({ + name: "content.mdx", + type: "text/mdx", + }), + expect.any(Object) + ); + }); }); describe("uploadFilesInBatch", () => { @@ -326,6 +395,95 @@ describe("Upload Utils", () => { ); expect(mockClient.vectorStores.files.upload).toHaveBeenCalledTimes(1); }); + + it("should fix mime types for TypeScript, Python, and MDX files in batch", async () => { + mockFs({ + "app.ts": "const app = 'TypeScript';", + "utils.py": "def hello(): pass", + "page.mdx": "# MDX Page\n\n", + "readme.md": "# Regular Markdown", + }); + + mockClient.vectorStores.files.upload.mockResolvedValue({}); + + const files = [ + { + path: "app.ts", + strategy: "fast" as const, + contextualization: false, + metadata: {}, + }, + { + path: "utils.py", + strategy: "fast" as const, + contextualization: false, + metadata: {}, + }, + { + path: "page.mdx", + strategy: "fast" as const, + contextualization: false, + metadata: {}, + }, + { + path: "readme.md", + strategy: "fast" as const, + contextualization: false, + metadata: {}, + }, + ]; + + await uploadFilesInBatch( + mockClient as unknown as Mixedbread, + "test-store", + files, + { + unique: false, + existingFiles: new Map(), + parallel: 4, + } + ); + + // Verify TypeScript file mime type was fixed + expect(mockClient.vectorStores.files.upload).toHaveBeenCalledWith( + "test-store", + expect.objectContaining({ + name: "app.ts", + type: "text/typescript", + }), + expect.any(Object) + ); + + // Verify Python file mime type was fixed + expect(mockClient.vectorStores.files.upload).toHaveBeenCalledWith( + "test-store", + expect.objectContaining({ + name: "utils.py", + type: "text/x-python", + }), + expect.any(Object) + ); + + // Verify MDX file mime type was fixed + expect(mockClient.vectorStores.files.upload).toHaveBeenCalledWith( + "test-store", + expect.objectContaining({ + name: "page.mdx", + type: "text/mdx", + }), + expect.any(Object) + ); + + // Verify regular markdown file kept its original mime type + expect(mockClient.vectorStores.files.upload).toHaveBeenCalledWith( + "test-store", + expect.objectContaining({ + name: "readme.md", + type: "text/markdown", + }), + expect.any(Object) + ); + }); }); afterAll(() => {