fix: fix mime types before uploading

.changeset/cold-bobcats-win.md

@@ -0,0 +1,5 @@
+---
+"@mixedbread/cli": patch
+---
+
+Fix metadata types before uploading

CLAUDE.md

@@ -76,6 +76,7 @@ pnpm release
 - Tests in `tests/` directories within each package
 - Mocks for external dependencies (chalk, ora, inquirer, glob, p-limit)
 - Run `pnpm test` from root or package directory
+- We don't need redundant tests in most cases. For example, I've removed handling edge cases like FILE.TS, my.test.ts, no file extension. Ask me follow-up questions if you're not sure if a test is redundant or not.
 
 ## Environment Setup
 
@@ -103,4 +104,5 @@ export MXBAI_API_KEY="your-api-key"
 - File upload with processing strategies (high_quality, fast, auto)
 - Git-based and hash-based sync capabilities
 - Manifest-based bulk uploads via YAML configuration
-- Support for aliases and configuration management
+- Support for aliases and configuration management
+```

packages/cli/src/utils/upload.ts

@@ -28,6 +28,37 @@ export interface UploadResults {
   successfulSize: number;
 }
 
+/**
+ * Fix MIME types for files that are commonly misidentified
+ */
+function fixMimeTypes(file: File): File {
+  const fileName = file.name.toLowerCase();
+  let correctedType = file.type;
+
+  // Fix .ts files that are detected as video/mp2t
+  if (fileName.endsWith(".ts") && file.type === "video/mp2t") {
+    correctedType = "text/typescript";
+  }
+  // Fix .py files that might be detected incorrectly
+  else if (fileName.endsWith(".py") && file.type !== "text/x-python") {
+    correctedType = "text/x-python";
+  }
+  // Fix .mdx files that are detected as text/x-markdown
+  else if (fileName.endsWith(".mdx") && file.type !== "text/mdx") {
+    correctedType = "text/mdx";
+  }
+
+  if (correctedType !== file.type) {
+    // Only create a new File object if we need to correct the type
+    return new File([file], file.name, {
+      type: correctedType,
+      lastModified: file.lastModified,
+    });
+  }
+
+  return file;
+}
+
 /**
  * Upload a single file to a vector store
  */
@@ -43,7 +74,9 @@ export async function uploadFile(
   const fileContent = readFileSync(filePath);
   const fileName = basename(filePath);
   const mimeType = lookup(filePath) || "application/octet-stream";
-  const file = new File([fileContent], fileName, { type: mimeType });
+  const file = fixMimeTypes(
+    new File([fileContent], fileName, { type: mimeType })
+  );
 
   // Upload the file
   await client.vectorStores.files.upload(vectorStoreIdentifier, file, {
@@ -132,9 +165,11 @@ export async function uploadFilesInBatch(
         const fileContent = readFileSync(file.path);
         const fileName = basename(file.path);
         const mimeType = lookup(file.path) || "application/octet-stream";
-        const fileToUpload = new File([fileContent], fileName, {
-          type: mimeType,
-        });
+        const fileToUpload = fixMimeTypes(
+          new File([fileContent], fileName, {
+            type: mimeType,
+          })
+        );
 
         await client.vectorStores.files.upload(
           vectorStoreIdentifier,

packages/cli/tests/utils/upload.test.ts

@@ -122,6 +122,75 @@ describe("Upload Utils", () => {
       );
       expect(mockConsoleWarn).not.toHaveBeenCalled();
     });
+
+    it("should fix TypeScript files with incorrect video/mp2t mime type", async () => {
+      mockFs({
+        "test.ts": "const hello = 'world';",
+      });
+
+      mockClient.vectorStores.files.upload.mockResolvedValue({});
+
+      await uploadFile(
+        mockClient as unknown as Mixedbread,
+        "test-store",
+        "test.ts"
+      );
+
+      expect(mockClient.vectorStores.files.upload).toHaveBeenCalledWith(
+        "test-store",
+        expect.objectContaining({
+          name: "test.ts",
+          type: "text/typescript",
+        }),
+        expect.any(Object)
+      );
+    });
+
+    it("should fix Python files with incorrect mime type", async () => {
+      mockFs({
+        "script.py": "print('Hello, World!')",
+      });
+
+      mockClient.vectorStores.files.upload.mockResolvedValue({});
+
+      await uploadFile(
+        mockClient as unknown as Mixedbread,
+        "test-store",
+        "script.py"
+      );
+
+      expect(mockClient.vectorStores.files.upload).toHaveBeenCalledWith(
+        "test-store",
+        expect.objectContaining({
+          name: "script.py",
+          type: "text/x-python",
+        }),
+        expect.any(Object)
+      );
+    });
+
+    it("should fix MDX files with incorrect mime type", async () => {
+      mockFs({
+        "content.mdx": "# Hello MDX\n\n<Component />",
+      });
+
+      mockClient.vectorStores.files.upload.mockResolvedValue({});
+
+      await uploadFile(
+        mockClient as unknown as Mixedbread,
+        "test-store",
+        "content.mdx"
+      );
+
+      expect(mockClient.vectorStores.files.upload).toHaveBeenCalledWith(
+        "test-store",
+        expect.objectContaining({
+          name: "content.mdx",
+          type: "text/mdx",
+        }),
+        expect.any(Object)
+      );
+    });
   });
 
   describe("uploadFilesInBatch", () => {
@@ -326,6 +395,95 @@ describe("Upload Utils", () => {
       );
       expect(mockClient.vectorStores.files.upload).toHaveBeenCalledTimes(1);
     });
+
+    it("should fix mime types for TypeScript, Python, and MDX files in batch", async () => {
+      mockFs({
+        "app.ts": "const app = 'TypeScript';",
+        "utils.py": "def hello(): pass",
+        "page.mdx": "# MDX Page\n\n<Hero />",
+        "readme.md": "# Regular Markdown",
+      });
+
+      mockClient.vectorStores.files.upload.mockResolvedValue({});
+
+      const files = [
+        {
+          path: "app.ts",
+          strategy: "fast" as const,
+          contextualization: false,
+          metadata: {},
+        },
+        {
+          path: "utils.py",
+          strategy: "fast" as const,
+          contextualization: false,
+          metadata: {},
+        },
+        {
+          path: "page.mdx",
+          strategy: "fast" as const,
+          contextualization: false,
+          metadata: {},
+        },
+        {
+          path: "readme.md",
+          strategy: "fast" as const,
+          contextualization: false,
+          metadata: {},
+        },
+      ];
+
+      await uploadFilesInBatch(
+        mockClient as unknown as Mixedbread,
+        "test-store",
+        files,
+        {
+          unique: false,
+          existingFiles: new Map(),
+          parallel: 4,
+        }
+      );
+
+      // Verify TypeScript file mime type was fixed
+      expect(mockClient.vectorStores.files.upload).toHaveBeenCalledWith(
+        "test-store",
+        expect.objectContaining({
+          name: "app.ts",
+          type: "text/typescript",
+        }),
+        expect.any(Object)
+      );
+
+      // Verify Python file mime type was fixed
+      expect(mockClient.vectorStores.files.upload).toHaveBeenCalledWith(
+        "test-store",
+        expect.objectContaining({
+          name: "utils.py",
+          type: "text/x-python",
+        }),
+        expect.any(Object)
+      );
+
+      // Verify MDX file mime type was fixed
+      expect(mockClient.vectorStores.files.upload).toHaveBeenCalledWith(
+        "test-store",
+        expect.objectContaining({
+          name: "page.mdx",
+          type: "text/mdx",
+        }),
+        expect.any(Object)
+      );
+
+      // Verify regular markdown file kept its original mime type
+      expect(mockClient.vectorStores.files.upload).toHaveBeenCalledWith(
+        "test-store",
+        expect.objectContaining({
+          name: "readme.md",
+          type: "text/markdown",
+        }),
+        expect.any(Object)
+      );
+    });
   });
 
   afterAll(() => {