Merge mia-dev into claude/hopeful-heisenberg-gbq809

.env.example

@@ -421,6 +421,11 @@ AZURE_AI_SEARCH_SEARCH_OPTION_SELECT=
 # IMAGE_GEN_OAI_BASEURL= # Custom OpenAI base URL for image generation tool
 # IMAGE_GEN_OAI_AZURE_API_VERSION= # Custom Azure OpenAI deployments
 # IMAGE_GEN_OAI_MODEL=gpt-image-1 # OpenAI image model (e.g., gpt-image-1, gpt-image-1.5)
+# Defaults for image_gen_oai / image_edit_oai when the model passes size or quality as auto:
+# IMAGE_GEN_OAI_OUTPUT_FORMAT=jpeg # png | jpeg | webp (Azure gpt-image-1 may reject webp)
+# IMAGE_GEN_OAI_OUTPUT_COMPRESSION=85 # 0-100, used for jpeg/webp
+# IMAGE_GEN_OAI_DEFAULT_SIZE=auto # auto | 1024x1024 | 1536x1024 | 1024x1536 | 256x256 | 512x512
+# IMAGE_GEN_OAI_DEFAULT_QUALITY=auto # auto | high | medium | low
 # IMAGE_GEN_OAI_DESCRIPTION=
 # IMAGE_GEN_OAI_DESCRIPTION_WITH_FILES=Custom description for image generation tool when files are present
 # IMAGE_GEN_OAI_DESCRIPTION_NO_FILES=Custom description for image generation tool when no files are present

.github/workflows/acr-build-and-push-libre-chat.yml

@@ -0,0 +1,47 @@
+name: Build and Push MIA - LibreChat Container to ACR
+
+on:
+  push:
+    branches: ["release"]
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    env:
+      IMAGE_NAME: mia-libre-chat
+      TAG: ${{ github.sha }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Azure login (OIDC)
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }}
+
+      - name: ACR sanity
+        run: |
+          az account show -o table
+          az acr show -n "${{ vars.ACR_NAME }}" -o table
+
+      - name: Login to ACR
+        run: az acr login --name "${{ vars.ACR_NAME }}"
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile.multi
+          push: true
+          tags: |
+            ${{ vars.ACR_REGISTRY_SERVER }}/${{ env.IMAGE_NAME }}:${{ env.TAG }}
+            ${{ vars.ACR_REGISTRY_SERVER }}/${{ env.IMAGE_NAME }}:latest

api/app/clients/tools/structured/DALLE3.js

@@ -9,6 +9,7 @@ const {
   extractBaseURL,
   getProxyDispatcher,
   getEnvProxyDispatcher,
+  enforceImageSizeLimit,
   createMinimalRetentionRequest,
 } = require('@librechat/api');
 const { FileContext, ContentTypes } = require('librechat-data-provider');
@@ -194,12 +195,18 @@ Error Message: ${error.message}`);
       }
       const imageResponse = await fetch(theImageUrl, fetchOptions);
       const arrayBuffer = await imageResponse.arrayBuffer();
-      const base64 = Buffer.from(arrayBuffer).toString('base64');
+      const buffer = Buffer.from(arrayBuffer);
+      const contentType = imageResponse.headers.get('content-type');
+      const mimeType = contentType?.split(';')[0]?.trim() || 'image/png';
+      const { buffer: finalBuffer, mimeType: finalMimeType } = await enforceImageSizeLimit(
+        buffer,
+        mimeType,
+      );
       const content = [
         {
           type: ContentTypes.IMAGE_URL,
           image_url: {
-            url: `data:image/png;base64,${base64}`,
+            url: `data:${finalMimeType};base64,${finalBuffer.toString('base64')}`,
           },
         },
       ];

api/app/clients/tools/structured/FluxAPI.js

@@ -7,6 +7,7 @@ const {
   applyAxiosProxyConfig,
   createMinimalRetentionRequest,
   getHttpsProxyAgent,
+  enforceImageSizeLimit,
 } = require('@librechat/api');
 const { FileContext, ContentTypes } = require('librechat-data-provider');
 
@@ -313,12 +314,18 @@ class FluxAPI extends Tool {
         }
         const imageResponse = await fetch(imageUrl, fetchOptions);
         const arrayBuffer = await imageResponse.arrayBuffer();
-        const base64 = Buffer.from(arrayBuffer).toString('base64');
+        const buffer = Buffer.from(arrayBuffer);
+        const contentType = imageResponse.headers.get('content-type');
+        const mimeType = contentType?.split(';')[0]?.trim() || 'image/png';
+        const { buffer: finalBuffer, mimeType: finalMimeType } = await enforceImageSizeLimit(
+          buffer,
+          mimeType,
+        );
         const content = [
           {
             type: ContentTypes.IMAGE_URL,
             image_url: {
-              url: `data:image/png;base64,${base64}`,
+              url: `data:${finalMimeType};base64,${finalBuffer.toString('base64')}`,
             },
           },
         ];
@@ -546,12 +553,18 @@ class FluxAPI extends Tool {
         }
         const imageResponse = await fetch(imageUrl, fetchOptions);
         const arrayBuffer = await imageResponse.arrayBuffer();
-        const base64 = Buffer.from(arrayBuffer).toString('base64');
+        const buffer = Buffer.from(arrayBuffer);
+        const contentType = imageResponse.headers.get('content-type');
+        const mimeType = contentType?.split(';')[0]?.trim() || 'image/png';
+        const { buffer: finalBuffer, mimeType: finalMimeType } = await enforceImageSizeLimit(
+          buffer,
+          mimeType,
+        );
         const content = [
           {
             type: ContentTypes.IMAGE_URL,
             image_url: {
-              url: `data:image/png;base64,${base64}`,
+              url: `data:${finalMimeType};base64,${finalBuffer.toString('base64')}`,
             },
           },
         ];

api/app/clients/tools/structured/GeminiImageGen.js

@@ -10,6 +10,7 @@ const {
   loadServiceKey,
   getBalanceConfig,
   getEnvProxyDispatcher,
+  enforceImageSizeLimit,
   getTransactionsConfig,
 } = require('@librechat/api');
 const { getStrategyFunctions } = require('~/server/services/Files/strategies');
@@ -318,7 +319,7 @@ function createGeminiImageTool(fields = {}) {
 
   const { req, imageFiles = [], userId, fileStrategy, GEMINI_API_KEY, GOOGLE_KEY } = fields;
 
-  const imageOutputType = fields.imageOutputType || EImageOutputType.PNG;
+  const imageOutputType = fields.imageOutputType || EImageOutputType.WEBP;
 
   const geminiImageGenTool = tool(
     async ({ prompt, image_ids, aspectRatio, imageSize }, runnableConfig) => {
@@ -419,14 +420,17 @@ function createGeminiImageTool(fields = {}) {
       }
 
       const rawBuffer = Buffer.from(rawImageData, 'base64');
-      const { buffer: convertedBuffer, format: outputFormat } = await convertImageFormat(
+      const { buffer: convertedBuffer, format } = await convertImageFormat(
         rawBuffer,
         imageOutputType,
       );
-      const imageData = convertedBuffer.toString('base64');
-      const mimeType = outputFormat === 'jpeg' ? 'image/jpeg' : `image/${outputFormat}`;
-
-      const dataUrl = `data:${mimeType};base64,${imageData}`;
+      const initialMimeType =
+        format === 'jpeg' ? 'image/jpeg' : format === 'webp' ? 'image/webp' : 'image/png';
+      const { buffer: finalBuffer, mimeType } = await enforceImageSizeLimit(
+        convertedBuffer,
+        initialMimeType,
+      );
+      const dataUrl = `data:${mimeType};base64,${finalBuffer.toString('base64')}`;
       const file_ids = [v4()];
       const content = [
         {

api/app/clients/tools/structured/OpenAIImageTools.js

@@ -11,6 +11,8 @@ const {
   extractBaseURL,
   getProxyDispatcher,
   applyAxiosProxyConfig,
+  enforceImageSizeLimit,
+  resolveImageGenOaiDefaults,
 } = require('@librechat/api');
 const { getStrategyFunctions } = require('~/server/services/Files/strategies');
 const { getFiles } = require('~/models');
@@ -56,7 +58,6 @@ function createAbortHandler() {
  * @param {string} fields.IMAGE_GEN_OAI_API_KEY - The OpenAI API key
  * @param {boolean} [fields.override] - Whether to override the API key check, necessary for app initialization
  * @param {MongoFile[]} [fields.imageFiles] - The images to be used for editing
- * @param {string} [fields.imageOutputType] - The image output type configuration
  * @param {string} [fields.fileStrategy] - The file storage strategy
  * @returns {Array<ReturnType<tool>>} - Array of image tools
  */
@@ -68,8 +69,8 @@ function createOpenAIImageTools(fields = {}) {
     throw new Error('This tool is only available for agents.');
   }
   const { req } = fields;
-  const imageOutputType = fields.imageOutputType || EImageOutputType.PNG;
   const appFileStrategy = fields.fileStrategy;
+  const budgetDefaults = resolveImageGenOaiDefaults();
 
   const getApiKey = () => {
     const apiKey = process.env.IMAGE_GEN_OAI_API_KEY ?? '';
@@ -117,7 +118,7 @@ function createOpenAIImageTools(fields = {}) {
         prompt,
         background = 'auto',
         n = 1,
-        output_compression = 100,
+        output_compression,
         quality = 'auto',
         size = 'auto',
       },
@@ -136,17 +137,15 @@ function createOpenAIImageTools(fields = {}) {
 
       /** @type {OpenAI} */
       const openai = new OpenAI(clientConfig);
-      let output_format = imageOutputType;
-      if (
-        background === 'transparent' &&
-        output_format !== EImageOutputType.PNG &&
-        output_format !== EImageOutputType.WEBP
-      ) {
-        logger.warn(
-          '[ImageGenOAI] Transparent background requires PNG or WebP format, defaulting to PNG',
-        );
+      let output_format = budgetDefaults.outputFormat;
+      if (background === 'transparent' && output_format !== EImageOutputType.PNG) {
+        logger.warn('[ImageGenOAI] Transparent background requires PNG format; using PNG');
         output_format = EImageOutputType.PNG;
       }
+      const effectiveCompression =
+        typeof output_compression === 'number' ? output_compression : budgetDefaults.outputCompression;
+      const effectiveQuality = quality === 'auto' ? budgetDefaults.defaultQuality : quality;
+      const effectiveSize = size === 'auto' ? budgetDefaults.defaultSize : size;
 
       let resp;
       /** @type {AbortSignal} */
@@ -170,10 +169,10 @@ function createOpenAIImageTools(fields = {}) {
             output_format,
             output_compression:
               output_format === EImageOutputType.WEBP || output_format === EImageOutputType.JPEG
-                ? output_compression
+                ? effectiveCompression
                 : undefined,
-            quality,
-            size,
+            quality: effectiveQuality,
+            size: effectiveSize,
           },
           {
             signal: derivedSignal,
@@ -206,11 +205,21 @@ Error Message: ${error.message}`);
         );
       }
 
+      const mimeType =
+        output_format === EImageOutputType.PNG
+          ? 'image/png'
+          : output_format === EImageOutputType.WEBP
+            ? 'image/webp'
+            : 'image/jpeg';
+      const { buffer: finalBuffer, mimeType: finalMimeType } = await enforceImageSizeLimit(
+        Buffer.from(base64Image, 'base64'),
+        mimeType,
+      );
       const content = [
         {
           type: ContentTypes.IMAGE_URL,
           image_url: {
-            url: `data:image/${output_format};base64,${base64Image}`,
+            url: `data:${finalMimeType};base64,${finalBuffer.toString('base64')}`,
           },
         },
       ];
@@ -244,14 +253,24 @@ Error Message: ${error.message}`);
         };
       }
 
+      const effectiveQuality = quality === 'auto' ? budgetDefaults.defaultQuality : quality;
+      const effectiveSize = size === 'auto' ? budgetDefaults.defaultSize : size;
+
       const formData = new FormData();
       formData.append('model', imageModel);
       formData.append('prompt', replaceUnwantedChars(prompt));
       // TODO: `mask` support
       // TODO: more than 1 image support
       // formData.append('n', n.toString());
-      formData.append('quality', quality);
-      formData.append('size', size);
+      formData.append('quality', effectiveQuality);
+      formData.append('size', effectiveSize);
+      formData.append('output_format', budgetDefaults.outputFormat);
+      if (
+        budgetDefaults.outputFormat === EImageOutputType.WEBP ||
+        budgetDefaults.outputFormat === EImageOutputType.JPEG
+      ) {
+        formData.append('output_compression', String(budgetDefaults.outputCompression));
+      }
 
       /** @type {Record<FileSources, undefined | NodeStreamDownloader<File>>} */
       const streamMethods = {};
@@ -376,11 +395,22 @@ Error Message: ${error.message}`);
           );
         }
 
+        const editFormat = budgetDefaults.outputFormat;
+        const mimeType =
+          editFormat === EImageOutputType.PNG
+            ? 'image/png'
+            : editFormat === EImageOutputType.WEBP
+              ? 'image/webp'
+              : 'image/jpeg';
+        const { buffer: finalBuffer, mimeType: finalMimeType } = await enforceImageSizeLimit(
+          Buffer.from(base64Image, 'base64'),
+          mimeType,
+        );
         const content = [
           {
             type: ContentTypes.IMAGE_URL,
             image_url: {
-              url: `data:image/${imageOutputType};base64,${base64Image}`,
+              url: `data:${finalMimeType};base64,${finalBuffer.toString('base64')}`,
             },
           },
         ];

api/app/clients/tools/structured/StableDiffusion.js

@@ -7,7 +7,7 @@ const { v4: uuidv4 } = require('uuid');
 const { logger } = require('@librechat/data-schemas');
 const { Tool } = require('@librechat/agents/langchain/tools');
 const { FileContext, ContentTypes } = require('librechat-data-provider');
-const { getBasePath } = require('@librechat/api');
+const { getBasePath, enforceImageSizeLimit } = require('@librechat/api');
 const paths = require('~/config/paths');
 
 const stableDiffusionJsonSchema = {
@@ -143,11 +143,16 @@ class StableDiffusionAPI extends Tool {
 
     try {
       if (this.isAgent) {
+        const pngBase64 = image.includes(',') ? image.split(',')[1] : image;
+        const { buffer: finalBuffer, mimeType: finalMimeType } = await enforceImageSizeLimit(
+          Buffer.from(pngBase64, 'base64'),
+          'image/png',
+        );
         const content = [
           {
             type: ContentTypes.IMAGE_URL,
             image_url: {
-              url: `data:image/png;base64,${image}`,
+              url: `data:${finalMimeType};base64,${finalBuffer.toString('base64')}`,
             },
           },
         ];