GML-1993 Local folder processing: added code for multiple files processing

common/chunkers/__init__.py

@@ -1,6 +1,8 @@
 from .base_chunker import BaseChunker
 from .character_chunker import CharacterChunker
+from .html_chunker import HTMLChunker
 from .markdown_chunker import MarkdownChunker
 from .regex_chunker import RegexChunker
 from .semantic_chunker import SemanticChunker
-from .recursive_chunker import RecursiveChunker
+from .recursive_chunker import RecursiveChunker
+from .single_chunker import SingleChunker

common/chunkers/html_chunker.py

@@ -0,0 +1,84 @@
+# Copyright (c) 2025 TigerGraph, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional, List, Tuple
+import re
+from common.chunkers.base_chunker import BaseChunker
+from langchain_text_splitters import HTMLSectionSplitter
+
+
+class HTMLChunker(BaseChunker):
+    """
+    HTML chunker that splits HTML content into chunks based on header tags.
+
+    - Automatically detects which headers (h1-h6) are present in the HTML
+    - Uses only the headers that exist in the document for optimal chunking
+    - If custom headers are provided, uses those instead of auto-detection
+    """
+
+    def __init__(
+        self,
+        headers: Optional[List[Tuple[str, str]]] = None  # e.g. [("h1", "Header 1"), ("h2", "Header 2")]
+    ):
+        self.headers = headers
+
+    def _detect_headers(self, html_content: str) -> List[Tuple[str, str]]:
+        """
+        Automatically detect which header tags (h1-h6) are present in the HTML.
+        Returns a list of header tuples for headers that exist in the document.
+        """
+        # All possible headers in hierarchical order
+        all_headers = [
+            ("h1", "Header 1"),
+            ("h2", "Header 2"),
+            ("h3", "Header 3"),
+            ("h4", "Header 4"),
+            ("h5", "Header 5"),
+            ("h6", "Header 6")
+        ]
+
+        # Detect which headers are actually present in the HTML
+        detected_headers = []
+        for tag, name in all_headers:
+            # Use regex to find header tags (case insensitive)
+            pattern = f'<{tag}[\\s>]'
+            if re.search(pattern, html_content, re.IGNORECASE):
+                detected_headers.append((tag, name))
+
+        # If no headers detected, use h1-h3 as fallback
+        if not detected_headers:
+            detected_headers = [
+                ("h1", "Header 1"),
+                ("h2", "Header 2"),
+                ("h3", "Header 3")
+            ]
+
+        return detected_headers
+
+    def chunk(self, input_string: str) -> List[str]:
+        # Use custom headers if provided, otherwise auto-detect from HTML
+        if self.headers:
+            headers_to_use = self.headers
+        else:
+            headers_to_use = self._detect_headers(input_string)
+
+        # Use HTMLSectionSplitter with detected/provided headers
+        splitter = HTMLSectionSplitter(headers_to_split_on=headers_to_use)
+        docs = splitter.split_text(input_string)
+
+        # Extract text content from Document objects
+        return [doc.page_content for doc in docs]
+
+    def __call__(self, input_string: str) -> List[str]:
+        return self.chunk(input_string)

common/chunkers/single_chunker.py

@@ -0,0 +1,30 @@
+"""
+Single Chunker - Always returns the entire content as ONE chunk.
+Used for images to preserve [IMAGE_REF:] markers and prevent splitting.
+"""
+from common.chunkers.base_chunker import BaseChunker
+
+
+class SingleChunker(BaseChunker):
+    """
+    Chunker that NEVER splits content - always returns ONE chunk.
+
+    This is critical for image descriptions to:
+    1. Keep [IMAGE_REF:] markers intact
+    2. Prevent losing image references when displayed in UI
+    3. Maintain semantic integrity of image descriptions
+    """
+
+    def chunk(self, text: str) -> list[str]:
+        """
+        Return the entire text as a single chunk, regardless of length.
+
+        Args:
+            text: The text to "chunk" (actually just return as-is)
+
+        Returns:
+            List with single element containing all text
+        """
+        # Always return ONE chunk with entire content
+        return [text] if text and text.strip() else []
+

common/config.py

@@ -91,6 +91,41 @@
     raise Exception("embedding_service is not found in llm_config")
 embedding_dimension = embedding_config.get("dimensions", 1536)
 
+# Get multimodal_service config (optional, for vision/image tasks)
+multimodal_config = llm_config.get("multimodal_service")
+
+# Merge shared authentication configuration from llm_config level into service configs
+# Services can still override by defining their own authentication_configuration
+shared_auth = llm_config.get("authentication_configuration", {})
+if shared_auth:
+    # Merge into embedding_config (service-specific auth takes precedence)
+    if "authentication_configuration" not in embedding_config:
+        embedding_config["authentication_configuration"] = shared_auth.copy()
+    else:
+        # Merge shared auth with service-specific auth (service-specific takes precedence)
+        merged_embedding_auth = shared_auth.copy()
+        merged_embedding_auth.update(embedding_config["authentication_configuration"])
+        embedding_config["authentication_configuration"] = merged_embedding_auth
+
+    # Merge into completion_config (service-specific auth takes precedence)
+    if "authentication_configuration" not in completion_config:
+        completion_config["authentication_configuration"] = shared_auth.copy()
+    else:
+        # Merge shared auth with service-specific auth (service-specific takes precedence)
+        merged_completion_auth = shared_auth.copy()
+        merged_completion_auth.update(completion_config["authentication_configuration"])
+        completion_config["authentication_configuration"] = merged_completion_auth
+
+    # Merge into multimodal_config if it exists (service-specific auth takes precedence)
+    if multimodal_config:
+        if "authentication_configuration" not in multimodal_config:
+            multimodal_config["authentication_configuration"] = shared_auth.copy()
+        else:
+            # Merge shared auth with service-specific auth (service-specific takes precedence)
+            merged_multimodal_auth = shared_auth.copy()
+            merged_multimodal_auth.update(multimodal_config["authentication_configuration"])
+            multimodal_config["authentication_configuration"] = merged_multimodal_auth
+
 if graphrag_config is None:
     graphrag_config = {"reuse_embedding": True}
 if "chunker" not in graphrag_config:
@@ -149,6 +184,38 @@ def get_llm_service(llm_config) -> LLM_Model:
     else:
         raise Exception("LLM Completion Service Not Supported")
 
+def get_multimodal_service() -> LLM_Model:
+    """
+    Get the multimodal/vision LLM service for image description tasks.
+    Uses multimodal_service if configured, otherwise falls back to completion_service.
+    Currently supports: OpenAI, Azure, GenAI, VertexAI
+    """
+    # Use multimodal_service if available, otherwise fallback to completion_service
+    service_config = multimodal_config if multimodal_config else completion_config
+
+    # Make a copy to avoid modifying the original config
+    config_copy = service_config.copy()
+
+    # Add default prompt_path if not present (required by LLM service classes but not used for multimodal)
+    if "prompt_path" not in config_copy:
+        config_copy["prompt_path"] = "./common/prompts/openai_gpt4/"
+
+    service_type = config_copy["llm_service"].lower()
+
+    if service_type == "openai":
+        return OpenAI(config_copy)
+    elif service_type == "azure":
+        return AzureOpenAI(config_copy)
+    elif service_type == "genai":
+        return GoogleGenAI(config_copy)
+    elif service_type == "vertexai":
+        return GoogleVertexAI(config_copy)
+    else:
+        raise Exception(
+            f"Multimodal service '{service_type}' not supported. "
+            "Only OpenAI, Azure, GenAI, and VertexAI are currently supported for vision tasks."
+        )
+
 if os.getenv("INIT_EMBED_STORE", "true") == "true":
     conn = TigerGraphConnection(
         host=db_config.get("hostname", "http://tigergraph"),

common/gsql/supportai/SupportAI_InitialLoadJSON_WithImages.gsql

@@ -0,0 +1,41 @@
+CREATE LOADING JOB load_documents_content_json_with_images_@uuid@ {
+    DEFINE FILENAME DocumentContent;
+
+    // Load Documents (only entries with content field, not image storage entries)
+    LOAD DocumentContent TO VERTEX Document VALUES(gsql_lower($"doc_id"), gsql_current_time_epoch(0), _, _) 
+        WHERE $"content" != ""
+        USING JSON_FILE="true";
+
+    // Load Content (for documents with content field)
+    // Both markdown and standalone image documents create Content vertices
+    LOAD DocumentContent TO VERTEX Content VALUES(gsql_lower($"doc_id"), $"doc_type", $"content", gsql_current_time_epoch(0)) 
+        WHERE $"content" != ""
+        USING JSON_FILE="true";
+
+    // Edge: Document -> Content (for documents with content)
+    LOAD DocumentContent TO EDGE HAS_CONTENT VALUES(
+        gsql_lower($"doc_id") Document, 
+        gsql_lower($"doc_id") Content
+    ) WHERE $"content" != ""
+      USING JSON_FILE="true";
+
+    // Load Images (ONLY for image storage entries which have image_data field)
+    LOAD DocumentContent TO VERTEX Image VALUES(
+        gsql_lower($"doc_id"),
+        $"image_data",
+        $"image_format",
+        gsql_lower($"parent_doc"),
+        $"page_number",
+        gsql_current_time_epoch(0)
+    ) WHERE $"image_data" != ""
+      USING JSON_FILE="true";
+
+    // Edge: Document -> Image (for image documents)
+    LOAD DocumentContent TO EDGE HAS_IMAGE VALUES(
+        gsql_lower($"parent_doc") Document,
+        gsql_lower($"doc_id") Image,
+        $"position"
+    ) WHERE $"doc_type" == "image" AND $"parent_doc" != ""
+      USING JSON_FILE="true";
+}
+

common/gsql/supportai/SupportAI_Schema_Images.gsql

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2025 TigerGraph, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+CREATE SCHEMA_CHANGE JOB add_image_support_schema {
+    ADD VERTEX Image(
+        PRIMARY_ID id STRING,
+        image_data STRING,           // base64 encoded image data
+        image_format STRING,          // jpg, png, gif, etc.
+        source_document STRING,       // parent document ID
+        page_number INT,              // for PDFs, which page
+        epoch_added UINT
+    ) WITH STATS="OUTDEGREE_BY_EDGETYPE", PRIMARY_ID_AS_ATTRIBUTE="true";
+
+    // Document can have multiple images (standalone or embedded)
+    ADD DIRECTED EDGE HAS_IMAGE(FROM Document, TO Image, position_index INT) 
+        WITH REVERSE_EDGE="reverse_HAS_IMAGE";
+
+    // DocumentChunk can reference images that appear in its content
+    ADD DIRECTED EDGE REFERENCES_IMAGE(FROM DocumentChunk, TO Image) 
+        WITH REVERSE_EDGE="reverse_REFERENCES_IMAGE";
+}
+

common/prompts/google_gemini/chatbot_response.txt

@@ -20,10 +20,11 @@ Example format for responses:
 - **Escalation**: "This issue may require human assistance. Please reach out via [Support Portal](https://www.tigergraph.com/support/) or email us at [support@tigergraph.com](mailto:support@tigergraph.com)."
 
 Your mission is to ensure a seamless, satisfying customer experience while upholding TigerGraph's values and commitment to enterprise excellence.
+Make sure to extract and include the image references in [IMAGE_REF:image_id] format in your generated_answer if they are present in the context. Images are critical visual information that must be included in your response. Do NOT modify or omit these image references.
 
 Give the context in JSON format, combine and rephrase it to answer the question.
 Use only the provided information in question and context without adding any reasoning or additional logic.
-Make sure all information in the question and context are covered in the generated answer.
+Make sure all information in the question and context are covered in the generated answer, including any image markdown references.
 Generate the answer in JSON format, make sure to escape necessary characters in order to return a valid JSON response only.
 Make sure all the fields required by the format instructions are included, set a field to empty if you don't have that information.
 

common/prompts/openai_gpt4/chatbot_response.txt

@@ -15,13 +15,11 @@ Example format for responses:
 - **Escalation**: "This issue may require human assistance. Please reach out via [Support Portal](https://www.tigergraph.com/support/) or email us at [support@tigergraph.com](mailto:support@tigergraph.com)."
 
 Format your answer using Markdown. Organize the content into paragraphs, bulleted or numbered lists, and include links to images where relevant.
+Make sure to extract and include the image references in [IMAGE_REF:image_id] format in your generated_answer if they are present in the context. Images are critical visual information that must be included in your response. Do NOT modify or omit these image references.
 
 Give the context in JSON format, combine and rephrase it to answer the question.
 Use only the provided information in context without adding any reasoning or additional logic.
-Make sure all information in the context are covered in the generated answer.
-Make sure to extract and include the image links in markdown syntax in the generated answer when their summaries are referenced, and preserve the link URLs in their original format.
-Use compact markdown syntax to geneate the answer, including title, bulleted or numbered list, images and tables if any, and place images or tables below the related text section.
-Ensure that each row of every table, including the header row, starts on a new line.
+Make sure all information in the context are covered in the generated answer, including any image markdown references.
 Generate the answer in JSON format, make sure to escape necessary characters in order to return a valid JSON response only.
 Make sure all the fields required by the format instructions are included, set a field to empty if you don't have that information.
 

common/py_schemas/schemas.py

@@ -119,7 +119,7 @@ class CreateVectorIndexConfig(BaseModel):
 class CreateIngestConfig(BaseModel):
     data_source: str
     data_source_config: Dict
-    loader_config: Dict = {"doc_id_field": str, "content_field": str}
+    loader_config: Optional[Dict] = None  # Made optional - will auto-generate defaults
     file_format: str = "json"
 
 

common/requirements.txt

@@ -108,8 +108,9 @@ ordered-set==4.1.0
 orjson==3.10.18
 packaging==24.2
 pandas==2.2.3
-pathtools==0.1.2
+#pathtools==0.1.2
 pillow==11.2.1
+PyMuPDF==1.26.4
 platformdirs==4.3.8
 pluggy==1.6.0
 prometheus_client==0.22.1
@@ -127,6 +128,8 @@ pygit2==1.18.0
 pyparsing==3.2.3
 pypdf==5.6.1
 pytest==8.4.1
+python-docx==1.1.2
+pytesseract==0.3.10
 python-dateutil==2.9.0.post0
 python-dotenv==1.1.1
 python-iso639==2025.2.18