feat(rag): Native RAG & Hybrid Vector Search Engine with Lucene & OpenSearch Parity

iped-api/src/main/java/iped/properties/ExtraProperties.java

@@ -203,4 +203,11 @@ public class ExtraProperties {
     public static final List<String> COMMUNICATION_BASIC_PROPS = Arrays.asList(MESSAGE_SUBJECT, MESSAGE_BODY,
             Message.MESSAGE_CC, Message.MESSAGE_BCC, Message.MESSAGE_RECIPIENT_ADDRESS, MESSAGE_IS_ATTACHMENT,
             MESSAGE_ATTACHMENT_COUNT.getName());
+
+    /**
+     * Extra attribute set to {@code "true"} on a parent evidence item when at least
+     * one of its text fragments has had an embedding generated and stored in the
+     * RAG vector index. Allows filtering/searching for embeddable items in the UI.
+     */
+    public static final String HAS_EMBEDDING = "hasEmbedding"; //$NON-NLS-1$
 }

iped-app/resources/config/IPEDConfig.txt

@@ -1,4 +1,4 @@
-########################################################################
+########################################################################
 # Processing Settings
 ########################################################################
 
@@ -104,6 +104,10 @@ indexFileContents = true
 # You must configure at least host/port options in conf/ElasticSearchConfig.txt
 enableIndexToElasticSearch = false
 
+# Enables Retrieval-Augmented Generation (RAG) capabilities.
+# Requires an embedding model server (e.g. Ollama). Advanced settings can be configured in conf/RAGConfig.txt.
+enableRAG = false
+
 # Enables exporting files to MinIO object storage cluster.
 # You must configure at least host/port options in conf/MinIOConfig.txt
 enableMinIO = false

iped-app/resources/config/conf/IndexTaskConfig.txt

@@ -17,8 +17,13 @@ filterNonLatinChars = false
 
 # Size (bytes) of the text segments extracted from items before indexing. Includes all items,
 # not just the ones indexed via strings. This avoids OutOfMemory errors while indexing items with large chunks of extracted text. 
+# Reduce to a smaller value (e.g. 4000) when using local embeddings (RAG) to fit context window limits.
 textSplitSize = 10485760
 
+# Overlap size (characters) between consecutive text segments.
+# Reduce to a smaller value (e.g. 1000) if textSplitSize is reduced for local embeddings.
+# textOverlapSize = 10000
+
 # Uses NIOFSDirectory instead of MMAPDirectory to open index (https://lucene.apache.org/core/4_9_0/core/org/apache/lucene/store/FSDirectory.html)
 # It is a bit slower, but prevents JVM crashes when reading index through network shares. 
 useNIOFSDirectory = false

iped-app/resources/config/conf/RAGConfig.txt

@@ -0,0 +1,171 @@
+###############################################################################
+#                    IPED RAG & AI ASSISTANT CONFIGURATION                    #
+#                                                                             #
+# Native Retrieval-Augmented Generation & Vector Search Engine for IPED       #
+# Default Opt-In: Set enableRAG = true to activate vector embedding indexing. #
+###############################################################################
+
+###############################################################################
+# 1. EMBEDDING GENERATION & MODEL PROVIDER
+###############################################################################
+
+# Provider for text embeddings. Options: local, remote, gemini
+# - local  : Ollama server (use 'local' for Ollama EVEN IF running on a remote GPU server over IP/network).
+# - remote : OpenAI-compatible API endpoint (e.g. OpenAI text-embedding-3-small, Azure, Cohere).
+# - gemini : Google Generative Language Embeddings API.
+embeddingProvider = local
+
+# Model name for text embeddings:
+# - Ollama (local) : bge-m3 (1024 dims), nomic-embed-text (768 dims), mxbai-embed-large (1024 dims)
+# - Gemini (remote): gemini-embedding-001 (768 dims)
+# - OpenAI (remote): text-embedding-3-small (1536 dims), text-embedding-3-large (3072 dims)
+embeddingModel = bge-m3
+
+# HTTP Endpoint for generating embeddings:
+# - Ollama (local) : http://localhost:11434/api/embeddings or http://<remote-gpu-ip>:11434/api/embeddings
+#   * Concurrency Tip: For dedicated GPU setups, environment variables like OLLAMA_NUM_PARALLEL (e.g. 2-4) and OLLAMA_MAX_QUEUE (e.g. 512) can improve throughput depending on available VRAM/GPU hardware (requires further testing for optimal lab setup).
+# - OpenAI (remote): https://api.openai.com/v1/embeddings
+# - Gemini         : leave empty to use official Google API endpoint automatically
+embeddingEndpoint = http://localhost:11434/api/embeddings
+
+# Vector dimension size of the embedding model (must match embeddingModel):
+# - bge-m3: 1024 | gemini-embedding-001: 768 | nomic-embed-text: 768 | text-embedding-3-small: 1536
+embeddingDimensions = 1024
+
+
+###############################################################################
+# 2. CHUNKING & LLM CONTEXT WINDOW TUNING
+# Controls text fragmenting during indexing and context size allocation for LLMs.
+#
+# MATHEMATICAL RELATIONSHIP & FORMULAS:
+# 1. Character-to-Token Ratio: 1 token ??? 4 characters in Portuguese (3500 chars ??? 875 tokens).
+# 2. Retrieved Context Formula:
+#    Retrieved Context Tokens ??? maxRetrievedChunks * (chunkSize / 4)
+# 3. Overlap Guideline:
+#    chunkOverlap should be 10% to 25% of chunkSize (e.g. 700 for 3500).
+#
+# RECOMMENDED SETTINGS BY GPU VRAM / HARDWARE (For Local Ollama Models):
+# - 6 GB VRAM GPU (RTX 2060 / 3050):
+#   llmContextWindow = 4096 | chunkSize = 3000 | maxRetrievedChunks = 4 (4B models)
+# - 8 GB VRAM GPU (RTX 3060 / 4060) [DEFAULT BASELINE]:
+#   llmContextWindow = 9216 | chunkSize = 3500 | maxRetrievedChunks = 5 (4B models ~6GB VRAM)
+# - 16 GB VRAM GPU (RTX A4000 / 4060 Ti 16GB / 4080):
+#   llmContextWindow = 9216 | chunkSize = 3500 | maxRetrievedChunks = 5 (8B models ~16GB VRAM, 100% GPU)
+#   * Note: Setting 16384 on 8B models balloons VRAM allocation to 25GB, forcing 43% CPU offload!
+# - 24 GB VRAM GPU (RTX 3090 / 4090):
+#   llmContextWindow = 16384 | chunkSize = 4000 | maxRetrievedChunks = 10 (8B models ~24GB VRAM)
+#
+# NOTE FOR COMMERCIAL CLOUD MODELS (Google Gemini / Anthropic Claude / OpenAI):
+# - Commercial APIs have massive native context windows (128k to 2M tokens).
+# - Therefore, llmContextWindow is IGNORED for commercial providers (llmProvider != local).
+# - For Cloud Models, simply increase maxRetrievedChunks = 15-20 and chunkSize = 4000.
+###############################################################################
+
+# Size (in characters) of each text chunk/fragment created during indexing.
+# Recommendation: 3000-5000 chars. Default: 3500
+chunkSize = 3500
+
+# Overlap (in characters) between consecutive chunks to prevent boundary context loss.
+# Recommendation: 10%-25% of chunkSize. Default: 700
+chunkOverlap = 700
+
+# Maximum number of fragment chunks to retrieve and include in LLM context.
+# Recommended for 8GB VRAM GPU: 5 | Cloud (Gemini/Claude/OpenAI): 15-20. Default: 5
+maxRetrievedChunks = 5
+
+# Context window size (in tokens) STRICTLY FOR LOCAL OLLAMA MODELS (llmProvider = local).
+# Dynamically configures 'num_ctx' on Ollama to prevent truncation and VRAM OOM.
+# NOTE: Ignored for commercial APIs (Gemini, Claude, OpenAI) which natively accept huge contexts.
+# Range: 2048 to 16384. Default: 9216 (optimized for 8GB VRAM GPUs).
+llmContextWindow = 9216
+
+
+###############################################################################
+# 3. VECTOR STORE BACKEND & HYBRID SEARCH RANKING (BM25 + KNN)
+###############################################################################
+
+# Vector store backend for semantic (KNN) search:
+# - lucene     : (default) Vectors stored directly in local Lucene index. Cases are 100% portable.
+# - opensearch : Vectors stored in external OpenSearch cluster via REST API.
+#                NOTE: Requires 'enableIndexToElasticSearch = true' in IPEDConfig.txt for cluster indexing.
+#                FALLBACK: Automatically falls back to 'lucene' if OpenSearch cluster is unreachable.
+vectorStoreMode = lucene
+
+# Minimum raw vector similarity threshold applied BEFORE Reciprocal Rank Fusion (RRF).
+# Discards weak vector matches below 0.62. Range: 0.0 to 1.0. Default: 0.62
+chunkSimilarityThreshold = 0.62
+
+# Weight of vector (semantic) search in Reciprocal Rank Fusion (RRF). Range: 0.0 to 1.0. Default: 0.4
+# Note: Lexical (BM25) weight is automatically (1.0 - vectorSearchWeight) = 0.6.
+vectorSearchWeight = 0.4
+
+
+###############################################################################
+# 4. EVIDENCE FILTERING RULES (BLACKLIST & WHITELIST)
+# Controls which evidence items generate embeddings during IPED indexing.
+###############################################################################
+
+# Filtering mode for generating embeddings during indexing. Options: blacklist, whitelist
+# - blacklist : excludes categories and MIME types matching the blacklist patterns below
+# - whitelist : ONLY includes categories and MIME types matching the whitelist patterns below (Default)
+embeddingFilterMode = whitelist
+
+# -----------------------------------------------------------------------------
+# BLACKLIST RULES (Active when embeddingFilterMode = blacklist)
+# Excludes non-semantic binary data, system registries, archives, executables, and unallocated space.
+# -----------------------------------------------------------------------------
+
+# Blacklisted IPED Categories:
+# Excludes: Programs/Libraries, Compressed Archives, Image Disks, Unallocated Space, File Slacks, Registry Files, Fonts, Geographic Data, Databases, XML Files.
+embeddingCategoryBlacklist = ^(Programs and Libraries|Compressed Archives|Image Disks|Unallocated|File Slacks|Main Registry Files|Other Registry Files|Fonts|Geographic Data|Databases|XML Files|Other)$
+
+# Blacklisted MIME Types:
+# Excludes: Binaries (.exe, .dll, .so, .elf), Android binaries (.dex, .apk), Java archives (.jar), Databases (.sqlite, .edb), XML dumps/reports, Archives (.zip, .rar, .7z, .tar, .gz), Fonts, Shortcuts (.lnk), Debug Symbols (.pdb).
+embeddingMimeTypeBlacklist = ^(application/octet-stream|application/x-dosexec|application/x-sharedlib|application/x-elf|application/x-executable|application/x-object|application/x-msdownload|application/x-ms-installer|application/x-msi|application/x-windows-registry-main|application/x-windows-registry|application/x-sqlite3|application/x-edb|application/x-msaccess|text/xml|application/xml|application/x-dex|application/vnd\.android\.dex|application/vnd\.android\.package-archive|application/java-archive|application/x-java-archive|font/.*|application/zip|application/x-7z-compressed|application/x-rar-compressed|application/x-tar|application/x-gzip|application/x-bzip2|application/x-ms-pdb|application/x-win-lnk)$
+
+# -----------------------------------------------------------------------------
+# WHITELIST RULES (Active when embeddingFilterMode = whitelist)
+# Restricts vector embeddings strictly to high-value textual evidence, documents, emails, and chats.
+# -----------------------------------------------------------------------------
+
+# Whitelisted IPED Categories:
+# Includes: Documents, Emails, Chats, SMS, Instant Messages, Notes, Plain Texts, HTML and Web pages, PDFs, Office Documents, Audio.
+embeddingCategoryWhitelist = ^(Documents|Emails and Mailboxes|Chats|SMS Messages|Instant Messages Artifacts|Notes|Plain Texts|HTML and Web pages|PDF Documents|Office Documents|Audio)$
+
+# Whitelisted MIME Types:
+# Includes: Plain Text, HTML, XHTML, CSV, TSV, Emails, PDF, Word, Excel, PowerPoint, OpenDocument, RTF, JSON, Audio files (transcriptions).
+embeddingMimeTypeWhitelist = ^(text/plain|text/html|application/xhtml\+xml|text/csv|text/tab-separated-values|message/rfc822|application/pdf|application/msword|application/vnd\.openxmlformats-officedocument\..*|application/vnd\.ms-excel|application/vnd\.ms-powerpoint|application/vnd\.oasis\.opendocument\..*|application/rtf|application/json|audio/.*)$
+
+
+###############################################################################
+# 5. LLM ASSISTANT INTEGRATION & QUERY SETTINGS
+###############################################################################
+
+# Enables natural language questions through LLM integration (AI Assistant Chat Tab in IPED-SearchApp)
+enableLLMQueries = true
+
+# Provider for LLM chat/queries. Options: local, remote, gemini, claude
+# - local  : Ollama server (use 'local' for Ollama EVEN IF running on a remote GPU server over IP/network!
+#            'local' dynamically manages num_ctx context window tuning for Ollama without OOM/truncation).
+# - remote : Official OpenAI API (gpt-4o, gpt-4o-mini) or OpenAI-compatible APIs requiring API Key (vLLM, LMStudio, Azure).
+# - gemini : Google Gemini API (use free API Key from Google AI Studio).
+# - claude : Anthropic Claude API.
+llmProvider = local
+
+# Model name for the LLM:
+# - Ollama (local) : qwen3:4b, gemma3:4b, llama3.1:8b, phi4
+# - OpenAI (remote): gpt-4o, gpt-4o-mini, o3-mini, gpt-4-turbo
+# - Gemini (remote): gemini-2.5-flash, gemini-1.5-pro
+# - Claude (remote): claude-sonnet-4-6
+llmModel = qwen3:8b
+
+# HTTP Endpoint for LLM requests:
+# - Ollama (local) : http://localhost:11434/v1 or http://<remote-gpu-ip>:11434/v1 (e.g. http://10.89.42.211:11434/v1)
+# - OpenAI (remote): https://api.openai.com/v1 (or custom enterprise endpoint, e.g. http://<host>:8000/v1)
+# - Gemini/Claude  : leave empty to use official API endpoint automatically
+llmEndpoint = http://localhost:11434/v1
+
+# API Key for remote cloud providers (remote, gemini, claude):
+# - Required when using OpenAI (gpt-4o / gpt-4o-mini), Gemini, Claude, or authenticated APIs.
+# - Leave blank for Ollama (llmProvider = local).
+llmApiKey =

iped-app/resources/localization/iped-desktop-messages.properties

@@ -464,4 +464,35 @@ FieldValuePopupMenu.LessThan=Filter less than...
 FieldValuePopupMenu.Equals=Equals to...
 FieldValuePopupMenu.StartsWith=Starts with...
 FieldValuePopupMenu.Clear=Clear
-FieldValuePopupMenu.Filter=Filter
+FieldValuePopupMenu.Filter=Filter
+# RAG AI Assistant panel
+App.RAGAssistant=AI Assistant
+RAGAssistant.Question=Question...
+RAGAssistant.Ask=Ask
+RAGAssistant.Cancel=Cancel
+RAGAssistant.History=History
+RAGAssistant.NewChat=New Chat
+RAGAssistant.DeleteChat=Delete Chat
+RAGAssistant.RenameChat=Rename Chat
+RAGAssistant.RenameChatPrompt=Enter new chat name:
+RAGAssistant.RenameChatTitle=Rename Chat
+RAGAssistant.ScopeSelected=Restrict to selection
+RAGAssistant.ScopeSelectedTooltip=Restricts RAG search exclusively to checked, selected or filtered items in the table.
+RAGAssistant.ConfirmDeleteChat=Are you sure you want to delete this chat thread from the history?
+RAGAssistant.Sources=Sources (double-click to open)
+RAGAssistant.DeleteQuestion=Delete Question
+RAGAssistant.ConfirmDelete=Are you sure you want to delete this question from the history?
+RAGAssistant.ConfirmDeleteTitle=Confirm Deletion
+RAGAssistant.Thinking=Thinking...
+RAGAssistant.ErrorTitle=AI Assistant Error
+RAGAssistant.NotEnabled=The AI Assistant (RAG) is not enabled for this case.\nEnable 'enableRAG=true' in IPEDConfig.txt and re-index the case.
+RAGAssistant.NoContextFound=No relevant documents were found in the material to answer this question.
+RAGAssistant.Header.Id=ID
+RAGAssistant.Header.Name=Name
+RAGAssistant.Header.Score=Score
+RAGAssistant.Header.Path=Path
+iped.app.ui.SemanticSimilarityFilterer.filtererName=Semantic Similarity Filter (AI)
+FilterValue.SemanticSimilarity=Semantic Similarity (AI)
+RAGAssistant.YouLabel=You
+RAGAssistant.AssistantLabel=Assistant
+RAGAssistant.ContextWindowWarning=Warning: context window limit reached ({0} chunks omitted).

iped-app/resources/localization/iped-desktop-messages_pt_BR.properties

@@ -465,3 +465,35 @@ FieldValuePopupMenu.Equals=Igual a...
 FieldValuePopupMenu.StartsWith=Iniciam com...
 FieldValuePopupMenu.Clear=Limpar
 FieldValuePopupMenu.Filter=Filtrar
+
+# Painel do Assistente IA (RAG)
+App.RAGAssistant=Assistente IA
+RAGAssistant.Question=Pergunta...
+RAGAssistant.Ask=Perguntar
+RAGAssistant.Cancel=Cancelar
+RAGAssistant.History=Hist\u00f3rico
+RAGAssistant.NewChat=Nova Conversa
+RAGAssistant.DeleteChat=Excluir Conversa
+RAGAssistant.RenameChat=Renomear Conversa
+RAGAssistant.RenameChatPrompt=Digite o novo nome para a conversa:
+RAGAssistant.RenameChatTitle=Renomear Conversa
+RAGAssistant.ScopeSelected=Restringir a selecao
+RAGAssistant.ScopeSelectedTooltip=Restringe a busca RAG apenas aos itens marcados, selecionados na tabela ou visiveis nos filtros atuais.
+RAGAssistant.ConfirmDeleteChat=Tem certeza de que deseja excluir esta conversa do historico?
+RAGAssistant.Sources=Fontes (duplo clique para abrir)
+RAGAssistant.DeleteQuestion=Excluir Pergunta
+RAGAssistant.ConfirmDelete=Tem certeza de que deseja excluir esta pergunta do historico?
+RAGAssistant.ConfirmDeleteTitle=Confirmar Exclusao
+RAGAssistant.Thinking=Consultando IA...
+RAGAssistant.ErrorTitle=Erro no Assistente IA
+RAGAssistant.NotEnabled=O Assistente IA (RAG) nao esta habilitado para este caso.\nHabilite 'enableRAG=true' no IPEDConfig.txt e reindexe o caso.
+RAGAssistant.NoContextFound=Nenhum documento relevante foi encontrado no material para responder a esta pergunta.
+RAGAssistant.Header.Id=ID
+RAGAssistant.Header.Name=Nome
+RAGAssistant.Header.Score=Score
+RAGAssistant.Header.Path=Caminho
+iped.app.ui.SemanticSimilarityFilterer.filtererName=Filtro de Similaridade Semantica (IA)
+FilterValue.SemanticSimilarity=Similaridade Semantica (IA)
+RAGAssistant.YouLabel=Voce
+RAGAssistant.AssistantLabel=Assistente
+RAGAssistant.ContextWindowWarning=Aviso: limite da janela de contexto atingido ({0} trechos omitidos).