From bed07b762aecde0124678be8e2a1f38fee45b337 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 12:45:56 +0000 Subject: [PATCH 1/3] feat: complete GenIE web app with 3-agent pipeline, encrypted key vault and real-time monitor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the design handoff (handoff.html) on the existing Python/FastAPI backend, making GenIE fully operational end to end: Frontend (spec/web): faithful static SPA port of the prototype — model picker with key status, input/output connection panels (URL, local folder, database, REST API, upload/download), extraction prompt, output format, agent monitor with live SSE log and tabular/JSON result preview with signed download links. Agents (spec/extraction/agents): Conector (all I/O, no LLM), Localizador (LLM extraction with chunking and retry/backoff), Organizador (LLM formatting with passthrough when no format is given) and the Orchestrator driving the pipeline with real-time events. API (spec/api/v1): /models, /keys (AES-256-GCM encrypted vault, masked previews only), /uploads (sanitized, size/extension limits), /runs (create, inspect, cancel, SSE stream with Last-Event-ID replay) and /downloads (HMAC-signed, 15-minute links). Security fixes: - API keys encrypted at rest (AES-256-GCM); no endpoint returns plaintext - CORS restricted to explicit allowlist; credentials disabled - Filesystem connector confined to allowed roots (anti path traversal) - Upload filename sanitization + extension allowlist + size limits - LLM factory cache no longer embeds key material (SHA-256 digest) - Transient credentials kept in memory only, never logged or streamed - data/ fully gitignored (master key, vault, uploads, outputs) Also: current model catalog (Gemini 2.5, GPT-4o, Claude Sonnet 4.6/Haiku 4.5), content parsers for CSV/XLSX/HTML/JSON, 35 new tests (76 total, all passing), updated README and .env.example. https://claude.ai/code/session_01CKjevqGgfWLggV1DG1Tmpq --- .env.example | 35 +- .gitignore | 6 +- README.md | 236 +++---- requirements.txt | 32 +- spec/api/v1/dependencies.py | 3 +- spec/api/v1/endpoints/downloads.py | 62 ++ spec/api/v1/endpoints/extract.py | 7 +- spec/api/v1/endpoints/health.py | 1 - spec/api/v1/endpoints/keys.py | 109 +++ spec/api/v1/endpoints/models.py | 42 ++ spec/api/v1/endpoints/providers.py | 14 +- spec/api/v1/endpoints/runs.py | 210 ++++++ spec/api/v1/endpoints/uploads.py | 102 +++ spec/api/v1/router.py | 12 +- spec/core/__init__.py | 13 +- spec/core/config.py | 27 +- spec/core/logging_config.py | 1 - spec/core/security.py | 355 +++++++++- spec/extraction/agents/__init__.py | 1 + spec/extraction/agents/connector.py | 664 +++++++++++++++++ spec/extraction/agents/locator.py | 185 +++++ spec/extraction/agents/orchestrator.py | 312 ++++++++ spec/extraction/agents/organizer.py | 127 ++++ spec/extraction/engine.py | 4 +- spec/extraction/layout/fingerprint.py | 1 - spec/extraction/llm/__init__.py | 2 +- spec/extraction/llm/factory.py | 43 +- spec/extraction/parsers/__init__.py | 2 +- spec/extraction/parsers/content.py | 213 ++++++ spec/main.py | 34 +- spec/models/__init__.py | 10 +- spec/models/config.py | 3 +- spec/models/extraction.py | 2 +- spec/models/library.py | 3 +- spec/models/output.py | 1 + spec/models/provider.py | 1 + spec/models/webapp.py | 165 +++++ spec/search_library/matcher.py | 2 +- spec/web/app.js | 738 +++++++++++++++++++ spec/web/index.html | 90 +++ spec/web/styles.css | 939 +++++++++++++++++++++++++ spec/webapp/__init__.py | 1 + spec/webapp/catalog.py | 61 ++ spec/webapp/jobs.py | 199 ++++++ tests/integration/test_health.py | 10 +- tests/integration/test_webapp_api.py | 240 +++++++ tests/unit/test_connector.py | 150 ++++ tests/unit/test_security.py | 103 +++ 48 files changed, 5309 insertions(+), 264 deletions(-) create mode 100644 spec/api/v1/endpoints/downloads.py create mode 100644 spec/api/v1/endpoints/keys.py create mode 100644 spec/api/v1/endpoints/models.py create mode 100644 spec/api/v1/endpoints/runs.py create mode 100644 spec/api/v1/endpoints/uploads.py create mode 100644 spec/extraction/agents/__init__.py create mode 100644 spec/extraction/agents/connector.py create mode 100644 spec/extraction/agents/locator.py create mode 100644 spec/extraction/agents/orchestrator.py create mode 100644 spec/extraction/agents/organizer.py create mode 100644 spec/extraction/parsers/content.py create mode 100644 spec/models/webapp.py create mode 100644 spec/web/app.js create mode 100644 spec/web/index.html create mode 100644 spec/web/styles.css create mode 100644 spec/webapp/__init__.py create mode 100644 spec/webapp/catalog.py create mode 100644 spec/webapp/jobs.py create mode 100644 tests/integration/test_webapp_api.py create mode 100644 tests/unit/test_connector.py create mode 100644 tests/unit/test_security.py diff --git a/.env.example b/.env.example index f046528..e63cace 100644 --- a/.env.example +++ b/.env.example @@ -6,20 +6,37 @@ LOG_LEVEL=DEBUG API_HOST=0.0.0.0 API_PORT=8000 -# LLM Providers (Google é o provider padrão) -GOOGLE_API_KEY=your-google-ai-key-here -OPENAI_API_KEY=sk-your-key-here -ANTHROPIC_API_KEY=sk-ant-your-key-here +# Security +# Chave-mestre para cifrar API keys em repouso (AES-256-GCM). +# Gere com: openssl rand -base64 32 +# Se vazio, o GenIE gera e guarda em ./data/.master_key (modo 0600). +GENIE_MASTER_KEY= -# Provider ativo e modelo (padrão: google / gemini-1.5-pro) +# Origens permitidas para CORS (separadas por vírgula) +CORS_ORIGINS=http://localhost:8000,http://127.0.0.1:8000,http://localhost:5173 + +# Raízes extras do filesystem que o conector pode ler/gravar +# (além do home do usuário e do diretório do projeto). Separe com ":". +# ALLOWED_FS_ROOTS=/srv/dados:/mnt/exames + +# Limites de upload +MAX_UPLOAD_MB=50 +MAX_FILES_PER_UPLOAD=20 + +# LLM Providers — opcional: prefira cadastrar pela interface web, +# que armazena as chaves cifradas. Estas variáveis são fallback. +# GOOGLE_API_KEY= +# OPENAI_API_KEY= +# ANTHROPIC_API_KEY= + +# Provider ativo e modelo padrão para o endpoint /extract (legado) LLM_PROVIDER=google -# LLM_MODEL=gemini-1.5-flash # descomente para sobrescrever o modelo padrão +# LLM_MODEL=gemini-2.5-flash # descomente para sobrescrever o modelo padrão # Storage and Data DATA_DIR=./data SEARCH_LIBRARY_PATH=./data/search_library/patterns.json CONFIG_DIR=./data/configs UPLOADS_DIR=./data/uploads - -# Optional: Database (Phase 2+) -# DATABASE_URL=postgresql://user:password@localhost/genie_db +OUTPUTS_DIR=./data/outputs +DB_PATH=./data/genie.db diff --git a/.gitignore b/.gitignore index af85baa..88f23b5 100644 --- a/.gitignore +++ b/.gitignore @@ -132,7 +132,7 @@ dmypy.json # Project-specific logs/ -data/uploads/* -data/search_library/patterns.json -data/search_library/patterns.db +# Runtime data: NEVER commit (contains master key and encrypted secrets) +data/ + *.db diff --git a/README.md b/README.md index b35ada1..254fd9f 100644 --- a/README.md +++ b/README.md @@ -1,160 +1,146 @@ -# GENIE - Generic Extractor of Information Engine +# GenIE — Generic Extractor of Information Engine -A Python framework for intelligent data extraction using LLMs. +Framework Python para extração inteligente de dados com LLMs, orquestrado por +três agentes cooperativos e operável por uma interface web completa. -## Quick Start +``` +Conector (I/O) → Localizador (extração via LLM) → Organizador (formato) → Conector (entrega) +``` -### Prerequisites -- Python 3.11+ -- Poetry (or pip) +## Interface Web -### Installation +A SPA embutida (servida pelo próprio FastAPI em `http://localhost:8000`) permite: -**Using Poetry (recommended):** -```bash -poetry install -poetry shell -``` +1. **Modelo de IA** — escolher Gemini / GPT / Claude e cadastrar a API Key + (cifrada com AES-256-GCM no servidor; nunca volta ao navegador). +2. **Entrada** — URL, pasta local, banco de dados, API REST ou upload de arquivos + (PDF, CSV, XLSX, JSON, TXT, HTML…). +3. **O que extrair** — instrução em linguagem natural para o Localizador. +4. **Saída** — webhook, pasta local, banco SQLite, API REST (ex.: TabEx) ou download. +5. **Formato da saída** — instrução em linguagem natural para o Organizador. -**Using pip:** -```bash -pip install -r requirements.txt -``` +O monitor à direita mostra os 3 agentes com progresso, log em tempo real (SSE) +e a prévia tabular/JSON do resultado, com links de download assinados. -### Configuration +## Quick Start -1. Copy `.env.example` to `.env`: ```bash -cp .env.example .env -``` - -2. Add your API keys to `.env`: -``` -ANTHROPIC_API_KEY=sk-ant-your-key-here -OPENAI_API_KEY=sk-your-key-here -``` +# 1. Instalar dependências (Python 3.11+) +pip install -r requirements.txt -### Running the Server +# 2. (Opcional) Configurar ambiente +cp .env.example .env +# Gere a chave-mestre para produção: openssl rand -base64 32 → GENIE_MASTER_KEY +# Em desenvolvimento o GenIE gera uma automaticamente em ./data/.master_key -```bash -uvicorn spec.main:app --reload --port 8000 +# 3. Rodar +uvicorn spec.main:app --port 8000 ``` -The API will be available at `http://localhost:8000` +Abra **http://localhost:8000** — cadastre a API Key do provedor (ex.: Google +Gemini), envie um arquivo, descreva o que extrair e clique em *Enviar requisição*. -- Docs: http://localhost:8000/docs -- ReDoc: http://localhost:8000/redoc +- Web app: http://localhost:8000 +- Docs da API: http://localhost:8000/docs - Health: http://localhost:8000/api/v1/health -## Project Structure +## Segurança + +- **API Keys nunca atravessam o navegador**: são enviadas uma única vez, + cifradas com **AES-256-GCM** (chave-mestre via `GENIE_MASTER_KEY` ou arquivo + `./data/.master_key`, modo 0600) e armazenadas em SQLite. Nenhum endpoint + devolve a chave — apenas `has_key` e um preview mascarado. +- **Credenciais transitórias** (senha de banco, token de API por execução) + ficam apenas em memória e nunca aparecem em logs, eventos SSE ou resultados. +- **Filesystem com allowlist**: o conector só lê/grava sob o home do usuário, + o diretório do projeto e raízes extras de `ALLOWED_FS_ROOTS` (anti path traversal). +- **Uploads**: nomes sanitizados, allowlist de extensões, limites de tamanho + (`MAX_UPLOAD_MB`) e quantidade (`MAX_FILES_PER_UPLOAD`). +- **Downloads assinados**: links HMAC-SHA256 com validade de 15 minutos. +- **CORS** restrito a uma allowlist explícita (`CORS_ORIGINS`). + +## API da aplicação web + +| Método | Rota | Descrição | +|---|---|---| +| `GET` | `/api/v1/models` | Catálogo de modelos + `has_key` por provedor | +| `GET` | `/api/v1/keys` | Provedores com chave (apenas preview mascarado) | +| `POST` | `/api/v1/keys` | `{provider, key, validate_key}` → valida e cifra | +| `DELETE` | `/api/v1/keys/{provider}` | Remove a chave | +| `POST` | `/api/v1/uploads` | multipart → `{upload_id, files}` | +| `POST` | `/api/v1/runs` | Cria execução → `{job_id}` | +| `GET` | `/api/v1/runs/{id}` | Estado atual + resultado | +| `GET` | `/api/v1/runs/{id}/events` | SSE com eventos dos agentes (suporta `Last-Event-ID`) | +| `POST` | `/api/v1/runs/{id}/cancel` | Interrompe a execução | +| `GET` | `/api/v1/downloads/{id}/{arquivo}` | Artefatos com link assinado | + +Endpoints do framework (extração programática): `POST /api/v1/extract`, +`GET/POST /api/v1/providers*` — ver `/docs`. + +### Exemplo de execução via API -``` -spec/ -├── api/ # REST API endpoints -│ └── v1/ -│ ├── endpoints/ # Endpoint implementations -│ ├── router.py # Route aggregator -│ └── dependencies.py # Dependency injection -├── core/ # Core infrastructure -│ ├── config.py # Settings management -│ ├── exceptions.py # Custom exceptions -│ ├── logging_config.py # Logging setup -│ └── security.py # Security utilities -├── models/ # Pydantic data models -├── extraction/ # Extraction engine -│ ├── engine.py # Main orchestrator -│ ├── llm/ # LLM providers -│ ├── parsers/ # Content parsers -│ └── layout/ # Layout fingerprinting -├── search_library/ # Pattern storage -├── output/ # Output management -└── main.py # FastAPI entry point +```bash +# Upload +UP=$(curl -s -F "files=@exames.pdf" localhost:8000/api/v1/uploads | jq -r .upload_id) + +# Run +curl -s -X POST localhost:8000/api/v1/runs -H 'Content-Type: application/json' -d "{ + \"model_id\": \"gemini-2.5-flash\", + \"input\": {\"type\": \"upload\", \"upload_id\": \"$UP\"}, + \"prompt\": \"Extraia Data, Nome do Exame, Resultado e Valor de Referência\", + \"output\": {\"type\": \"download\"}, + \"format\": \"Um registro por exame com data ISO-8601\" +}" ``` -## API Endpoints +## Conectores -### Health Check -```http -GET /api/v1/health -``` +| Tipo | Entrada | Saída | +|---|---|---| +| URL | HTML/PDF/JSON públicos; links de arquivo do Google Drive | POST webhook | +| Pasta local | varredura recursiva (allowlist de raízes) | `output.json` + `output.csv` | +| Banco de dados | SQLite nativo; Postgres/MySQL via SQLAlchemy opcional | SQLite (cria/evolui tabela) | +| API REST | GET com Bearer token | POST com Bearer (lote ou por registro) | +| Upload / Download | multipart seguro | links assinados (15 min) | -### Extract Data -```http -POST /api/v1/extract -Content-Type: application/json - -{ - "config_id": "config_001", - "source": { - "type": "text", - "content": "Document content here..." - }, - "force_llm": false, - "options": { - "auto_create_patterns": true - } -} -``` +## Integração TabEx -## Testing +O GenIE opera de forma independente e como serviço para outros apps. +Para entregar dados ao TabEx, use saída **API REST** apontando para o endpoint +do TabEx com o token de acesso, e descreva o body esperado no campo +*Formato da saída* — o Organizador monta os payloads e o Conector entrega. -Run all tests: -```bash -pytest -``` +## Estrutura -Run specific test file: -```bash -pytest tests/unit/test_models.py -v ``` - -Run with coverage: -```bash -pytest --cov=spec --cov-report=html +spec/ +├── api/v1/endpoints/ # extract, providers, models, keys, uploads, runs, downloads +├── core/ # config, exceptions, security (AES-256-GCM), logging +├── extraction/ +│ ├── agents/ # connector, locator, organizer, orchestrator +│ ├── llm/ # factory + providers (Google, OpenAI, Anthropic) +│ ├── parsers/ # pdf, text, content (csv/xlsx/html/json) +│ └── layout/ # fingerprint +├── models/ # Pydantic v2 (extraction, webapp, …) +├── search_library/ # padrões reutilizáveis (JSON) +├── webapp/ # catálogo de modelos + gestor de jobs/SSE +└── web/ # SPA (index.html, styles.css, app.js) ``` -## Development - -### Code Style -- **Formatter:** Black (88 chars line length) -- **Linter:** Ruff -- **Type Checker:** Mypy - -Format code: -```bash -black spec/ tests/ -ruff check . --fix -``` +## Testes -Type checking: ```bash -mypy spec/ +pytest # suíte completa +pytest tests/unit -v # unidade +pytest --cov=spec # cobertura ``` -## Documentation +## Documentação -- [Architecture Guide](./docs/guides/GENIE-ARCHITECTURE.md) -- [Phase 1 Plan](./docs/guides/PHASE-1-PLAN.md) -- [Specification v2](./docs/guides/GENIE-SPEC-v2.md) +- [Arquitetura](./docs/guides/GENIE-ARCHITECTURE.md) +- [Especificação v2](./docs/guides/GENIE-SPEC-v2.md) +- [Exemplos](./docs/examples/GENIE-EXAMPLES.md) -## License +## Licença MIT - -## Project Status - -**Phase 1: MVP Core** - In Development - -- ✓ Project setup and tooling -- ✓ Core infrastructure -- ✓ Pydantic models -- ✓ LLM provider interface (Anthropic) -- ✓ Text and PDF parsers -- ✓ Layout fingerprinting -- ✓ Search library (JSON storage) -- ✓ Extraction engine -- ✓ REST API endpoints -- ⏳ Comprehensive testing -- ⏳ End-to-end validation - -See [PHASE-1-PLAN.md](./docs/guides/PHASE-1-PLAN.md) for detailed roadmap. diff --git a/requirements.txt b/requirements.txt index 8af782d..70815e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,21 @@ # Core dependencies -fastapi==0.110.0 -uvicorn[standard]==0.27.0 -pydantic==2.6.0 -pydantic-settings==2.2.0 -anthropic==0.18.0 -openai==1.12.0 +fastapi>=0.110.0 +uvicorn[standard]>=0.27.0 +pydantic>=2.6.0 +pydantic-settings>=2.2.0 +anthropic>=0.40.0 +openai>=1.50.0 google-genai>=1.0.0 -python-multipart==0.0.9 -PyPDF2==3.0.0 -cryptography==42.0.0 +python-multipart>=0.0.9 +PyPDF2>=3.0.0 +cryptography>=41.0.0 +httpx>=0.27.0 +openpyxl>=3.1.0 # Development dependencies -pytest==8.0.0 -pytest-asyncio==0.23.0 -pytest-cov==4.1.0 -ruff==0.2.0 -black==24.2.0 -mypy==1.8.0 +pytest>=8.2.0 +pytest-asyncio>=0.24.0 +pytest-cov>=4.1.0 +ruff>=0.2.0 +black>=24.2.0 +mypy>=1.8.0 diff --git a/spec/api/v1/dependencies.py b/spec/api/v1/dependencies.py index 53b76a9..c433942 100644 --- a/spec/api/v1/dependencies.py +++ b/spec/api/v1/dependencies.py @@ -4,7 +4,6 @@ into endpoint handlers using FastAPI's Depends() mechanism. """ -from typing import Optional import logging from fastapi import Depends @@ -12,8 +11,8 @@ from spec.core.config import Settings, get_settings from spec.extraction.engine import ExtractionEngine from spec.extraction.llm.factory import LLMProviderFactory -from spec.search_library.json_storage import JSONStorage from spec.output.manager import OutputManager +from spec.search_library.json_storage import JSONStorage def get_app_settings() -> Settings: diff --git a/spec/api/v1/endpoints/downloads.py b/spec/api/v1/endpoints/downloads.py new file mode 100644 index 0000000..9553134 --- /dev/null +++ b/spec/api/v1/endpoints/downloads.py @@ -0,0 +1,62 @@ +"""Signed, short-lived download endpoint for run artifacts.""" + +import logging +import re +import time +from pathlib import Path + +from fastapi import APIRouter, HTTPException +from fastapi.responses import FileResponse + +from spec.core.config import get_settings +from spec.core.security import get_cipher + +logger = logging.getLogger(__name__) + +router = APIRouter() + +_JOB_ID_RE = re.compile(r"^genie-[a-f0-9]{10}$") +_ARTIFACT_RE = re.compile(r"^output\.(json|csv)$") + +_MEDIA_TYPES = {".json": "application/json", ".csv": "text/csv"} + + +@router.get("/{job_id}/{artifact}") +async def download_artifact( + job_id: str, artifact: str, exp: int, sig: str +) -> FileResponse: + """Serve a run artifact when the signed link is valid and unexpired. + + Args: + job_id: Run identifier + artifact: Artifact filename (output.json / output.csv) + exp: Expiration timestamp (unix seconds) + sig: HMAC-SHA256 signature of "job_id:artifact:exp" + + Returns: + FileResponse: The artifact + + Raises: + HTTPException: 403 for invalid/expired links, 404 when missing + """ + + if not _JOB_ID_RE.match(job_id) or not _ARTIFACT_RE.match(artifact): + raise HTTPException(status_code=403, detail="Link de download inválido") + + if exp < int(time.time()): + raise HTTPException(status_code=403, detail="Link de download expirado") + + if not get_cipher().verify(f"{job_id}:{artifact}:{exp}", sig): + raise HTTPException(status_code=403, detail="Assinatura de download inválida") + + settings = get_settings() + path = (Path(settings.outputs_dir) / job_id / artifact).resolve() + outputs_root = Path(settings.outputs_dir).resolve() + if not path.is_relative_to(outputs_root) or not path.is_file(): + raise HTTPException(status_code=404, detail="Arquivo não encontrado") + + return FileResponse( + path, + media_type=_MEDIA_TYPES.get(path.suffix, "application/octet-stream"), + filename=f"genie-{job_id}{path.suffix}", + ) diff --git a/spec/api/v1/endpoints/extract.py b/spec/api/v1/endpoints/extract.py index 6ba47a7..1a06a23 100644 --- a/spec/api/v1/endpoints/extract.py +++ b/spec/api/v1/endpoints/extract.py @@ -1,8 +1,9 @@ """Document extraction endpoint.""" -from fastapi import APIRouter, Depends import logging +from fastapi import APIRouter, Depends + from spec.api.v1.dependencies import get_extraction_engine from spec.extraction.engine import ExtractionEngine from spec.models.extraction import ExtractionRequest, ExtractionResponse @@ -51,9 +52,7 @@ async def extract_data( } """ - logger.info( - f"Processing extraction request for config: {request.config_id}" - ) + logger.info(f"Processing extraction request for config: {request.config_id}") # Call the extraction engine response = await engine.extract(request) diff --git a/spec/api/v1/endpoints/health.py b/spec/api/v1/endpoints/health.py index 0285d4e..d556296 100644 --- a/spec/api/v1/endpoints/health.py +++ b/spec/api/v1/endpoints/health.py @@ -1,7 +1,6 @@ """Health check endpoint for API status verification.""" from datetime import datetime -from typing import Optional from fastapi import APIRouter, Depends diff --git a/spec/api/v1/endpoints/keys.py b/spec/api/v1/endpoints/keys.py new file mode 100644 index 0000000..ec48d9b --- /dev/null +++ b/spec/api/v1/endpoints/keys.py @@ -0,0 +1,109 @@ +"""Encrypted API key management endpoints. + +Security contract: +- Keys are encrypted (AES-256-GCM) before touching disk. +- No endpoint ever returns a key in plaintext — only masked previews. +- Optional live validation performs one minimal LLM call before saving. +""" + +import logging + +from fastapi import APIRouter, HTTPException + +from spec.core.exceptions import InvalidConfig, LLMProviderError +from spec.core.security import get_key_vault +from spec.extraction.llm.factory import LLMProviderFactory +from spec.models.webapp import KeyInfo, KeyRequest +from spec.webapp.catalog import MODELS + +logger = logging.getLogger(__name__) + +router = APIRouter() + +_KNOWN_PROVIDERS = {m["provider"] for m in MODELS} + + +@router.get("", response_model=list[KeyInfo]) +async def list_keys() -> list[KeyInfo]: + """List providers that have a stored key (masked previews only). + + Returns: + list[KeyInfo]: Provider + masked preview pairs + """ + + vault = get_key_vault() + return [ + KeyInfo(provider=provider, masked=vault.masked(provider) or "") + for provider in sorted(_KNOWN_PROVIDERS) + if vault.has(provider) + ] + + +@router.post("", response_model=KeyInfo) +async def store_key(request: KeyRequest) -> KeyInfo: + """Validate (optionally) and store a provider API key, encrypted. + + Args: + request: Provider, plaintext key and validation flag + + Returns: + KeyInfo: Masked confirmation (the key is never echoed back) + + Raises: + HTTPException: 400 for unknown provider or failed validation + """ + + provider_name = request.provider.lower().strip() + if provider_name not in _KNOWN_PROVIDERS: + raise HTTPException( + status_code=400, + detail=f"Provedor desconhecido '{request.provider}'. " + f"Suportados: {sorted(_KNOWN_PROVIDERS)}", + ) + + if request.validate_key: + try: + provider = LLMProviderFactory().get_provider( + provider_name=provider_name, + api_key=request.key, + ) + await provider.extract( + content="ping", + schema={"ok": True}, + instructions='Responda apenas com o JSON {"ok": true}.', + ) + except (LLMProviderError, InvalidConfig) as e: + logger.warning("Key validation failed for %s: %s", provider_name, e) + raise HTTPException( + status_code=400, + detail=f"A chave informada foi recusada pelo provedor '{provider_name}'. " + "Verifique a chave e tente novamente.", + ) + except Exception as e: # noqa: BLE001 - network/SDK failures + logger.error("Unexpected validation error for %s: %s", provider_name, e) + raise HTTPException( + status_code=400, + detail=f"Não foi possível validar a chave do provedor '{provider_name}': {e}", + ) + + masked = get_key_vault().store(provider_name, request.key) + return KeyInfo(provider=provider_name, masked=masked) + + +@router.delete("/{provider}") +async def delete_key(provider: str) -> dict[str, bool]: + """Remove a stored provider key. + + Args: + provider: Provider name + + Returns: + dict: {"deleted": bool} + """ + + deleted = get_key_vault().delete(provider.lower().strip()) + if not deleted: + raise HTTPException( + status_code=404, detail=f"Sem chave armazenada para '{provider}'" + ) + return {"deleted": True} diff --git a/spec/api/v1/endpoints/models.py b/spec/api/v1/endpoints/models.py new file mode 100644 index 0000000..ccc5141 --- /dev/null +++ b/spec/api/v1/endpoints/models.py @@ -0,0 +1,42 @@ +"""Model catalog endpoint: lists selectable models with key status.""" + +import logging + +from fastapi import APIRouter + +from spec.core.security import get_key_vault +from spec.models.webapp import ModelInfo +from spec.webapp.catalog import MODELS + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +@router.get("", response_model=list[ModelInfo]) +async def list_models() -> list[ModelInfo]: + """List the model catalog with per-provider key status. + + The API key itself is never returned — only a boolean and a masked + preview (first 4 characters). + + Returns: + list[ModelInfo]: Catalog entries with has_key/masked_key + """ + + vault = get_key_vault() + result = [] + for model in MODELS: + masked = vault.masked(model["provider"]) + result.append( + ModelInfo( + id=model["id"], + provider=model["provider"], + provider_label=model["provider_label"], + label=model["label"], + note=model["note"], + has_key=masked is not None, + masked_key=masked, + ) + ) + return result diff --git a/spec/api/v1/endpoints/providers.py b/spec/api/v1/endpoints/providers.py index 8f497d2..17ee1d4 100644 --- a/spec/api/v1/endpoints/providers.py +++ b/spec/api/v1/endpoints/providers.py @@ -8,7 +8,11 @@ from spec.api.v1.dependencies import get_llm_factory from spec.core.exceptions import InvalidConfig, LLMProviderError from spec.extraction.llm.factory import LLMProviderFactory -from spec.models.provider import ProviderConfigRequest, ProviderConfigResponse, ProviderInfo +from spec.models.provider import ( + ProviderConfigRequest, + ProviderConfigResponse, + ProviderInfo, +) logger = logging.getLogger(__name__) @@ -139,7 +143,9 @@ async def configure_provider( detail=f"API key validation failed for '{request.provider}': {e}", ) except Exception as e: - logger.error(f"Unexpected error validating {request.provider}: {e}", exc_info=True) + logger.error( + f"Unexpected error validating {request.provider}: {e}", exc_info=True + ) raise HTTPException( status_code=400, detail=f"Could not validate API key for '{request.provider}': {e}", @@ -155,7 +161,9 @@ async def configure_provider( meta = next(m for m in provider_metadata if m["name"] == request.provider) active_model = request.model or meta["default_model"] - logger.info(f"Provider configured successfully: {request.provider} / {active_model}") + logger.info( + f"Provider configured successfully: {request.provider} / {active_model}" + ) return ProviderConfigResponse( success=True, diff --git a/spec/api/v1/endpoints/runs.py b/spec/api/v1/endpoints/runs.py new file mode 100644 index 0000000..0b3a8f9 --- /dev/null +++ b/spec/api/v1/endpoints/runs.py @@ -0,0 +1,210 @@ +"""Run lifecycle endpoints: create, inspect, cancel and stream via SSE.""" + +import asyncio +import json +import logging +from typing import AsyncGenerator, Optional + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import StreamingResponse + +from spec.core.config import get_settings +from spec.core.security import get_key_vault +from spec.extraction.agents.orchestrator import Orchestrator +from spec.models.webapp import RunCreated, RunInfo, RunRequest +from spec.webapp.catalog import find_model +from spec.webapp.jobs import get_job_manager + +logger = logging.getLogger(__name__) + +router = APIRouter() + +_HEARTBEAT_SECONDS = 15.0 + + +def _has_provider_key(provider: str) -> bool: + """Check key availability: encrypted vault first, env fallback. + + Args: + provider: Provider name + + Returns: + bool: True if a key is available + """ + + if get_key_vault().has(provider): + return True + settings = get_settings() + env_keys = { + "google": settings.google_api_key, + "openai": settings.openai_api_key, + "anthropic": settings.anthropic_api_key, + } + return bool(env_keys.get(provider)) + + +@router.post("", response_model=RunCreated, status_code=201) +async def create_run(request: RunRequest) -> RunCreated: + """Create and start an extraction run. + + Args: + request: Model, input, prompt, output and format + + Returns: + RunCreated: Job id with initial status + + Raises: + HTTPException: 400 for unknown model or missing key/input + """ + + model = find_model(request.model_id) + if model is None: + raise HTTPException( + status_code=400, detail=f"Modelo desconhecido: {request.model_id}" + ) + + if not _has_provider_key(model["provider"]): + raise HTTPException( + status_code=400, + detail=f"Configure uma API Key para {model['provider_label']} antes de executar.", + ) + + if request.input.type == "upload" and not request.input.upload_id: + raise HTTPException( + status_code=400, detail="Envie os arquivos antes de executar (upload)." + ) + if request.input.type != "upload" and not request.input.target.strip(): + raise HTTPException( + status_code=400, detail="Informe o endereço da origem dos dados." + ) + if request.output.type not in ("download",) and not request.output.target.strip(): + raise HTTPException( + status_code=400, detail="Informe o endereço do destino dos dados." + ) + + manager = get_job_manager() + job = manager.create(request) + orchestrator = Orchestrator(manager) + job.task = asyncio.create_task(orchestrator.run_job(job)) + + logger.info( + "Run %s created (model=%s, in=%s, out=%s)", + job.id, + request.model_id, + request.input.type, + request.output.type, + ) + return RunCreated(job_id=job.id, status=job.status) + + +@router.get("/{job_id}", response_model=RunInfo) +async def get_run(job_id: str) -> RunInfo: + """Return the current state of a run. + + Args: + job_id: Run identifier + + Returns: + RunInfo: Status, event count, result/error + """ + + job = get_job_manager().get(job_id) + if job is None: + raise HTTPException(status_code=404, detail=f"Job '{job_id}' não encontrado") + return RunInfo( + job_id=job.id, + status=job.status, + model_id=job.request.model_id, + events=len(job.events), + result=job.result, + error=job.error, + ) + + +@router.post("/{job_id}/cancel", response_model=RunInfo) +async def cancel_run(job_id: str) -> RunInfo: + """Cancel a running job (also aborts in-flight LLM calls). + + Args: + job_id: Run identifier + + Returns: + RunInfo: Updated state + """ + + manager = get_job_manager() + job = manager.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail=f"Job '{job_id}' não encontrado") + + if job.task and not job.task.done(): + job.task.cancel() + try: + await job.task + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + + return RunInfo( + job_id=job.id, + status=job.status, + model_id=job.request.model_id, + events=len(job.events), + result=job.result, + error=job.error, + ) + + +@router.get("/{job_id}/events") +async def stream_events(job_id: str, request: Request) -> StreamingResponse: + """Stream run events as Server-Sent Events. + + Replays history when ``Last-Event-ID`` is provided, then follows live + events until the run finishes. Sends heartbeat comments to keep proxies + from closing the connection. + + Args: + job_id: Run identifier + request: Incoming request (for Last-Event-ID and disconnects) + + Returns: + StreamingResponse: text/event-stream + """ + + manager = get_job_manager() + job = manager.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail=f"Job '{job_id}' não encontrado") + + last_id: Optional[int] = None + header = request.headers.get("last-event-id") + if header and header.isdigit(): + last_id = int(header) + + async def event_source() -> AsyncGenerator[str, None]: + stream = manager.stream(job, last_event_id=last_id) + iterator = stream.__aiter__() + try: + while True: + if await request.is_disconnected(): + return + try: + event = await asyncio.wait_for( + iterator.__anext__(), timeout=_HEARTBEAT_SECONDS + ) + except asyncio.TimeoutError: + yield ": ping\n\n" + continue + except StopAsyncIteration: + return + payload = json.dumps( + event.model_dump(exclude_none=True), ensure_ascii=False + ) + yield f"id: {event.seq}\ndata: {payload}\n\n" + finally: + await stream.aclose() + + return StreamingResponse( + event_source(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) diff --git a/spec/api/v1/endpoints/uploads.py b/spec/api/v1/endpoints/uploads.py new file mode 100644 index 0000000..67b3653 --- /dev/null +++ b/spec/api/v1/endpoints/uploads.py @@ -0,0 +1,102 @@ +"""Multipart upload endpoint with sanitization and limits.""" + +import logging +import re +from pathlib import Path + +from fastapi import APIRouter, HTTPException, UploadFile + +from spec.core.config import get_settings +from spec.extraction.agents.connector import new_upload_id +from spec.extraction.parsers.content import SUPPORTED_EXTENSIONS +from spec.models.webapp import UploadedFile, UploadResponse + +logger = logging.getLogger(__name__) + +router = APIRouter() + +_SAFE_NAME_RE = re.compile(r"[^\w.\- ]", re.UNICODE) +_CHUNK = 1024 * 1024 + + +def _sanitize_filename(raw: str, index: int) -> str: + """Build a safe filename from a client-provided name. + + Args: + raw: Original filename from the multipart part + index: Position in the batch (fallback naming) + + Returns: + str: Safe basename without path separators + """ + + name = Path(raw or f"arquivo-{index}").name + name = _SAFE_NAME_RE.sub("_", name).strip(". ") + return name or f"arquivo-{index}" + + +@router.post("", response_model=UploadResponse) +async def upload_files(files: list[UploadFile]) -> UploadResponse: + """Receive files for a future run and store them under a batch id. + + Enforces per-file size limits, a batch count limit and an extension + allowlist; filenames are sanitized against path traversal. + + Args: + files: Multipart files + + Returns: + UploadResponse: Batch id and accepted file metadata + + Raises: + HTTPException: 400/413 for invalid or oversized uploads + """ + + settings = get_settings() + + if not files: + raise HTTPException(status_code=400, detail="Nenhum arquivo enviado") + if len(files) > settings.max_files_per_upload: + raise HTTPException( + status_code=400, + detail=f"Máximo de {settings.max_files_per_upload} arquivos por envio", + ) + + upload_id = new_upload_id() + upload_dir = Path(settings.uploads_dir) / upload_id + upload_dir.mkdir(parents=True, exist_ok=True) + + max_bytes = settings.max_upload_mb * 1024 * 1024 + accepted: list[UploadedFile] = [] + + for index, upload in enumerate(files, 1): + name = _sanitize_filename(upload.filename or "", index) + suffix = Path(name).suffix.lower() + if suffix not in SUPPORTED_EXTENSIONS: + raise HTTPException( + status_code=400, + detail=f"Extensão não suportada: '{name}'. " + f"Aceitas: {', '.join(sorted(SUPPORTED_EXTENSIONS))}", + ) + + destination = upload_dir / name + if destination.exists(): + destination = upload_dir / f"{Path(name).stem}-{index}{suffix}" + + size = 0 + with open(destination, "wb") as out: + while chunk := await upload.read(_CHUNK): + size += len(chunk) + if size > max_bytes: + out.close() + destination.unlink(missing_ok=True) + raise HTTPException( + status_code=413, + detail=f"'{name}' excede o limite de {settings.max_upload_mb} MB", + ) + out.write(chunk) + + accepted.append(UploadedFile(name=destination.name, size=size)) + + logger.info("Upload %s: %d file(s) accepted", upload_id, len(accepted)) + return UploadResponse(upload_id=upload_id, files=accepted) diff --git a/spec/api/v1/router.py b/spec/api/v1/router.py index f4a78ff..61166de 100644 --- a/spec/api/v1/router.py +++ b/spec/api/v1/router.py @@ -6,9 +6,14 @@ from fastapi import APIRouter -from spec.api.v1.endpoints.health import router as health_router +from spec.api.v1.endpoints.downloads import router as downloads_router from spec.api.v1.endpoints.extract import router as extract_router +from spec.api.v1.endpoints.health import router as health_router +from spec.api.v1.endpoints.keys import router as keys_router +from spec.api.v1.endpoints.models import router as models_router from spec.api.v1.endpoints.providers import router as providers_router +from spec.api.v1.endpoints.runs import router as runs_router +from spec.api.v1.endpoints.uploads import router as uploads_router # Create main v1 router router = APIRouter() @@ -17,3 +22,8 @@ router.include_router(health_router, tags=["health"]) router.include_router(extract_router, tags=["extraction"]) router.include_router(providers_router, prefix="/providers", tags=["providers"]) +router.include_router(models_router, prefix="/models", tags=["models"]) +router.include_router(keys_router, prefix="/keys", tags=["keys"]) +router.include_router(uploads_router, prefix="/uploads", tags=["uploads"]) +router.include_router(runs_router, prefix="/runs", tags=["runs"]) +router.include_router(downloads_router, prefix="/downloads", tags=["downloads"]) diff --git a/spec/core/__init__.py b/spec/core/__init__.py index 393ca04..63555da 100644 --- a/spec/core/__init__.py +++ b/spec/core/__init__.py @@ -2,15 +2,15 @@ from spec.core.config import Settings, get_settings from spec.core.exceptions import ( + ExtractionFailed, GenieException, + InvalidConfig, LayoutNotRecognized, - ExtractionFailed, LLMProviderError, - InvalidConfig, StorageError, ) -from spec.core.logging_config import setup_logging, get_logger -from spec.core.security import SecureKeyStore +from spec.core.logging_config import get_logger, setup_logging +from spec.core.security import KeyVault, SecretCipher, get_cipher, get_key_vault __all__ = [ "Settings", @@ -23,5 +23,8 @@ "StorageError", "setup_logging", "get_logger", - "SecureKeyStore", + "KeyVault", + "SecretCipher", + "get_cipher", + "get_key_vault", ] diff --git a/spec/core/config.py b/spec/core/config.py index 381a5df..95c0678 100644 --- a/spec/core/config.py +++ b/spec/core/config.py @@ -1,7 +1,8 @@ """Application configuration using Pydantic Settings.""" -from typing import Optional from pathlib import Path +from typing import Optional + from pydantic import model_validator from pydantic_settings import BaseSettings @@ -35,6 +36,19 @@ class Settings(BaseSettings): search_library_path: str = "./data/search_library/patterns.json" config_dir: str = "./data/configs" uploads_dir: str = "./data/uploads" + outputs_dir: str = "./data/outputs" + db_path: str = "./data/genie.db" + + master_key: Optional[str] = None # env: GENIE_MASTER_KEY (base64, 32 bytes) + genie_master_key: Optional[str] = None # alias accepted for convenience + + cors_origins: str = ( + "http://localhost:8000,http://127.0.0.1:8000,http://localhost:5173" + ) + allowed_fs_roots: Optional[str] = None # extra roots, separated by os.pathsep + max_upload_mb: int = 50 + max_files_per_upload: int = 20 + download_link_ttl_seconds: int = 900 anthropic_api_key: Optional[str] = None openai_api_key: Optional[str] = None @@ -43,14 +57,21 @@ class Settings(BaseSettings): llm_provider: str = "google" llm_model: Optional[str] = None - model_config = {"env_file": ".env", "env_file_encoding": "utf-8", "case_sensitive": False} + model_config = { + "env_file": ".env", + "env_file_encoding": "utf-8", + "case_sensitive": False, + } @model_validator(mode="after") def create_directories(self) -> "Settings": - """Ensure required directories exist after initialization.""" + """Ensure required directories exist and normalize aliases.""" + if self.genie_master_key and not self.master_key: + self.master_key = self.genie_master_key Path(self.data_dir).mkdir(parents=True, exist_ok=True) Path(self.config_dir).mkdir(parents=True, exist_ok=True) Path(self.uploads_dir).mkdir(parents=True, exist_ok=True) + Path(self.outputs_dir).mkdir(parents=True, exist_ok=True) Path(self.search_library_path).parent.mkdir(parents=True, exist_ok=True) return self diff --git a/spec/core/logging_config.py b/spec/core/logging_config.py index b7a8594..4057fa1 100644 --- a/spec/core/logging_config.py +++ b/spec/core/logging_config.py @@ -3,7 +3,6 @@ import logging import sys from pathlib import Path -from typing import Optional def setup_logging(log_level: str = "INFO") -> logging.Logger: diff --git a/spec/core/security.py b/spec/core/security.py index 062c144..5137404 100644 --- a/spec/core/security.py +++ b/spec/core/security.py @@ -1,62 +1,349 @@ -"""Security utilities for API key management and encryption. +"""Security utilities: AES-256-GCM encryption and encrypted API key vault. -Note: This is a placeholder implementation. Full encryption will be implemented -in Phase 5 (Production). +The master key is resolved in this order: +1. ``GENIE_MASTER_KEY`` environment variable (base64, 32 bytes) +2. ``{data_dir}/.master_key`` file (auto-generated on first run, mode 0600) + +API keys are encrypted at rest in SQLite and are NEVER returned in +plaintext by any API endpoint — only a masked preview (first 4 chars). """ +import base64 +import hashlib +import hmac +import logging +import os +import sqlite3 +import threading +import time +from pathlib import Path from typing import Optional +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from spec.core.config import get_settings +from spec.core.exceptions import InvalidConfig, StorageError + +logger = logging.getLogger(__name__) + +_MASTER_KEY_BYTES = 32 +_NONCE_BYTES = 12 + + +def _load_or_create_master_key(data_dir: str, env_value: Optional[str]) -> bytes: + """Resolve the 32-byte master key from env or key file. + + Args: + data_dir: Data directory for the fallback key file + env_value: Value of GENIE_MASTER_KEY env var (base64) or None + + Returns: + bytes: 32-byte master key + + Raises: + InvalidConfig: If the env value is malformed + """ + + if env_value: + try: + key = base64.b64decode(env_value) + except Exception as e: + raise InvalidConfig(f"GENIE_MASTER_KEY is not valid base64: {e}") + if len(key) != _MASTER_KEY_BYTES: + raise InvalidConfig( + "GENIE_MASTER_KEY must decode to exactly 32 bytes " + "(generate with: openssl rand -base64 32)" + ) + return key + + key_path = Path(data_dir) / ".master_key" + if key_path.exists(): + key = key_path.read_bytes() + if len(key) != _MASTER_KEY_BYTES: + raise InvalidConfig(f"Corrupt master key file: {key_path}") + return key + + key = os.urandom(_MASTER_KEY_BYTES) + key_path.parent.mkdir(parents=True, exist_ok=True) + key_path.write_bytes(key) + os.chmod(key_path, 0o600) + logger.warning( + "Generated new master key at %s (mode 0600). " + "Set GENIE_MASTER_KEY env var for production deployments.", + key_path, + ) + return key + + +class SecretCipher: + """AES-256-GCM encryption helper bound to the application master key.""" + + def __init__(self, master_key: bytes) -> None: + """Initialize the cipher. + + Args: + master_key: 32-byte symmetric key + """ + + if len(master_key) != _MASTER_KEY_BYTES: + raise InvalidConfig("Master key must be 32 bytes") + self._aesgcm = AESGCM(master_key) + self._master_key = master_key + + def encrypt(self, plaintext: str) -> bytes: + """Encrypt a string, returning nonce-prefixed ciphertext. + + Args: + plaintext: Secret to encrypt + + Returns: + bytes: nonce (12 bytes) + ciphertext + GCM tag + """ + + nonce = os.urandom(_NONCE_BYTES) + return nonce + self._aesgcm.encrypt(nonce, plaintext.encode("utf-8"), None) + + def decrypt(self, blob: bytes) -> str: + """Decrypt nonce-prefixed ciphertext back to a string. + + Args: + blob: nonce + ciphertext + tag as produced by encrypt() + + Returns: + str: Decrypted secret + + Raises: + StorageError: If decryption fails (wrong key or corrupt data) + """ + + try: + nonce, ciphertext = blob[:_NONCE_BYTES], blob[_NONCE_BYTES:] + return self._aesgcm.decrypt(nonce, ciphertext, None).decode("utf-8") + except Exception as e: + raise StorageError(f"Failed to decrypt secret: {e}") + + def sign(self, message: str) -> str: + """Compute an HMAC-SHA256 signature for short-lived signed URLs. + + Args: + message: Message to sign + + Returns: + str: Hex-encoded signature + """ + + return hmac.new( + self._master_key, message.encode("utf-8"), hashlib.sha256 + ).hexdigest() + + def verify(self, message: str, signature: str) -> bool: + """Verify an HMAC-SHA256 signature in constant time. + + Args: + message: Original message + signature: Hex signature to check + + Returns: + bool: True if the signature is valid + """ + + return hmac.compare_digest(self.sign(message), signature) + -class SecureKeyStore: - """Placeholder for secure API key storage. +def mask_secret(secret: str) -> str: + """Build a safe masked preview of a secret (first 4 chars + bullets). - In Phase 1, this is a stub. In Phase 5, this will implement proper - encryption using the cryptography library. + Args: + secret: Secret value - Attributes: - _keys: In-memory key store (for development only) + Returns: + str: Masked preview, e.g. "AIza••••" """ - def __init__(self) -> None: - """Initialize the secure key store.""" - self._keys: dict[str, str] = {} + prefix = secret[:4] if len(secret) > 8 else "" + return f"{prefix}{'•' * 8}" - def store_api_key(self, name: str, key: str) -> None: - """Store an API key. + +class KeyVault: + """Encrypted, persistent store for LLM provider API keys. + + Keys are encrypted with AES-256-GCM before hitting disk (SQLite). + Plaintext is only ever materialized in memory, on demand, for + outbound provider calls. + """ + + def __init__(self, db_path: str, cipher: SecretCipher) -> None: + """Initialize the vault. + + Args: + db_path: Path to the SQLite database file + cipher: Cipher used for encryption at rest + """ + + self._db_path = db_path + self._cipher = cipher + self._lock = threading.Lock() + Path(db_path).parent.mkdir(parents=True, exist_ok=True) + self._init_db() + + def _connect(self) -> sqlite3.Connection: + """Open a SQLite connection.""" + + return sqlite3.connect(self._db_path, timeout=10) + + def _init_db(self) -> None: + """Create the api_keys table if missing and restrict file perms.""" + + with self._lock, self._connect() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS api_keys ( + provider TEXT PRIMARY KEY, + ciphertext BLOB NOT NULL, + masked TEXT NOT NULL, + created_at INTEGER NOT NULL + ) + """ + ) + try: + os.chmod(self._db_path, 0o600) + except OSError: + pass + + def store(self, provider: str, key: str) -> str: + """Encrypt and persist an API key for a provider. Args: - name: Key identifier - key: API key value + provider: Provider name (e.g. "google") + key: Plaintext API key - Note: - In Phase 5, this will encrypt the key before storing. + Returns: + str: Masked preview of the stored key + """ + + if not key or not key.strip(): + raise InvalidConfig("API key cannot be empty") + + key = key.strip() + masked = mask_secret(key) + blob = self._cipher.encrypt(key) + + with self._lock, self._connect() as conn: + conn.execute( + "INSERT INTO api_keys (provider, ciphertext, masked, created_at) " + "VALUES (?, ?, ?, ?) " + "ON CONFLICT(provider) DO UPDATE SET " + "ciphertext=excluded.ciphertext, masked=excluded.masked, " + "created_at=excluded.created_at", + (provider, blob, masked, int(time.time())), + ) + + logger.info("Stored encrypted API key for provider: %s", provider) + return masked + + def get_plaintext(self, provider: str) -> Optional[str]: + """Decrypt and return a provider key for internal outbound calls. + + NEVER expose the return value through any API response or log. + + Args: + provider: Provider name + + Returns: + Optional[str]: Plaintext key or None if absent """ - self._keys[name] = key - def get_api_key(self, name: str) -> Optional[str]: - """Retrieve an API key. + with self._lock, self._connect() as conn: + row = conn.execute( + "SELECT ciphertext FROM api_keys WHERE provider = ?", (provider,) + ).fetchone() + + if row is None: + return None + return self._cipher.decrypt(row[0]) + + def masked(self, provider: str) -> Optional[str]: + """Return the masked preview for a provider key. Args: - name: Key identifier + provider: Provider name Returns: - Optional[str]: API key value or None if not found + Optional[str]: Masked preview or None if absent + """ + + with self._lock, self._connect() as conn: + row = conn.execute( + "SELECT masked FROM api_keys WHERE provider = ?", (provider,) + ).fetchone() + return row[0] if row else None + + def has(self, provider: str) -> bool: + """Check whether a key exists for a provider. + + Args: + provider: Provider name - Note: - In Phase 5, this will decrypt the key after retrieval. + Returns: + bool: True if a key is stored """ - return self._keys.get(name) - def delete_api_key(self, name: str) -> bool: - """Delete an API key. + return self.masked(provider) is not None + + def delete(self, provider: str) -> bool: + """Remove a provider key. Args: - name: Key identifier + provider: Provider name Returns: - bool: True if key was deleted, False if not found + bool: True if a key was deleted """ - if name in self._keys: - del self._keys[name] - return True - return False + + with self._lock, self._connect() as conn: + cur = conn.execute("DELETE FROM api_keys WHERE provider = ?", (provider,)) + deleted = cur.rowcount > 0 + if deleted: + logger.info("Deleted API key for provider: %s", provider) + return deleted + + +_cipher: Optional[SecretCipher] = None +_vault: Optional[KeyVault] = None + + +def get_cipher() -> SecretCipher: + """Get the global SecretCipher singleton. + + Returns: + SecretCipher: Cipher bound to the application master key + """ + + global _cipher + if _cipher is None: + settings = get_settings() + master = _load_or_create_master_key(settings.data_dir, settings.master_key) + _cipher = SecretCipher(master) + return _cipher + + +def get_key_vault() -> KeyVault: + """Get the global KeyVault singleton. + + Returns: + KeyVault: Encrypted API key store + """ + + global _vault + if _vault is None: + settings = get_settings() + _vault = KeyVault(settings.db_path, get_cipher()) + return _vault + + +def reset_security_singletons() -> None: + """Reset cached cipher/vault (used by tests).""" + + global _cipher, _vault + _cipher = None + _vault = None diff --git a/spec/extraction/agents/__init__.py b/spec/extraction/agents/__init__.py new file mode 100644 index 0000000..3859dee --- /dev/null +++ b/spec/extraction/agents/__init__.py @@ -0,0 +1 @@ +"""Cooperative agents: Conector (I/O), Localizador (extraction), Organizador (formatting).""" diff --git a/spec/extraction/agents/connector.py b/spec/extraction/agents/connector.py new file mode 100644 index 0000000..9a9ef8a --- /dev/null +++ b/spec/extraction/agents/connector.py @@ -0,0 +1,664 @@ +"""Connector agent: opens input sources and delivers formatted output. + +The Connector never calls an LLM. It handles all I/O: URLs, local folders, +databases, REST APIs, uploaded files (input) and webhooks, folders, +databases, REST APIs and signed downloads (output). +""" + +import csv +import json +import logging +import re +import sqlite3 +import uuid +from pathlib import Path +from typing import Any, Callable, Dict, List + +import httpx + +from spec.core.config import get_settings +from spec.core.exceptions import ExtractionFailed, InvalidConfig +from spec.extraction.parsers.content import ( + SUPPORTED_EXTENSIONS, + bytes_to_text, + file_to_text, +) +from spec.models.webapp import InputSpec, OutputSpec + +logger = logging.getLogger(__name__) + +EmitFn = Callable[..., None] + +_MAX_DIR_FILES = 200 +_MAX_API_ITEMS = 500 +_API_BATCH_CHARS = 20_000 +_HTTP_TIMEOUT = httpx.Timeout(60.0, connect=15.0) + +_DRIVE_FILE_RE = re.compile( + r"drive\.google\.com/(?:file/d/|open\?id=|uc\?.*id=)([\w-]+)" +) +_DRIVE_FOLDER_RE = re.compile(r"drive\.google\.com/drive/(?:u/\d+/)?folders/") +_SAFE_IDENT_RE = re.compile(r"[^A-Za-z0-9_]") +_UPLOAD_ID_RE = re.compile(r"^[a-f0-9]{32}$") + + +def _safe_identifier(name: str, fallback: str) -> str: + """Sanitize a SQL identifier (table/column name). + + Args: + name: Proposed identifier + fallback: Used when nothing safe remains + + Returns: + str: Identifier containing only [A-Za-z0-9_] + """ + + cleaned = _SAFE_IDENT_RE.sub("_", name.strip())[:64].strip("_") + if not cleaned or cleaned[0].isdigit(): + cleaned = f"{fallback}_{cleaned}" if cleaned else fallback + return cleaned + + +def _redact_url(url: str) -> str: + """Strip credentials from a URL for display/log purposes.""" + + return re.sub(r"//[^/@]+@", "//••••@", url) + + +def allowed_fs_roots() -> List[Path]: + """Resolve the filesystem roots GenIE may read from / write to. + + Returns: + list[Path]: Allowed root directories + """ + + settings = get_settings() + roots = [ + Path.home().resolve(), + Path(settings.data_dir).resolve(), + Path.cwd().resolve(), + ] + if settings.allowed_fs_roots: + import os + + for raw in settings.allowed_fs_roots.split(os.pathsep): + if raw.strip(): + roots.append(Path(raw.strip()).resolve()) + return roots + + +def ensure_path_allowed(path: Path) -> Path: + """Resolve a path and verify it sits under an allowed root. + + Args: + path: Path requested by the user + + Returns: + Path: Resolved absolute path + + Raises: + InvalidConfig: If the path escapes all allowed roots + """ + + resolved = path.expanduser().resolve() + for root in allowed_fs_roots(): + if resolved == root or resolved.is_relative_to(root): + return resolved + raise InvalidConfig( + f"Acesso negado ao caminho '{path}'. Caminhos permitidos: diretório do usuário, " + "diretório do projeto e raízes definidas em ALLOWED_FS_ROOTS." + ) + + +class ConnectorAgent: + """I/O layer of the GenIE pipeline (no LLM calls).""" + + async def open_input(self, spec: InputSpec, emit: EmitFn) -> List[Dict[str, Any]]: + """Open the input source and return a list of content items. + + Args: + spec: Input specification + emit: Event emitter (agent fixed to "conector" by caller) + + Returns: + list[dict]: Items with id, name and text content + + Raises: + InvalidConfig: For unsupported/blocked sources + ExtractionFailed: When the source cannot be read + """ + + if spec.type == "upload": + return self._open_upload(spec, emit) + if spec.type == "path": + return self._open_path(spec, emit) + if spec.type == "url": + return await self._open_url(spec, emit) + if spec.type == "api": + return await self._open_api(spec, emit) + if spec.type == "db": + return self._open_db(spec, emit) + raise InvalidConfig(f"Tipo de entrada não suportado: {spec.type}") + + # ── Inputs ──────────────────────────────────────────────────────────── + + def _open_upload(self, spec: InputSpec, emit: EmitFn) -> List[Dict[str, Any]]: + """Read previously uploaded files for this run.""" + + if not spec.upload_id or not _UPLOAD_ID_RE.match(spec.upload_id): + raise InvalidConfig("Upload inválido: envie os arquivos antes de executar.") + + settings = get_settings() + upload_dir = (Path(settings.uploads_dir) / spec.upload_id).resolve() + if not upload_dir.is_dir(): + raise InvalidConfig( + f"Upload '{spec.upload_id}' não encontrado ou expirado." + ) + + items = [] + for file_path in sorted(upload_dir.iterdir()): + if not file_path.is_file(): + continue + emit(message=f"Lendo arquivo enviado: {file_path.name}") + items.append( + { + "id": f"up-{len(items) + 1}", + "name": file_path.name, + "content": file_to_text(file_path), + } + ) + + if not items: + raise ExtractionFailed("Nenhum arquivo legível encontrado no upload.") + return items + + def _open_path(self, spec: InputSpec, emit: EmitFn) -> List[Dict[str, Any]]: + """Read a local file or recursively scan a folder.""" + + if not spec.target.strip(): + raise InvalidConfig("Informe o caminho da pasta ou arquivo.") + + root = ensure_path_allowed(Path(spec.target)) + if not root.exists(): + raise InvalidConfig(f"Caminho não encontrado: {root}") + + files: List[Path] + if root.is_file(): + files = [root] + else: + files = sorted( + p + for p in root.rglob("*") + if p.is_file() and p.suffix.lower() in SUPPORTED_EXTENSIONS + )[:_MAX_DIR_FILES] + + if not files: + raise ExtractionFailed( + f"Nenhum arquivo suportado em '{root}'. " + f"Extensões: {', '.join(sorted(SUPPORTED_EXTENSIONS))}" + ) + + items = [] + for idx, file_path in enumerate(files, 1): + emit(message=f"Lendo {file_path.name} ({idx}/{len(files)})") + try: + items.append( + { + "id": f"fs-{idx}", + "name": file_path.name, + "content": file_to_text(file_path), + } + ) + except ExtractionFailed as e: + emit(message=f"Ignorando {file_path.name}: {e}", level="error") + + if not items: + raise ExtractionFailed("Nenhum arquivo pôde ser lido na pasta informada.") + return items + + async def _open_url(self, spec: InputSpec, emit: EmitFn) -> List[Dict[str, Any]]: + """Download a URL (HTML, PDF, JSON, text) and convert it to text.""" + + url = spec.target.strip() + if not url: + raise InvalidConfig("Informe a URL de origem.") + if not url.lower().startswith(("http://", "https://")): + raise InvalidConfig("URL inválida: use http:// ou https://") + + if _DRIVE_FOLDER_RE.search(url): + raise InvalidConfig( + "Pastas do Google Drive exigem credenciais de service account " + "(não configuradas). Use links diretos de arquivo, Upload ou Pasta local." + ) + + drive_match = _DRIVE_FILE_RE.search(url) + if drive_match: + url = ( + f"https://drive.google.com/uc?export=download&id={drive_match.group(1)}" + ) + emit(message="Link do Google Drive convertido para download direto") + + emit(message=f"Baixando {_redact_url(url)}") + async with httpx.AsyncClient( + timeout=_HTTP_TIMEOUT, follow_redirects=True + ) as client: + try: + response = await client.get(url) + response.raise_for_status() + except httpx.HTTPError as e: + raise ExtractionFailed(f"Falha ao acessar a URL: {e}") + + name = Path(httpx.URL(url).path).name or "pagina.html" + content = bytes_to_text( + response.content, name, response.headers.get("content-type") + ) + return [{"id": "url-1", "name": name, "content": content}] + + async def _open_api(self, spec: InputSpec, emit: EmitFn) -> List[Dict[str, Any]]: + """Call a REST API (GET, optional Bearer token) and batch the JSON.""" + + url = spec.target.strip() + if not url.lower().startswith(("http://", "https://")): + raise InvalidConfig("Endpoint de API inválido: use http:// ou https://") + + headers = {"Accept": "application/json"} + if spec.token.strip(): + headers["Authorization"] = f"Bearer {spec.token.strip()}" + + emit(message=f"Consultando API {_redact_url(url)}") + async with httpx.AsyncClient( + timeout=_HTTP_TIMEOUT, follow_redirects=True + ) as client: + try: + response = await client.get(url, headers=headers) + response.raise_for_status() + except httpx.HTTPError as e: + raise ExtractionFailed(f"Falha na chamada à API de entrada: {e}") + + try: + payload = response.json() + except json.JSONDecodeError: + return [ + { + "id": "api-1", + "name": "resposta.txt", + "content": bytes_to_text( + response.content, + "resposta.txt", + response.headers.get("content-type"), + ), + } + ] + + elements = payload if isinstance(payload, list) else [payload] + elements = elements[:_MAX_API_ITEMS] + + items: List[Dict[str, Any]] = [] + batch: List[str] = [] + batch_chars = 0 + for element in elements: + text = json.dumps(element, ensure_ascii=False, indent=2) + if batch and batch_chars + len(text) > _API_BATCH_CHARS: + items.append( + { + "id": f"api-{len(items) + 1}", + "name": f"lote-{len(items) + 1}.json", + "content": "\n".join(batch), + } + ) + batch, batch_chars = [], 0 + batch.append(text) + batch_chars += len(text) + if batch: + items.append( + { + "id": f"api-{len(items) + 1}", + "name": f"lote-{len(items) + 1}.json", + "content": "\n".join(batch), + } + ) + return items + + def _open_db(self, spec: InputSpec, emit: EmitFn) -> List[Dict[str, Any]]: + """Read rows from a database (SQLite natively, others via SQLAlchemy).""" + + target = spec.target.strip() + if not target: + raise InvalidConfig("Informe a URL do banco (ex.: sqlite:///dados.db)") + + if target.startswith("sqlite:///") or target.endswith( + (".db", ".sqlite", ".sqlite3") + ): + return self._open_sqlite(target, spec.query, emit) + return self._open_sqlalchemy(spec, emit) + + def _open_sqlite( + self, target: str, query: str, emit: EmitFn + ) -> List[Dict[str, Any]]: + """Read a SQLite database in read-only mode.""" + + raw_path = ( + target[len("sqlite:///") :] if target.startswith("sqlite:///") else target + ) + db_path = ensure_path_allowed(Path(raw_path)) + if not db_path.is_file(): + raise InvalidConfig(f"Banco SQLite não encontrado: {db_path}") + + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + try: + if query.strip(): + emit(message="Executando consulta personalizada") + rows = conn.execute(query).fetchmany(2000) + return [self._rows_item("consulta", rows)] + + tables = [ + r[0] + for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' " + "AND name NOT LIKE 'sqlite_%'" + ).fetchall() + ] + if not tables: + raise ExtractionFailed("Banco SQLite sem tabelas de usuário.") + + items = [] + for table in tables[:20]: + safe = _safe_identifier(table, "tabela") + emit(message=f"Lendo tabela {safe}") + rows = conn.execute(f'SELECT * FROM "{safe}" LIMIT 500').fetchall() + items.append(self._rows_item(safe, rows)) + return items + except sqlite3.Error as e: + raise ExtractionFailed(f"Erro ao ler o banco SQLite: {e}") + finally: + conn.close() + + def _open_sqlalchemy(self, spec: InputSpec, emit: EmitFn) -> List[Dict[str, Any]]: + """Read external databases (Postgres/MySQL/…) via SQLAlchemy if installed.""" + + try: + import sqlalchemy + except ImportError: + raise InvalidConfig( + "Para conectar a este banco instale: pip install sqlalchemy " + "e o driver adequado (psycopg2-binary, pymysql, …). " + "Bancos SQLite funcionam sem dependências extras." + ) + + url = sqlalchemy.engine.make_url(spec.target) + if spec.user: + url = url.set(username=spec.user) + if spec.password: + url = url.set(password=spec.password) + + emit(message=f"Conectando a {url.render_as_string(hide_password=True)}") + engine = sqlalchemy.create_engine(url) + try: + with engine.connect() as conn: + if spec.query.strip(): + result = conn.execute(sqlalchemy.text(spec.query)) + rows = [dict(r._mapping) for r in result.fetchmany(2000)] + return [self._rows_item("consulta", rows)] + + inspector = sqlalchemy.inspect(engine) + items = [] + for table in inspector.get_table_names()[:20]: + emit(message=f"Lendo tabela {table}") + result = conn.execute( + sqlalchemy.text(f'SELECT * FROM "{table}" LIMIT 500') + ) + rows = [dict(r._mapping) for r in result.fetchall()] + items.append(self._rows_item(table, rows)) + return items + except sqlalchemy.exc.SQLAlchemyError as e: + raise ExtractionFailed(f"Erro ao ler o banco de dados: {e}") + finally: + engine.dispose() + + @staticmethod + def _rows_item(name: str, rows: Any) -> Dict[str, Any]: + """Serialize DB rows into a text item for the Locator.""" + + dicts = [dict(r) for r in rows] + return { + "id": f"db-{name}", + "name": f"{name} ({len(dicts)} linhas)", + "content": json.dumps(dicts, ensure_ascii=False, indent=1, default=str), + } + + # ── Outputs ─────────────────────────────────────────────────────────── + + async def deliver( + self, + spec: OutputSpec, + records: List[Any], + hints: Dict[str, Any], + job_id: str, + emit: EmitFn, + ) -> Dict[str, Any]: + """Deliver formatted records to the configured destination. + + Args: + spec: Output specification + records: Formatted payloads from the Organizer + hints: Delivery hints (method, headers, batch) from the Organizer + job_id: Run identifier (used for download artifacts) + emit: Event emitter + + Returns: + dict: Delivery receipt (never includes credentials) + """ + + if spec.type == "download": + return self._deliver_download(records, job_id, emit) + if spec.type == "path": + return self._deliver_path(spec, records, emit) + if spec.type in ("url", "api"): + return await self._deliver_http(spec, records, hints, emit) + if spec.type == "db": + return self._deliver_db(spec, records, emit) + raise InvalidConfig(f"Tipo de saída não suportado: {spec.type}") + + def _write_artifacts(self, records: List[Any], directory: Path) -> Dict[str, str]: + """Write output.json (+ output.csv when tabular) into a directory.""" + + directory.mkdir(parents=True, exist_ok=True) + json_path = directory / "output.json" + json_path.write_text( + json.dumps(records, ensure_ascii=False, indent=2, default=str), + encoding="utf-8", + ) + artifacts = {"json": str(json_path)} + + flat = [r for r in records if isinstance(r, dict)] + if flat: + columns: List[str] = [] + for record in flat: + for key in record: + if not str(key).startswith("_") and key not in columns: + columns.append(key) + if columns: + csv_path = directory / "output.csv" + with open(csv_path, "w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter( + f, fieldnames=columns, extrasaction="ignore" + ) + writer.writeheader() + for record in flat: + writer.writerow({k: record.get(k, "") for k in columns}) + artifacts["csv"] = str(csv_path) + return artifacts + + def _deliver_download( + self, records: List[Any], job_id: str, emit: EmitFn + ) -> Dict[str, Any]: + """Persist artifacts for signed in-browser download.""" + + settings = get_settings() + directory = Path(settings.outputs_dir) / job_id + artifacts = self._write_artifacts(records, directory) + emit( + message=f"Arquivo(s) de saída gerado(s): {', '.join(Path(p).name for p in artifacts.values())}" + ) + return { + "mode": "download", + "artifacts": sorted(Path(p).name for p in artifacts.values()), + } + + def _deliver_path( + self, spec: OutputSpec, records: List[Any], emit: EmitFn + ) -> Dict[str, Any]: + """Write artifacts into a user-specified local folder.""" + + if not spec.target.strip(): + raise InvalidConfig("Informe a pasta de destino.") + directory = ensure_path_allowed(Path(spec.target)) + if directory.suffix: + directory = directory.parent / directory.stem + artifacts = self._write_artifacts(records, directory) + emit(message=f"Gravado em {directory}") + return { + "mode": "path", + "directory": str(directory), + "files": sorted(Path(p).name for p in artifacts.values()), + } + + async def _deliver_http( + self, + spec: OutputSpec, + records: List[Any], + hints: Dict[str, Any], + emit: EmitFn, + ) -> Dict[str, Any]: + """POST records to a webhook (url) or REST API (api, Bearer auth).""" + + url = spec.target.strip() + if not url.lower().startswith(("http://", "https://")): + raise InvalidConfig("Destino HTTP inválido: use http:// ou https://") + + headers = {"Content-Type": "application/json"} + for key, value in (hints.get("headers") or {}).items(): + if str(key).lower() != "authorization": + headers[str(key)] = str(value) + if spec.type == "api" and spec.token.strip(): + headers["Authorization"] = f"Bearer {spec.token.strip()}" + + method = str(hints.get("method") or "POST").upper() + batch = bool(hints.get("batch", True)) + + async with httpx.AsyncClient(timeout=_HTTP_TIMEOUT) as client: + if batch: + emit( + message=f"Enviando {len(records)} registro(s) em lote para {_redact_url(url)}" + ) + response = await client.request( + method, url, json=records, headers=headers + ) + statuses = [response.status_code] + else: + statuses = [] + for idx, record in enumerate(records[:100], 1): + emit( + message=f"POST {idx}/{min(len(records), 100)} → {_redact_url(url)}" + ) + response = await client.request( + method, url, json=record, headers=headers + ) + statuses.append(response.status_code) + + failed = [s for s in statuses if s >= 400] + if failed: + raise ExtractionFailed( + f"Destino HTTP retornou erro(s): {sorted(set(failed))} " + f"em {len(failed)}/{len(statuses)} chamada(s)." + ) + return { + "mode": spec.type, + "calls": len(statuses), + "status": sorted(set(statuses)), + } + + def _deliver_db( + self, spec: OutputSpec, records: List[Any], emit: EmitFn + ) -> Dict[str, Any]: + """Insert records into a SQLite table (created/extended automatically).""" + + target = spec.target.strip() + if not ( + target.startswith("sqlite:///") + or target.endswith((".db", ".sqlite", ".sqlite3")) + ): + raise InvalidConfig( + "Saída para banco suporta SQLite nativamente (sqlite:///caminho.db). " + "Para outros bancos use a saída 'API REST' de um serviço intermediário " + "ou instale sqlalchemy + driver." + ) + + raw_path = ( + target[len("sqlite:///") :] if target.startswith("sqlite:///") else target + ) + db_path = ensure_path_allowed(Path(raw_path)) + db_path.parent.mkdir(parents=True, exist_ok=True) + + flat = [r for r in records if isinstance(r, dict)] + if not flat: + raise ExtractionFailed("Nenhum registro tabular para inserir no banco.") + + table = _safe_identifier(spec.table or "genie_output", "genie_output") + columns: List[str] = [] + for record in flat: + for key in record: + safe = _safe_identifier(str(key), "campo") + if not safe.startswith("_") and safe not in columns: + columns.append(safe) + + conn = sqlite3.connect(db_path) + try: + cols_sql = ", ".join(f'"{c}" TEXT' for c in columns) + conn.execute(f'CREATE TABLE IF NOT EXISTS "{table}" ({cols_sql})') + existing = {row[1] for row in conn.execute(f'PRAGMA table_info("{table}")')} + for column in columns: + if column not in existing: + conn.execute(f'ALTER TABLE "{table}" ADD COLUMN "{column}" TEXT') + + placeholders = ", ".join("?" for _ in columns) + quoted = ", ".join(f'"{c}"' for c in columns) + for record in flat: + by_safe_key = { + _safe_identifier(str(k), "campo"): v for k, v in record.items() + } + values = [] + for column in columns: + value = by_safe_key.get(column) + if isinstance(value, (dict, list)): + value = json.dumps(value, ensure_ascii=False, default=str) + elif value is not None: + value = str(value) + values.append(value) + conn.execute( + f'INSERT INTO "{table}" ({quoted}) VALUES ({placeholders})', values + ) + conn.commit() + except sqlite3.Error as e: + raise ExtractionFailed(f"Erro ao gravar no banco: {e}") + finally: + conn.close() + + emit(message=f"{len(flat)} registro(s) inseridos em {db_path.name}:{table}") + return { + "mode": "db", + "database": str(db_path), + "table": table, + "rows": len(flat), + } + + +def new_upload_id() -> str: + """Generate a new upload batch identifier. + + Returns: + str: 32-char hex id + """ + + return uuid.uuid4().hex diff --git a/spec/extraction/agents/locator.py b/spec/extraction/agents/locator.py new file mode 100644 index 0000000..e9d2e3f --- /dev/null +++ b/spec/extraction/agents/locator.py @@ -0,0 +1,185 @@ +"""Locator agent: extracts the requested information from each item via LLM.""" + +import asyncio +import logging +from typing import Any, Callable, Dict, List, Tuple + +from spec.core.exceptions import ExtractionFailed, LLMProviderError +from spec.extraction.llm.base import BaseLLMProvider + +logger = logging.getLogger(__name__) + +EmitFn = Callable[..., None] + +_MAX_CHUNK_CHARS = 24_000 +_CHUNK_OVERLAP = 500 +_MAX_RETRIES = 3 +_RETRYABLE_MARKERS = ("429", "503", "rate", "overload", "timeout", "temporarily") + +_SCHEMA: Dict[str, Any] = { + "records": [{"...campos extraídos conforme a instrução...": "valor"}], + "confidence": 0.0, + "notes": "observações curtas, se houver", +} + +_INSTRUCTIONS_TEMPLATE = """Você é o Localizador do GenIE. Sua tarefa é extrair informações específicas do documento fornecido. + +Instrução do usuário: +{user_prompt} + +Regras: +- Responda APENAS com JSON válido no formato do schema. +- "records" é uma lista de objetos; cada objeto usa nomes de campos claros e consistentes derivados da instrução do usuário. +- Se nada relevante for encontrado, devolva "records": []. +- "confidence" entre 0.0 e 1.0. +""" + + +def _chunk(text: str) -> List[str]: + """Split long content into overlapping chunks that fit a single call. + + Args: + text: Item content + + Returns: + list[str]: One or more chunks + """ + + if len(text) <= _MAX_CHUNK_CHARS: + return [text] + chunks = [] + start = 0 + while start < len(text): + chunks.append(text[start : start + _MAX_CHUNK_CHARS]) + start += _MAX_CHUNK_CHARS - _CHUNK_OVERLAP + return chunks + + +class LocatorAgent: + """Extraction layer of the GenIE pipeline (the only stage that reads documents).""" + + def __init__(self, provider: BaseLLMProvider) -> None: + """Initialize the agent. + + Args: + provider: LLM provider used for extraction + """ + + self.provider = provider + + async def run( + self, + items: List[Dict[str, Any]], + prompt: str, + emit: EmitFn, + ) -> Tuple[List[Dict[str, Any]], float, List[str]]: + """Extract records from every item. + + Args: + items: Content items from the Connector + prompt: User extraction instruction + emit: Event emitter (agent fixed to "localizador" by caller) + + Returns: + tuple: (records, mean confidence, notes) + + Raises: + ExtractionFailed: If every item fails + """ + + instructions = _INSTRUCTIONS_TEMPLATE.format(user_prompt=prompt.strip()) + + records: List[Dict[str, Any]] = [] + confidences: List[float] = [] + notes: List[str] = [] + failures = 0 + + total_chunks = sum(len(_chunk(item["content"])) for item in items) + done_chunks = 0 + + for item in items: + chunks = _chunk(item["content"]) + for chunk_idx, chunk in enumerate(chunks, 1): + done_chunks += 1 + suffix = ( + f" (parte {chunk_idx}/{len(chunks)})" if len(chunks) > 1 else "" + ) + emit( + message=f"Analisando {item['name']}{suffix}", + progress=int(done_chunks / max(total_chunks, 1) * 90), + ) + document = f"Documento (id={item['id']}, nome={item['name']}):\n---\n{chunk}\n---" + try: + parsed = await self._extract_with_retry( + document, instructions, emit + ) + except LLMProviderError as e: + failures += 1 + emit(message=f"Falha em {item['name']}: {e}", level="error") + continue + + item_records = ( + parsed.get("records") if isinstance(parsed, dict) else parsed + ) + if isinstance(item_records, dict): + item_records = [item_records] + if isinstance(item_records, list): + found = [r for r in item_records if isinstance(r, dict)] + records.extend(found) + if found: + emit( + message=f"{len(found)} registro(s) em {item['name']}{suffix}" + ) + if isinstance(parsed, dict): + if isinstance(parsed.get("confidence"), (int, float)): + confidences.append(float(parsed["confidence"])) + if parsed.get("notes"): + notes.append(str(parsed["notes"])) + + if failures and not records: + raise ExtractionFailed( + f"Extração falhou em todos os {failures} documento(s)/lote(s). " + "Verifique a chave de API e o modelo selecionado." + ) + + confidence = ( + round(sum(confidences) / len(confidences), 2) if confidences else 0.0 + ) + return records, confidence, notes + + async def _extract_with_retry( + self, document: str, instructions: str, emit: EmitFn + ) -> Dict[str, Any]: + """Call the LLM with exponential backoff on transient errors. + + Args: + document: Document chunk wrapped with id/name header + instructions: Locator system instructions + emit: Event emitter + + Returns: + dict: Parsed JSON from the model + + Raises: + LLMProviderError: After exhausting retries + """ + + delay = 2.0 + for attempt in range(1, _MAX_RETRIES + 1): + try: + return await self.provider.extract( + content=document, + schema=_SCHEMA, + instructions=instructions, + ) + except LLMProviderError as e: + transient = any(m in str(e).lower() for m in _RETRYABLE_MARKERS) + if attempt == _MAX_RETRIES or not transient: + raise + emit( + message=f"Erro transitório do provedor (tentativa {attempt}/{_MAX_RETRIES}), aguardando {delay:.0f}s…", + level="error", + ) + await asyncio.sleep(delay) + delay *= 2 + raise LLMProviderError("Retries exhausted") # pragma: no cover diff --git a/spec/extraction/agents/orchestrator.py b/spec/extraction/agents/orchestrator.py new file mode 100644 index 0000000..afefa0f --- /dev/null +++ b/spec/extraction/agents/orchestrator.py @@ -0,0 +1,312 @@ +"""Orchestrator: drives Conector → Localizador → Organizador → Conector. + +Emits real-time events through the JobManager so the web UI can render +agent progress, the execution log and the final result. +""" + +import asyncio +import logging +import re +import time +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from spec.core.config import get_settings +from spec.core.exceptions import GenieException +from spec.core.security import get_cipher, get_key_vault +from spec.extraction.agents.connector import ConnectorAgent +from spec.extraction.agents.locator import LocatorAgent +from spec.extraction.agents.organizer import OrganizerAgent +from spec.extraction.llm.factory import LLMProviderFactory +from spec.models.webapp import InputSpec, OutputSpec +from spec.webapp.catalog import find_model +from spec.webapp.jobs import Job, JobManager + +logger = logging.getLogger(__name__) + +_IN_LABELS = { + "url": "URL", + "path": "Pasta local", + "db": "Banco de dados", + "api": "API REST", + "upload": "Upload", +} +_OUT_LABELS = { + "url": "URL (webhook)", + "path": "Pasta local", + "db": "Banco de dados", + "api": "API REST", + "download": "Download", +} + + +def _display_target(spec: InputSpec | OutputSpec) -> str: + """Build a credential-free display string for a source/destination.""" + + target = re.sub(r"//[^/@]+@", "//••••@", spec.target or "") + if isinstance(spec, InputSpec) and spec.type == "upload": + return "arquivos enviados" + if not target: + return "saida.json" if getattr(spec, "type", "") == "download" else "—" + return target + + +def build_download_links(job_id: str, artifacts: List[str]) -> Dict[str, str]: + """Create short-lived signed download URLs for run artifacts. + + Args: + job_id: Run identifier + artifacts: Artifact filenames (e.g. ["output.csv", "output.json"]) + + Returns: + dict: Format → relative signed URL + """ + + settings = get_settings() + cipher = get_cipher() + expires = int(time.time()) + settings.download_link_ttl_seconds + + links: Dict[str, str] = {} + for name in artifacts: + fmt = name.rsplit(".", 1)[-1] + signature = cipher.sign(f"{job_id}:{name}:{expires}") + links[fmt] = f"/api/v1/downloads/{job_id}/{name}?exp={expires}&sig={signature}" + return links + + +class Orchestrator: + """Tech-lead of the three agents: coordinates, never extracts by itself.""" + + def __init__(self, manager: JobManager) -> None: + """Initialize the orchestrator. + + Args: + manager: Job registry used for event emission + """ + + self.manager = manager + + def _resolve_provider(self, model_id: str): + """Build the LLM provider for a catalog model using the encrypted vault. + + Args: + model_id: Catalog model id + + Returns: + BaseLLMProvider: Ready-to-use provider + + Raises: + GenieException: If the model is unknown or has no key + """ + + model = find_model(model_id) + if model is None: + raise GenieException(f"Modelo desconhecido: {model_id}") + + api_key = get_key_vault().get_plaintext(model["provider"]) + factory = LLMProviderFactory() + return factory.get_provider( + provider_name=model["provider"], + model=model["id"], + api_key=api_key, + ) + + async def run_job(self, job: Job) -> None: + """Execute the full pipeline for a job, emitting events throughout. + + Args: + job: Job created by the JobManager (status "queued") + """ + + emit = self.manager.emit + request = job.request + model = find_model(request.model_id) or {"label": request.model_id} + started = time.monotonic() + + def agent_emit(agent: str): + def _fn( + message: str = "", level: str = "", progress: Optional[int] = None + ) -> None: + emit( + job, + agent=agent, + type="progress" if progress is not None else "log", + message=message, + level=level, + progress=progress, + ) + + return _fn + + job.status = "running" + connector = ConnectorAgent() + + try: + emit( + job, + agent="sistema", + type="log", + message=f"Sessão iniciada · modelo={model['label']}", + ) + + # Resolve provider first: fail fast if the key is missing. + provider = self._resolve_provider(request.model_id) + + # ── Conector: input ───────────────────────────────────────── + in_label = _IN_LABELS.get(request.input.type, request.input.type) + emit( + job, + agent="conector", + type="progress", + progress=10, + message=f"Abrindo canal de entrada ({in_label})…", + ) + items = await connector.open_input(request.input, agent_emit("conector")) + emit( + job, + agent="conector", + type="done", + progress=100, + level="ok", + message=f"Conexão estabelecida · {len(items)} item(ns) enumerado(s)", + ) + + # ── Localizador ───────────────────────────────────────────── + emit( + job, + agent="localizador", + type="progress", + progress=5, + message="Carregando contexto da extração…", + ) + preview = request.prompt[:96] + ("…" if len(request.prompt) > 96 else "") + emit(job, agent="localizador", type="log", message=f'Prompt: "{preview}"') + locator = LocatorAgent(provider) + records, confidence, notes = await locator.run( + items, request.prompt, agent_emit("localizador") + ) + for note in notes[:5]: + emit(job, agent="localizador", type="log", message=f"Nota: {note}") + emit( + job, + agent="localizador", + type="done", + progress=100, + level="ok", + message=f"Extração concluída · {len(records)} registro(s) · confiança={confidence:.2f}", + ) + + # ── Organizador ───────────────────────────────────────────── + emit( + job, + agent="organizador", + type="progress", + progress=10, + message="Validando schema de saída…", + ) + organizer = OrganizerAgent(provider) + formatted, hints = await organizer.run( + records, request.format, request.output.type, agent_emit("organizador") + ) + emit( + job, + agent="organizador", + type="done", + progress=100, + level="ok", + message=f"{len(formatted)} registro(s) prontos para entrega", + ) + + # ── Conector: delivery ────────────────────────────────────── + out_label = _OUT_LABELS.get(request.output.type, request.output.type) + emit( + job, + agent="conector", + type="log", + message=f"Reabrindo canal de saída ({out_label})…", + ) + receipt = await connector.deliver( + request.output, formatted, hints, job.id, agent_emit("conector") + ) + emit( + job, + agent="conector", + type="log", + level="ok", + message="Entrega concluída", + ) + + download_links: Dict[str, str] = {} + if request.output.type == "download": + download_links = build_download_links( + job.id, receipt.get("artifacts", []) + ) + + elapsed = time.monotonic() - started + result: Dict[str, Any] = { + "job_id": job.id, + "model": model["label"], + "source": { + "type": request.input.type, + "target": _display_target(request.input), + }, + "extraction": request.prompt, + "delivered_to": { + "type": request.output.type, + "target": _display_target(request.output), + "format": request.format or "json", + }, + "records": [r for r in formatted if isinstance(r, dict)] or records, + "confidence": confidence, + "receipt": receipt, + "downloads": download_links, + "delivered_at": datetime.now(timezone.utc).isoformat(), + } + + job.result = result + job.status = "done" + emit( + job, + agent="sistema", + type="finish", + level="ok", + status="done", + message=f"Job {job.id} finalizado em {elapsed:.2f}s", + result=result, + ) + + except asyncio.CancelledError: + job.status = "cancelled" + job.error = "Execução interrompida pelo usuário" + emit( + job, + agent="sistema", + type="error", + level="error", + status="cancelled", + message="Execução interrompida pelo usuário", + ) + raise + except GenieException as e: + job.status = "error" + job.error = str(e) + emit( + job, + agent="sistema", + type="error", + level="error", + status="error", + message=str(e), + ) + except Exception as e: # noqa: BLE001 - last-resort guard for the task + logger.error("Job %s crashed: %s", job.id, e, exc_info=True) + job.status = "error" + job.error = f"Erro interno: {e}" + emit( + job, + agent="sistema", + type="error", + level="error", + status="error", + message=f"Erro interno: {e}", + ) diff --git a/spec/extraction/agents/organizer.py b/spec/extraction/agents/organizer.py new file mode 100644 index 0000000..fc542a5 --- /dev/null +++ b/spec/extraction/agents/organizer.py @@ -0,0 +1,127 @@ +"""Organizer agent: reshapes extracted records into the requested output format.""" + +import json +import logging +from typing import Any, Callable, Dict, List, Tuple + +from spec.core.exceptions import LLMProviderError +from spec.extraction.llm.base import BaseLLMProvider + +logger = logging.getLogger(__name__) + +EmitFn = Callable[..., None] + +_MAX_RECORDS_PER_CALL = 60 + +_SCHEMA: Dict[str, Any] = { + "formatted": ["...payloads no formato pedido, prontos para entrega..."], + "delivery_hints": {"method": "POST", "headers": {}, "batch": True}, +} + +_INSTRUCTIONS_TEMPLATE = """Você é o Organizador do GenIE. Reformate os dados extraídos no formato pedido pelo usuário. + +Formato de saída desejado: +{output_format} + +Tipo de destino: {output_type} + +Regras: +- "formatted" é uma lista de payloads prontos para o Conector entregar, um por registro (a menos que o formato peça agrupamento). +- Normalize datas para ISO-8601 e números para tipos numéricos quando possível. +- Não descarte registros: em caso de dúvida, inclua o registro com um campo "_warnings" (lista de strings). +- "delivery_hints" indica como entregar (method, headers extras, batch true/false). +- Responda APENAS com JSON válido no formato do schema. +""" + + +class OrganizerAgent: + """Formatting layer of the GenIE pipeline.""" + + def __init__(self, provider: BaseLLMProvider) -> None: + """Initialize the agent. + + Args: + provider: LLM provider used for formatting + """ + + self.provider = provider + + async def run( + self, + records: List[Dict[str, Any]], + output_format: str, + output_type: str, + emit: EmitFn, + ) -> Tuple[List[Any], Dict[str, Any]]: + """Format records per the user instruction. + + When no format instruction is given, records pass through unchanged + (zero LLM cost), honoring GenIE's cost-efficiency principle. + + Args: + records: Records extracted by the Locator + output_format: Free-text formatting instruction (may be empty) + output_type: Destination kind (url/path/db/api/download) + emit: Event emitter (agent fixed to "organizador" by caller) + + Returns: + tuple: (formatted payloads, delivery hints) + """ + + if not records: + emit(message="Nenhum registro para formatar") + return [], {} + + if not output_format.strip(): + emit( + message=f"Sem instrução de formato — entregando {len(records)} registro(s) como JSON" + ) + return list(records), {} + + instructions = _INSTRUCTIONS_TEMPLATE.format( + output_format=output_format.strip(), output_type=output_type + ) + + formatted: List[Any] = [] + hints: Dict[str, Any] = {} + + batches = [ + records[i : i + _MAX_RECORDS_PER_CALL] + for i in range(0, len(records), _MAX_RECORDS_PER_CALL) + ] + for batch_idx, batch in enumerate(batches, 1): + emit( + message=f"Formatando lote {batch_idx}/{len(batches)} ({len(batch)} registro(s))", + progress=int(batch_idx / len(batches) * 90), + ) + payload = json.dumps(batch, ensure_ascii=False, indent=1, default=str) + try: + parsed = await self.provider.extract( + content=f"Dados brutos (JSON):\n{payload}", + schema=_SCHEMA, + instructions=instructions, + ) + except LLMProviderError as e: + emit( + message=f"Organizador indisponível ({e}) — usando registros brutos", + level="error", + ) + formatted.extend(batch) + continue + + batch_formatted = ( + parsed.get("formatted") if isinstance(parsed, dict) else parsed + ) + if isinstance(batch_formatted, dict): + batch_formatted = [batch_formatted] + if isinstance(batch_formatted, list) and batch_formatted: + formatted.extend(batch_formatted) + else: + formatted.extend(batch) + + if isinstance(parsed, dict) and isinstance( + parsed.get("delivery_hints"), dict + ): + hints = parsed["delivery_hints"] + + return formatted, hints diff --git a/spec/extraction/engine.py b/spec/extraction/engine.py index ff42f40..f5e02a9 100644 --- a/spec/extraction/engine.py +++ b/spec/extraction/engine.py @@ -2,8 +2,8 @@ import logging import uuid -from typing import Any, Dict, Optional from time import time +from typing import Any, Dict from spec.core.exceptions import ExtractionFailed, InvalidConfig from spec.extraction.layout.fingerprint import LayoutFingerprint @@ -111,7 +111,7 @@ async def extract(self, request: ExtractionRequest) -> ExtractionResponse: extracted_data, pattern ) if is_valid: - logger.info(f"Extraction successful via library") + logger.info("Extraction successful via library") else: logger.warning("Pattern validation failed, falling back to LLM") method_used = "unknown" # Reset for LLM diff --git a/spec/extraction/layout/fingerprint.py b/spec/extraction/layout/fingerprint.py index 19c2ed5..1863366 100644 --- a/spec/extraction/layout/fingerprint.py +++ b/spec/extraction/layout/fingerprint.py @@ -3,7 +3,6 @@ import hashlib import logging import re -from typing import Optional logger = logging.getLogger(__name__) diff --git a/spec/extraction/llm/__init__.py b/spec/extraction/llm/__init__.py index cf76427..8fe16f9 100644 --- a/spec/extraction/llm/__init__.py +++ b/spec/extraction/llm/__init__.py @@ -1,8 +1,8 @@ """LLM provider implementations and factory.""" +from spec.extraction.llm.anthropic import AnthropicProvider from spec.extraction.llm.base import BaseLLMProvider from spec.extraction.llm.factory import LLMProviderFactory -from spec.extraction.llm.anthropic import AnthropicProvider from spec.extraction.llm.openai import OpenAIProvider __all__ = [ diff --git a/spec/extraction/llm/factory.py b/spec/extraction/llm/factory.py index 3e691e3..3aba439 100644 --- a/spec/extraction/llm/factory.py +++ b/spec/extraction/llm/factory.py @@ -1,21 +1,22 @@ """Factory for creating LLM provider instances.""" +import hashlib import logging from typing import Optional from spec.core.config import Settings, get_settings from spec.core.exceptions import InvalidConfig -from spec.extraction.llm.base import BaseLLMProvider from spec.extraction.llm.anthropic import AnthropicProvider +from spec.extraction.llm.base import BaseLLMProvider from spec.extraction.llm.google import GoogleProvider from spec.extraction.llm.openai import OpenAIProvider logger = logging.getLogger(__name__) _DEFAULT_MODELS: dict[str, str] = { - "google": "gemini-1.5-pro", + "google": "gemini-2.5-flash", "openai": "gpt-4o", - "anthropic": "claude-sonnet-4-20250514", + "anthropic": "claude-sonnet-4-6", } _SUPPORTED_PROVIDERS = frozenset(_DEFAULT_MODELS.keys()) @@ -70,7 +71,9 @@ def get_provider( if not resolved_key: raise InvalidConfig(f"API key for provider '{provider_name}' is not configured") - cache_key = f"{provider_name}:{resolved_model}:{resolved_key[:10]}" + # Never put key material (even a prefix) in cache keys: hash it. + key_digest = hashlib.sha256(resolved_key.encode("utf-8")).hexdigest()[:16] + cache_key = f"{provider_name}:{resolved_model}:{key_digest}" if cache_key in self._providers: return self._providers[cache_key] @@ -121,12 +124,10 @@ def set_provider_config( if provider not in _SUPPORTED_PROVIDERS: raise InvalidConfig(f"Unknown LLM provider: {provider}") - if provider == "google": - self.settings.google_api_key = api_key - elif provider == "openai": - self.settings.openai_api_key = api_key - elif provider == "anthropic": - self.settings.anthropic_api_key = api_key + # Persist encrypted (AES-256-GCM) instead of keeping plaintext in settings. + from spec.core.security import get_key_vault + + get_key_vault().store(provider, api_key) self.settings.llm_provider = provider self.settings.llm_model = model @@ -147,29 +148,28 @@ def list_providers() -> list[dict]: { "name": "google", "display_name": "Google Gemini", - "default_model": "gemini-1.5-pro", - "available_models": ["gemini-1.5-pro", "gemini-1.5-flash", "gemini-2.0-flash"], + "default_model": "gemini-2.5-flash", + "available_models": ["gemini-2.5-flash", "gemini-2.5-pro"], }, { "name": "openai", "display_name": "OpenAI GPT", "default_model": "gpt-4o", - "available_models": ["gpt-4o", "gpt-4o-mini", "gpt-4-turbo"], + "available_models": ["gpt-4o", "gpt-4o-mini"], }, { "name": "anthropic", "display_name": "Anthropic Claude", - "default_model": "claude-sonnet-4-20250514", + "default_model": "claude-sonnet-4-6", "available_models": [ - "claude-sonnet-4-20250514", - "claude-3-5-haiku-20241022", - "claude-opus-4-5", + "claude-sonnet-4-6", + "claude-haiku-4-5-20251001", ], }, ] def _get_api_key_for_provider(self, provider_name: str) -> Optional[str]: - """Resolve API key from settings for the given provider. + """Resolve API key: encrypted vault first, env settings as fallback. Args: provider_name: Provider name @@ -178,6 +178,13 @@ def _get_api_key_for_provider(self, provider_name: str) -> Optional[str]: Optional[str]: API key or None if not configured """ + # Lazy import to avoid a circular dependency at module load time. + from spec.core.security import get_key_vault + + vault_key = get_key_vault().get_plaintext(provider_name) + if vault_key: + return vault_key + key_map: dict[str, Optional[str]] = { "google": self.settings.google_api_key, "openai": self.settings.openai_api_key, diff --git a/spec/extraction/parsers/__init__.py b/spec/extraction/parsers/__init__.py index 1f3d7d8..70a441e 100644 --- a/spec/extraction/parsers/__init__.py +++ b/spec/extraction/parsers/__init__.py @@ -1,7 +1,7 @@ """Content parsers for various document formats.""" -from spec.extraction.parsers.text import TextParser from spec.extraction.parsers.pdf import PDFParser +from spec.extraction.parsers.text import TextParser __all__ = [ "TextParser", diff --git a/spec/extraction/parsers/content.py b/spec/extraction/parsers/content.py new file mode 100644 index 0000000..626ba35 --- /dev/null +++ b/spec/extraction/parsers/content.py @@ -0,0 +1,213 @@ +"""Content-to-text extraction helpers shared by the Connector agent. + +Turns heterogeneous payloads (PDF, XLSX, CSV, JSON, HTML, plain text) +into plain text suitable for LLM extraction. +""" + +import csv +import io +import json +import logging +import re +from html.parser import HTMLParser +from pathlib import Path +from typing import Optional + +from PyPDF2 import PdfReader + +from spec.core.exceptions import ExtractionFailed + +logger = logging.getLogger(__name__) + +SUPPORTED_EXTENSIONS = frozenset( + { + ".pdf", + ".txt", + ".md", + ".csv", + ".json", + ".html", + ".htm", + ".xml", + ".yaml", + ".yml", + ".xlsx", + ".log", + ".tsv", + } +) + +_MAX_TEXT_CHARS = 400_000 + + +class _TextExtractor(HTMLParser): + """Minimal HTML-to-text converter (keeps visible text only).""" + + _SKIP_TAGS = frozenset({"script", "style", "noscript", "template", "head"}) + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self._chunks: list[str] = [] + self._skip_depth = 0 + + def handle_starttag(self, tag: str, attrs: list) -> None: + if tag in self._SKIP_TAGS: + self._skip_depth += 1 + + def handle_endtag(self, tag: str) -> None: + if tag in self._SKIP_TAGS and self._skip_depth > 0: + self._skip_depth -= 1 + + def handle_data(self, data: str) -> None: + if self._skip_depth == 0 and data.strip(): + self._chunks.append(data.strip()) + + def text(self) -> str: + return "\n".join(self._chunks) + + +def html_to_text(html: str) -> str: + """Convert HTML markup into readable plain text. + + Args: + html: Raw HTML + + Returns: + str: Visible text content + """ + + parser = _TextExtractor() + try: + parser.feed(html) + return parser.text() + except Exception: + return re.sub(r"<[^>]+>", " ", html) + + +def pdf_bytes_to_text(data: bytes, name: str = "document.pdf") -> str: + """Extract text from PDF bytes. + + Args: + data: PDF file content + name: Filename used in error messages + + Returns: + str: Extracted text + + Raises: + ExtractionFailed: If the PDF has no extractable text (e.g. scanned) + """ + + try: + reader = PdfReader(io.BytesIO(data)) + parts = [] + for page in reader.pages: + try: + parts.append(page.extract_text() or "") + except Exception as e: # pragma: no cover - defensive per page + logger.warning("Failed to read a page of %s: %s", name, e) + text = "\n".join(p for p in parts if p).strip() + except ExtractionFailed: + raise + except Exception as e: + raise ExtractionFailed(f"Falha ao processar PDF '{name}': {e}") + + if not text: + raise ExtractionFailed( + f"PDF '{name}' não contém texto extraível (provavelmente digitalizado). " + "Suporte a OCR está planejado; envie um PDF nativo ou um TXT/CSV." + ) + return text + + +def xlsx_bytes_to_text(data: bytes, name: str = "sheet.xlsx") -> str: + """Extract cell values from XLSX bytes as TSV-like text. + + Args: + data: Workbook content + name: Filename used in error messages + + Returns: + str: Tab-separated rows, one sheet after another + """ + + try: + from openpyxl import load_workbook + + workbook = load_workbook(io.BytesIO(data), read_only=True, data_only=True) + lines: list[str] = [] + for sheet in workbook.worksheets: + lines.append(f"# Planilha: {sheet.title}") + for row in sheet.iter_rows(values_only=True): + cells = ["" if c is None else str(c) for c in row] + if any(cells): + lines.append("\t".join(cells)) + return "\n".join(lines) + except ImportError: + raise ExtractionFailed("openpyxl não instalado — necessário para ler .xlsx") + except Exception as e: + raise ExtractionFailed(f"Falha ao processar planilha '{name}': {e}") + + +def bytes_to_text(data: bytes, name: str, content_type: Optional[str] = None) -> str: + """Convert raw bytes into text based on file extension / content type. + + Args: + data: Raw payload + name: Filename or URL (used to infer the format) + content_type: Optional MIME type hint + + Returns: + str: Plain text (truncated to a safe maximum) + + Raises: + ExtractionFailed: If the format cannot be converted + """ + + suffix = Path(name.split("?")[0]).suffix.lower() + ctype = (content_type or "").split(";")[0].strip().lower() + + if suffix == ".pdf" or ctype == "application/pdf": + text = pdf_bytes_to_text(data, name) + elif suffix == ".xlsx" or ctype.endswith("spreadsheetml.sheet"): + text = xlsx_bytes_to_text(data, name) + elif suffix in (".html", ".htm") or ctype == "text/html": + text = html_to_text(data.decode("utf-8", errors="replace")) + elif suffix == ".json" or ctype == "application/json": + raw = data.decode("utf-8", errors="replace") + try: + text = json.dumps(json.loads(raw), indent=2, ensure_ascii=False) + except json.JSONDecodeError: + text = raw + elif suffix in (".csv", ".tsv"): + raw = data.decode("utf-8", errors="replace") + delimiter = "\t" if suffix == ".tsv" else "," + rows = list(csv.reader(io.StringIO(raw), delimiter=delimiter)) + text = "\n".join("\t".join(row) for row in rows) + elif suffix == ".docx": + raise ExtractionFailed( + f"Formato .docx ainda não suportado ('{name}'). Converta para PDF ou TXT." + ) + else: + text = data.decode("utf-8", errors="replace") + + text = text.strip() + if len(text) > _MAX_TEXT_CHARS: + logger.warning( + "Truncating '%s' from %d to %d chars", name, len(text), _MAX_TEXT_CHARS + ) + text = text[:_MAX_TEXT_CHARS] + return text + + +def file_to_text(path: Path) -> str: + """Read a local file and convert it to plain text. + + Args: + path: File path + + Returns: + str: Plain text content + """ + + return bytes_to_text(path.read_bytes(), path.name) diff --git a/spec/main.py b/spec/main.py index 54e4bdd..d02fa11 100644 --- a/spec/main.py +++ b/spec/main.py @@ -1,14 +1,16 @@ """FastAPI application entry point for GENIE framework.""" from contextlib import asynccontextmanager +from pathlib import Path from typing import AsyncGenerator from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles -from spec.core.config import get_settings -from spec.core.logging_config import setup_logging, get_logger from spec.api.v1.router import router as v1_router +from spec.core.config import get_settings +from spec.core.logging_config import get_logger, setup_logging # Setup logging before anything else settings = get_settings() @@ -48,31 +50,23 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: lifespan=lifespan, ) -# Configure CORS middleware (allow all for development) +# Configure CORS: explicit origin allowlist (the SPA is served same-origin, +# so this only matters for external consumers like TabEx during development). app.add_middleware( CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], + allow_origins=[o.strip() for o in settings.cors_origins.split(",") if o.strip()], + allow_credentials=False, + allow_methods=["GET", "POST", "DELETE"], + allow_headers=["Content-Type", "Authorization", "Last-Event-ID"], ) # Include API routers app.include_router(v1_router, prefix="/api/v1") - -@app.get("/") -async def root() -> dict[str, str]: - """Root endpoint. - - Returns: - dict: Welcome message and API version - """ - return { - "message": "GENIE - Generic Extractor of Information Engine", - "version": "0.1.0", - "docs": "/docs", - } +# Serve the SPA (spec/web) at the root path +_WEB_DIR = Path(__file__).parent / "web" +if _WEB_DIR.is_dir(): + app.mount("/", StaticFiles(directory=_WEB_DIR, html=True), name="web") if __name__ == "__main__": diff --git a/spec/models/__init__.py b/spec/models/__init__.py index a5b57c4..6b05e43 100644 --- a/spec/models/__init__.py +++ b/spec/models/__init__.py @@ -1,14 +1,14 @@ """Pydantic data models for GENIE framework.""" -from spec.models.extraction import ExtractionRequest, ExtractionResponse from spec.models.config import ( - InputConfig, - OutputConfig, - LLMConfig, BehaviorConfig, ExtractionConfig, + InputConfig, + LLMConfig, + OutputConfig, ) -from spec.models.library import PatternField, SearchPattern, LibraryMetadata +from spec.models.extraction import ExtractionRequest, ExtractionResponse +from spec.models.library import LibraryMetadata, PatternField, SearchPattern from spec.models.output import FieldDefinition, OutputSchema __all__ = [ diff --git a/spec/models/config.py b/spec/models/config.py index aa26f2e..e0304ab 100644 --- a/spec/models/config.py +++ b/spec/models/config.py @@ -1,6 +1,7 @@ """Configuration models for extraction, input, output, and LLM settings.""" -from typing import Any, Dict, Optional +from typing import Dict, Optional + from pydantic import BaseModel, Field diff --git a/spec/models/extraction.py b/spec/models/extraction.py index e9b32a6..d3739ef 100644 --- a/spec/models/extraction.py +++ b/spec/models/extraction.py @@ -1,7 +1,7 @@ """Pydantic models for extraction requests and responses.""" from typing import Any, Dict, Optional -from datetime import datetime + from pydantic import BaseModel, Field diff --git a/spec/models/library.py b/spec/models/library.py index a62ebbe..7d10692 100644 --- a/spec/models/library.py +++ b/spec/models/library.py @@ -1,7 +1,8 @@ """Models for search library patterns and pattern fields.""" -from typing import List, Optional, Dict, Any from datetime import datetime +from typing import List, Optional + from pydantic import BaseModel, Field diff --git a/spec/models/output.py b/spec/models/output.py index aec8c7b..635b36e 100644 --- a/spec/models/output.py +++ b/spec/models/output.py @@ -1,6 +1,7 @@ """Models for output schema definition and field definitions.""" from typing import Dict, Optional + from pydantic import BaseModel, Field diff --git a/spec/models/provider.py b/spec/models/provider.py index c368186..fc081a3 100644 --- a/spec/models/provider.py +++ b/spec/models/provider.py @@ -1,6 +1,7 @@ """Pydantic models for LLM provider management.""" from typing import Optional + from pydantic import BaseModel, ConfigDict, Field diff --git a/spec/models/webapp.py b/spec/models/webapp.py new file mode 100644 index 0000000..386b98b --- /dev/null +++ b/spec/models/webapp.py @@ -0,0 +1,165 @@ +"""Pydantic models for the GenIE web application (runs, keys, uploads).""" + +from typing import Any, Dict, List, Literal, Optional + +from pydantic import BaseModel, Field + +InputType = Literal["url", "path", "db", "api", "upload"] +OutputType = Literal["url", "path", "db", "api", "download"] +AgentName = Literal["conector", "localizador", "organizador", "sistema"] +EventType = Literal["progress", "log", "done", "error", "finish"] +RunStatus = Literal["queued", "running", "done", "error", "cancelled"] + + +class InputSpec(BaseModel): + """Source specification for the Connector agent. + + Attributes: + type: Input kind (url, path, db, api, upload) + target: Address (URL, filesystem path, DB URL, API endpoint) + user: Username for db connections + password: Password for db connections (transient, never persisted) + token: Bearer token for api connections (transient, never persisted) + query: Optional SQL query for db inputs + upload_id: Upload batch id for upload inputs + """ + + type: InputType + target: str = "" + user: str = "" + password: str = "" + token: str = "" + query: str = "" + upload_id: Optional[str] = None + + +class OutputSpec(BaseModel): + """Destination specification for the Connector agent. + + Attributes: + type: Output kind (url, path, db, api, download) + target: Address (webhook URL, path, DB URL, API endpoint) + user: Username for db destinations + password: Password for db destinations (transient) + token: Bearer token for api destinations (transient) + table: Destination table name for db outputs + """ + + type: OutputType + target: str = "" + user: str = "" + password: str = "" + token: str = "" + table: str = "" + + +class RunRequest(BaseModel): + """Request body for creating an extraction run. + + Attributes: + model_id: Catalog model id (e.g. "gemini-2.5-flash") + input: Input source spec + prompt: What the Locator agent must extract + output: Output destination spec + format: Free-text instructions for the Organizer agent + """ + + model_id: str = Field(..., min_length=1) + input: InputSpec + prompt: str = Field(..., min_length=1) + output: OutputSpec + format: str = "" + + +class RunCreated(BaseModel): + """Response for a created run.""" + + job_id: str + status: RunStatus + + +class RunInfo(BaseModel): + """Current state of a run (no credentials are ever included). + + Attributes: + job_id: Run identifier + status: Current status + model_id: Model used + events: Number of emitted events + result: Final result payload when done + error: Error message when failed + """ + + job_id: str + status: RunStatus + model_id: str + events: int + result: Optional[Dict[str, Any]] = None + error: Optional[str] = None + + +class AgentEvent(BaseModel): + """Event streamed over SSE while a run executes. + + Attributes: + seq: Monotonic sequence number within the run + ts: Wall-clock time, HH:MM:SS + agent: Emitting agent + type: Event kind + level: Log level hint for the UI ("", "ok", "error") + message: Human-readable message + progress: Agent progress 0-100 (when type == progress) + status: Run status (when type == finish/error) + result: Final result payload (when type == finish) + """ + + seq: int + ts: str + agent: AgentName + type: EventType + level: str = "" + message: str = "" + progress: Optional[int] = None + status: Optional[RunStatus] = None + result: Optional[Dict[str, Any]] = None + + +class ModelInfo(BaseModel): + """Catalog entry for a selectable LLM model.""" + + id: str + provider: str + provider_label: str + label: str + note: str + has_key: bool + masked_key: Optional[str] = None + + +class KeyRequest(BaseModel): + """Request body for storing a provider API key.""" + + provider: str = Field(..., min_length=1) + key: str = Field(..., min_length=1) + validate_key: bool = True + + +class KeyInfo(BaseModel): + """Safe response after storing a key (never echoes the key).""" + + provider: str + masked: str + + +class UploadedFile(BaseModel): + """Metadata for one uploaded file.""" + + name: str + size: int + + +class UploadResponse(BaseModel): + """Response for an upload batch.""" + + upload_id: str + files: List[UploadedFile] diff --git a/spec/search_library/matcher.py b/spec/search_library/matcher.py index d704840..e96e17f 100644 --- a/spec/search_library/matcher.py +++ b/spec/search_library/matcher.py @@ -2,7 +2,7 @@ import logging import re -from typing import Any, Dict, Optional +from typing import Any, Dict logger = logging.getLogger(__name__) diff --git a/spec/web/app.js b/spec/web/app.js new file mode 100644 index 0000000..02fa2f3 --- /dev/null +++ b/spec/web/app.js @@ -0,0 +1,738 @@ +/* GenIE SPA — production port of the design prototype (vanilla JS). + Talks to the FastAPI backend: /api/v1/{models,keys,uploads,runs,downloads}. */ + +"use strict"; + +const API = "/api/v1"; + +// ── Icons (stroke = currentColor) ──────────────────────────────────────────── +const ICON_PATHS = { + Key: '', + Plus: '', + Link: '', + Folder: '', + Db: '', + Api: '', + Upload: '', + Down: '', + Send: '', + Stop: '', + X: '', + Copy: '', + Check: '', + Lock: '', + Spark: '', + Brain: '', + Box: '', + Eye: '', + EyeOff: '', + Reload: '', + Chev: '', +}; + +function icon(name, size = 14) { + return ``; +} + +function esc(value) { + return String(value ?? "") + .replace(/&/g, "&").replace(//g, ">") + .replace(/"/g, """).replace(/'/g, "'"); +} + +// ── Catalogs ───────────────────────────────────────────────────────────────── +const CONN_TYPES_IN = [ + { id: "url", label: "URL", icon: "Link", placeholder: "https://exemplo.com/relatorio.pdf" }, + { id: "path", label: "Pasta local", icon: "Folder", placeholder: "/home/voce/Documents/exames" }, + { id: "db", label: "Banco de dados", icon: "Db", placeholder: "sqlite:///dados.db ou postgres://host:5432/db" }, + { id: "api", label: "API REST", icon: "Api", placeholder: "https://api.servico.com/v1/exames" }, + { id: "upload", label: "Upload", icon: "Upload", placeholder: null }, +]; +const CONN_TYPES_OUT = [ + { id: "url", label: "URL (webhook)", icon: "Link", placeholder: "https://hooks.servico.com/genie/abc" }, + { id: "path", label: "Pasta local", icon: "Folder", placeholder: "/home/voce/Documents/saida" }, + { id: "db", label: "Banco de dados", icon: "Db", placeholder: "sqlite:///saida.db" }, + { id: "api", label: "API REST", icon: "Api", placeholder: "https://api.tabex.com/v2/exames" }, + { id: "download", label: "Download", icon: "Down", placeholder: null }, +]; +const AGENTS = [ + { id: "conector", name: "Conector", role: "I/O", desc: "Estabelece conexões de entrada e saída.", icon: "Link" }, + { id: "localizador", name: "Localizador", role: "Extração", desc: "Vasculha o conteúdo e extrai o que foi pedido.", icon: "Brain" }, + { id: "organizador", name: "Organizador", role: "Formato", desc: "Estrutura os dados no formato de saída exigido.", icon: "Box" }, +]; +const KEY_PLACEHOLDERS = { openai: "sk-…", anthropic: "sk-ant-…", google: "AIza…" }; + +// ── State ──────────────────────────────────────────────────────────────────── +const state = { + models: [], + modelId: null, + input: { type: "url", target: "", user: "", password: "", token: "", files: [] }, + output: { type: "api", target: "", user: "", password: "", token: "" }, + run: freshRun(), + eventSource: null, +}; + +function freshRun() { + const agents = {}; + AGENTS.forEach((a) => { agents[a.id] = { status: "idle", progress: 0, message: "" }; }); + return { status: "idle", jobId: null, agents, logs: [], result: null, resultTab: "table" }; +} + +function currentModel() { + return state.models.find((m) => m.id === state.modelId) || null; +} + +// ── API helpers ────────────────────────────────────────────────────────────── +async function apiJson(path, options = {}) { + const response = await fetch(API + path, { + headers: options.body instanceof FormData ? {} : { "Content-Type": "application/json" }, + ...options, + }); + const data = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(data.detail ? String(data.detail) : `HTTP ${response.status}`); + } + return data; +} + +async function loadModels() { + state.models = await apiJson("/models"); + if (!state.modelId || !currentModel()) state.modelId = state.models[0]?.id || null; +} + +// ── Topbar + model section ─────────────────────────────────────────────────── +function renderModelArea() { + const select = document.getElementById("model-select"); + select.innerHTML = state.models + .map((m) => ``) + .join(""); + + const model = currentModel(); + const hasKey = !!model?.has_key; + document.getElementById("model-note").textContent = model?.note || ""; + + const keyState = document.getElementById("key-state"); + keyState.className = `key-state ${hasKey ? "" : "missing"}`; + keyState.innerHTML = `${icon("Lock", 11)} ${hasKey ? "chave configurada" : "sem chave"}`; + + const manageBtn = document.getElementById("manage-key-btn"); + manageBtn.innerHTML = hasKey + ? `${icon("Key", 12)} Gerenciar chave` + : `${icon("Plus", 12)} Nova API Key`; + + const top = document.getElementById("top-key-state"); + top.title = hasKey ? "Chave configurada com segurança" : "Sem chave para este modelo"; + top.innerHTML = ` + + ${esc(model?.provider_label || "")} ${esc(model?.label || "")}`; +} + +// ── Connection sections (02 / 04) ──────────────────────────────────────────── +function renderConnSection(kind) { + const isInput = kind === "in"; + const TYPES = isInput ? CONN_TYPES_IN : CONN_TYPES_OUT; + const value = isInput ? state.input : state.output; + const section = document.getElementById(isInput ? "section-input" : "section-output"); + const type = TYPES.find((t) => t.id === value.type) || TYPES[0]; + const needsCreds = value.type === "db"; + const needsToken = value.type === "api"; + const isUpload = isInput && value.type === "upload"; + const isDownload = !isInput && value.type === "download"; + + const options = TYPES + .map((t) => ``) + .join(""); + + let composite; + if (!isUpload && !isDownload) { + composite = ` +
+ + +
`; + } else { + composite = ` +
+ +
+ ${isUpload ? "arraste arquivos na área abaixo →" : "GenIE devolverá o arquivo no navegador"} +
+
`; + } + + let aux = ""; + if (needsCreds) { + aux = ` +
+
+
+
+
+ + +
+
`; + } + if (needsToken) { + aux = ` +
+
+ + +
`; + } + if (isUpload) { + const fileRows = (value.files || []) + .map((f, i) => ` +
${icon("Box", 12)}${esc(f.name)} + ${fmtSize(f.size)} + +
`) + .join(""); + aux = ` +
+
${icon("Upload", 18)}
+
+
Arraste arquivos aqui ou clique para escolher
+
.pdf .csv .xlsx .json .txt .html — múltiplos arquivos suportados
+
+ +
+ ${value.files?.length ? `
${fileRows}
` : ""}`; + } + if (isDownload) { + aux = ` +
+
${icon("Down", 18)}
+
+
Saída como arquivo de download
+
JSON e CSV — links assinados, válidos por 15 minutos
+
+
`; + } + + let tokens = ""; + if (value.target || isUpload || isDownload) { + const chips = [`${esc(type.label)}`]; + if (value.user) chips.push(`user=${esc(value.user)}`); + if (value.password) chips.push('pass=••••'); + if (value.token) chips.push('token=••••'); + if (isUpload && value.files?.length) chips.push(`${value.files.length} arquivo(s)`); + tokens = `
${chips.join("")}
`; + } + + section.innerHTML = ` +
+
${isInput ? "02" : "04"}
+
${isInput ? "Origem dos dados (entrada)" : "Destino dos dados (saída)"}
+
conector
+
+ ${composite}${aux}${tokens}`; + + wireConnSection(section, kind); +} + +function wireConnSection(section, kind) { + const isInput = kind === "in"; + const value = isInput ? state.input : state.output; + + section.querySelector('[data-role="type"]').addEventListener("change", (e) => { + Object.assign(value, { type: e.target.value, target: "", user: "", password: "", token: "" }); + if (isInput) value.files = []; + renderConnSection(kind); + renderActionBar(); + }); + + const bind = (role, key) => { + const el = section.querySelector(`[data-role="${role}"]`); + if (el) el.addEventListener("input", (e) => { value[key] = e.target.value; renderActionBar(); }); + }; + bind("target", "target"); + bind("user", "user"); + bind("password", "password"); + bind("token", "token"); + + const toggle = section.querySelector('[data-role="toggle-pass"]'); + if (toggle) { + toggle.addEventListener("click", () => { + const field = section.querySelector('[data-role="password"], [data-role="token"]'); + const show = field.type === "password"; + field.type = show ? "text" : "password"; + toggle.innerHTML = show ? icon("EyeOff") : icon("Eye"); + }); + } + + const dropzone = section.querySelector('[data-role="dropzone"]'); + if (dropzone) { + const fileInput = section.querySelector('[data-role="file-input"]'); + dropzone.addEventListener("click", () => fileInput.click()); + dropzone.addEventListener("dragover", (e) => e.preventDefault()); + dropzone.addEventListener("drop", (e) => { + e.preventDefault(); + value.files = [...value.files, ...e.dataTransfer.files]; + renderConnSection(kind); renderActionBar(); + }); + fileInput.addEventListener("change", (e) => { + value.files = [...value.files, ...e.target.files]; + renderConnSection(kind); renderActionBar(); + }); + } + section.querySelectorAll("[data-remove]").forEach((btn) => { + btn.addEventListener("click", () => { + value.files = value.files.filter((_, i) => i !== Number(btn.dataset.remove)); + renderConnSection(kind); renderActionBar(); + }); + }); +} + +function fmtSize(bytes) { + if (bytes < 1024) return bytes + " B"; + if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + " KB"; + return (bytes / (1024 * 1024)).toFixed(1) + " MB"; +} + +// ── Action bar ─────────────────────────────────────────────────────────────── +function canRun() { + const model = currentModel(); + const prompt = document.getElementById("prompt").value.trim(); + const hasInput = state.input.type === "upload" ? state.input.files.length > 0 : !!state.input.target.trim(); + return !!model?.has_key && hasInput && prompt.length > 0 && state.run.status !== "running" && state.run.status !== "starting"; +} + +function renderActionBar() { + const model = currentModel(); + const inText = state.input.type + (state.input.target ? `:${state.input.target.slice(0, 24)}` : state.input.files?.length ? `(${state.input.files.length})` : ""); + const outText = state.output.type + (state.output.target ? `:${state.output.target.slice(0, 24)}` : ""); + document.getElementById("summary").innerHTML = ` + ${esc(model?.label || "—")} + in ${esc(inText || "—")} + out ${esc(outText || "—")}`; + + const slot = document.getElementById("action-buttons"); + if (state.run.status === "running" || state.run.status === "starting") { + slot.innerHTML = ``; + document.getElementById("stop-btn").addEventListener("click", stopRun); + } else { + const clear = state.run.status !== "idle" + ? `` : ""; + slot.innerHTML = `${clear} + `; + document.getElementById("run-btn").addEventListener("click", startRun); + const clearBtn = document.getElementById("clear-btn"); + if (clearBtn) clearBtn.addEventListener("click", () => { closeStream(); state.run = freshRun(); renderMonitor(); renderActionBar(); }); + } +} + +// ── Monitor ────────────────────────────────────────────────────────────────── +function renderMonitor() { + const monitor = document.getElementById("monitor"); + const run = state.run; + + if (run.status === "idle" && run.logs.length === 0) { + monitor.innerHTML = ` +
+
+ Monitor de agentes +

Pronto para orquestrar

+

Três agentes — Conector, Localizador e Organizador — trabalham em sequência: + o conector busca a fonte, o localizador extrai o que você pediu, o organizador formata e o conector + entrega no destino.

+
+ ${AGENTS.map((a) => ` +
+
${icon(a.icon)}
${a.name}
+
${a.desc}
+
`).join("")} +
+
+
`; + return; + } + + const statusLabel = { starting: "iniciando", running: "executando", done: "concluído", error: "erro", cancelled: "interrompido", idle: "ocioso" }[run.status] || run.status; + const pillClass = run.status === "starting" ? "running" : run.status; + const total = Math.round(AGENTS.reduce((s, a) => s + (run.agents[a.id].progress || 0), 0) / AGENTS.length); + + monitor.innerHTML = ` +
+
+ Monitor de agentes + ${statusLabel} +
+ job ${esc(run.jobId || "")} · ${run.logs.length} eventos · ${total}% +
+
${AGENTS.map(agentCardHtml).join("")}
+
+
+
Log de execução
+
${run.logs.map(logLineHtml).join("")}
+
+
${run.result ? outputPreviewHtml() : ""}
+
+
`; + + const log = document.getElementById("log"); + log.scrollTop = log.scrollHeight; + wireOutputPreview(); +} + +function agentCardHtml(agent) { + const s = state.run.agents[agent.id]; + const tag = { idle: "aguardando", active: "executando", done: "concluído" }[s.status] || s.status; + return ` +
+
+
${icon(agent.icon)}
+
+
${agent.name}
${agent.role}
+
+
${tag}
+
+
${esc(s.message || agent.desc)}
+
+
`; +} + +function logLineHtml(line) { + return ` +
+ ${esc(line.t)} + ${esc(line.agent)} + ${esc(line.m)} +
`; +} + +function outputPreviewHtml() { + const result = state.run.result; + if (!result) return ""; + const outType = (CONN_TYPES_OUT.find((t) => t.id === state.output.type) || {}).label || state.output.type; + const target = result.delivered_to?.target || "—"; + const records = Array.isArray(result.records) ? result.records.filter((r) => r && typeof r === "object") : []; + const isTabular = records.length > 0; + const tab = isTabular ? state.run.resultTab : "json"; + + const downloads = Object.entries(result.downloads || {}) + .map(([fmt, url]) => `${icon("Down", 11)} ${esc(fmt.toUpperCase())}`) + .join(""); + + let body; + if (tab === "table" && isTabular) { + const columns = []; + records.forEach((r) => Object.keys(r).forEach((k) => { if (!k.startsWith("_") && !columns.includes(k)) columns.push(k); })); + body = ` + + ${columns.map((c) => ``).join("")} + ${records.map((r) => ` + ${columns.map((c) => { + const out = r._outOfRange?.[c] ? "v out-of-range" : "v"; + const cell = r[c] === null || r[c] === undefined ? "" : typeof r[c] === "object" ? JSON.stringify(r[c]) : String(r[c]); + return ``; + }).join("")}`).join("")} + +
${esc(c)}
${esc(cell)}
`; + } else { + body = `
${esc(JSON.stringify(result, null, 2))}
`; + } + + return ` +
+
+ Saída entregue + ${icon("Send", 11)}→${esc(outType)} · ${esc(target)} +
+ ${downloads} +
+ + +
+ +
+
+
${body}
+
`; +} + +function wireOutputPreview() { + const tabTable = document.getElementById("tab-table"); + const tabJson = document.getElementById("tab-json"); + const copyBtn = document.getElementById("copy-json"); + if (tabTable) tabTable.addEventListener("click", () => { state.run.resultTab = "table"; renderMonitor(); }); + if (tabJson) tabJson.addEventListener("click", () => { state.run.resultTab = "json"; renderMonitor(); }); + if (copyBtn) copyBtn.addEventListener("click", () => { + navigator.clipboard?.writeText(JSON.stringify(state.run.result, null, 2)); + copyBtn.innerHTML = icon("Check"); + setTimeout(() => { copyBtn.innerHTML = icon("Copy"); }, 1500); + }); +} + +// ── Run engine (real backend) ──────────────────────────────────────────────── +function closeStream() { + if (state.eventSource) { state.eventSource.close(); state.eventSource = null; } +} + +async function startRun() { + if (!canRun()) return; + closeStream(); + state.run = freshRun(); + state.run.status = "starting"; + renderActionBar(); + renderMonitor(); + + const prompt = document.getElementById("prompt").value.trim(); + const format = document.getElementById("format").value; + + try { + let uploadId = null; + if (state.input.type === "upload") { + const form = new FormData(); + state.input.files.forEach((file) => form.append("files", file)); + pushLocalLog("sistema", `Enviando ${state.input.files.length} arquivo(s)…`); + const upload = await apiJson("/uploads", { method: "POST", body: form }); + uploadId = upload.upload_id; + pushLocalLog("sistema", `Upload concluído (${upload.files.length} arquivo(s))`, "ok"); + } + + const body = { + model_id: state.modelId, + input: { + type: state.input.type, + target: state.input.target, + user: state.input.user, + password: state.input.password, + token: state.input.token, + upload_id: uploadId, + }, + prompt, + output: { + type: state.output.type, + target: state.output.target, + user: state.output.user, + password: state.output.password, + token: state.output.token, + }, + format, + }; + + const created = await apiJson("/runs", { method: "POST", body: JSON.stringify(body) }); + state.run.jobId = created.job_id; + state.run.status = "running"; + state.run.agents.conector.status = "active"; + renderActionBar(); + subscribe(created.job_id); + } catch (error) { + state.run.status = "error"; + pushLocalLog("sistema", String(error.message || error), "error"); + renderActionBar(); + renderMonitor(); + } +} + +function subscribe(jobId) { + const source = new EventSource(`${API}/runs/${jobId}/events`); + state.eventSource = source; + + source.onmessage = (message) => { + let event; + try { event = JSON.parse(message.data); } catch { return; } + applyEvent(event); + }; + source.onerror = async () => { + if (state.run.status !== "running") { closeStream(); return; } + // EventSource auto-reconnects with Last-Event-ID; double-check job state. + try { + const info = await apiJson(`/runs/${jobId}`); + if (info.status !== "running" && info.status !== "queued") { + closeStream(); + state.run.status = info.status; + if (info.result) state.run.result = info.result; + if (info.error) pushLocalLog("sistema", info.error, "error"); + renderActionBar(); renderMonitor(); + } + } catch { /* transient; let EventSource retry */ } + }; +} + +function applyEvent(event) { + const run = state.run; + const agents = run.agents; + + if (agents[event.agent]) { + const agent = agents[event.agent]; + if (event.type === "done") { + agent.status = "done"; agent.progress = 100; + if (event.message) agent.message = event.message; + } else { + if (agent.status !== "done") agent.status = "active"; + if (typeof event.progress === "number") agent.progress = event.progress; + if (event.message) agent.message = event.message; + } + } + + if (event.message) { + run.logs.push({ t: event.ts, agent: event.agent, m: event.message, level: event.level || "" }); + } + + if (event.type === "finish") { + run.status = event.status || "done"; + run.result = event.result || null; + closeStream(); + renderActionBar(); + } else if (event.type === "error") { + run.status = event.status || "error"; + closeStream(); + renderActionBar(); + } + + renderMonitor(); +} + +function pushLocalLog(agent, message, level = "") { + const t = new Date().toLocaleTimeString("pt-BR", { hour12: false }).slice(0, 8); + state.run.logs.push({ t, agent, m: message, level }); + renderMonitor(); +} + +async function stopRun() { + const jobId = state.run.jobId; + closeStream(); + if (jobId) { + try { + const info = await apiJson(`/runs/${jobId}/cancel`, { method: "POST" }); + state.run.status = info.status || "cancelled"; + } catch { state.run.status = "cancelled"; } + } else { + state.run.status = "idle"; + } + renderActionBar(); + renderMonitor(); +} + +// ── API Key modal ──────────────────────────────────────────────────────────── +function openKeyModal() { + const model = currentModel(); + if (!model) return; + const root = document.getElementById("modal-root"); + const masked = model.masked_key || ""; + + root.innerHTML = ` + `; + + const back = document.getElementById("modal-back"); + const input = document.getElementById("key-input"); + const saveBtn = document.getElementById("key-save"); + const errorRow = document.getElementById("key-error"); + const close = () => { root.innerHTML = ""; }; + + back.addEventListener("click", close); + document.getElementById("modal").addEventListener("click", (e) => e.stopPropagation()); + document.getElementById("key-cancel").addEventListener("click", close); + input.addEventListener("input", () => { saveBtn.disabled = !input.value.trim(); }); + input.focus(); + + saveBtn.addEventListener("click", async () => { + saveBtn.disabled = true; + saveBtn.innerHTML = ` Validando…`; + errorRow.classList.remove("show"); + try { + await apiJson("/keys", { + method: "POST", + body: JSON.stringify({ provider: model.provider, key: input.value.trim(), validate_key: true }), + }); + input.value = ""; + close(); + await loadModels(); + renderModelArea(); renderActionBar(); + } catch (error) { + errorRow.textContent = String(error.message || error); + errorRow.classList.add("show"); + saveBtn.innerHTML = `${icon("Check", 12)} Salvar`; + saveBtn.disabled = false; + } + }); + + const removeBtn = document.getElementById("key-remove"); + if (removeBtn) { + removeBtn.addEventListener("click", async () => { + try { + await apiJson(`/keys/${encodeURIComponent(model.provider)}`, { method: "DELETE" }); + close(); + await loadModels(); + renderModelArea(); renderActionBar(); + } catch (error) { + errorRow.textContent = String(error.message || error); + errorRow.classList.add("show"); + } + }); + } +} + +// ── Example ────────────────────────────────────────────────────────────────── +function loadExample() { + state.input = { type: "upload", target: "", user: "", password: "", token: "", files: state.input.type === "upload" ? state.input.files : [] }; + document.getElementById("prompt").value = + "Extraia, para cada exame encontrado nos arquivos enviados, os campos: Data, Nome do Exame, Resultado e Valor de Referência. Ignore cabeçalhos, rodapés e dados de contato do laboratório."; + state.output = { type: "download", target: "", user: "", password: "", token: "" }; + document.getElementById("format").value = + 'Gere um registro JSON por exame com os campos:\n{\n "data": "YYYY-MM-DD",\n "exame": "",\n "resultado": "",\n "referencia": ""\n}'; + renderConnSection("in"); + renderConnSection("out"); + renderActionBar(); +} + +// ── Boot ───────────────────────────────────────────────────────────────────── +async function boot() { + document.getElementById("model-select").addEventListener("change", (e) => { + state.modelId = e.target.value; + renderModelArea(); renderActionBar(); + }); + document.getElementById("manage-key-btn").addEventListener("click", openKeyModal); + document.getElementById("example-btn").addEventListener("click", loadExample); + document.getElementById("prompt").addEventListener("input", renderActionBar); + document.querySelectorAll("[data-icon]").forEach((el) => { el.innerHTML = icon(el.dataset.icon, 11); }); + + renderConnSection("in"); + renderConnSection("out"); + renderMonitor(); + + try { + await loadModels(); + } catch (error) { + pushLocalLog("sistema", `Falha ao carregar modelos: ${error.message || error}`, "error"); + } + renderModelArea(); + renderActionBar(); +} + +boot(); diff --git a/spec/web/index.html b/spec/web/index.html new file mode 100644 index 0000000..01c2db7 --- /dev/null +++ b/spec/web/index.html @@ -0,0 +1,90 @@ + + + + + + GenIE — Generic Extractor of Information Engine + + + + + + +
+
+
+
+
+ GenIE + v1.0 +
+
+
Workspace Pipeline padrão
+
+
+
+ +
+ +
+
+ + +
+
+
01
+
Modelo de IA
+
+
+
+
+ +
…
+
+ +
+
+ + +
+ + +
+
+
03
+
O que extrair
+
localizador
+ +
+ +
+ + +
+ + +
+
+
05
+
Formato da saída
+
organizador
+
+ +
+
+ +
+
+ +
+
+ + +
+
+
+ +
+ + + diff --git a/spec/web/styles.css b/spec/web/styles.css new file mode 100644 index 0000000..e636b97 --- /dev/null +++ b/spec/web/styles.css @@ -0,0 +1,939 @@ +/* GenIE — single page app */ + +:root { + --bg: oklch(0.16 0.012 265); + --bg-1: oklch(0.20 0.013 265); + --bg-2: oklch(0.23 0.014 265); + --bg-3: oklch(0.27 0.014 265); + --line: oklch(0.30 0.015 265); + --line-strong: oklch(0.38 0.018 265); + --fg: oklch(0.97 0.005 265); + --fg-1: oklch(0.82 0.008 265); + --fg-2: oklch(0.62 0.012 265); + --fg-3: oklch(0.46 0.012 265); + + --accent: oklch(0.72 0.18 290); + --accent-soft: oklch(0.72 0.18 290 / 0.14); + --accent-line: oklch(0.72 0.18 290 / 0.35); + --cyan: oklch(0.80 0.13 200); + --cyan-soft: oklch(0.80 0.13 200 / 0.14); + --green: oklch(0.78 0.15 155); + --amber: oklch(0.82 0.14 80); + --red: oklch(0.70 0.20 25); + + --r-xs: 4px; + --r-sm: 6px; + --r-md: 10px; + --r-lg: 14px; + --r-xl: 20px; + + --shadow-1: 0 1px 0 0 oklch(1 0 0 / 0.04) inset, 0 1px 2px oklch(0 0 0 / 0.4); + --shadow-2: 0 10px 30px -10px oklch(0 0 0 / 0.6), 0 1px 0 0 oklch(1 0 0 / 0.04) inset; + + --font-sans: "Geist", "Inter", system-ui, -apple-system, "Segoe UI", sans-serif; + --font-mono: "JetBrains Mono", "Geist Mono", ui-monospace, "SF Mono", Menlo, monospace; +} + +* { + box-sizing: border-box; +} + +html, body, #root { + height: 100%; + margin: 0; + background: var(--bg); + color: var(--fg); + font-family: var(--font-sans); + font-size: 14px; + line-height: 1.45; + -webkit-font-smoothing: antialiased; + letter-spacing: -0.005em; +} + +button { + font-family: inherit; + font-size: inherit; + color: inherit; + cursor: pointer; +} + +input, textarea, select { + font-family: inherit; + font-size: inherit; + color: inherit; +} + +textarea { + resize: none; +} + +/* ───── Layout ───── */ +.app { + display: grid; + grid-template-rows: 56px 1fr; + height: 100vh; + overflow: hidden; +} + +.topbar { + display: flex; + align-items: center; + gap: 16px; + padding: 0 20px; + border-bottom: 1px solid var(--line); + background: linear-gradient(180deg, var(--bg-1), var(--bg)); +} + +.brand { + display: flex; + align-items: center; + gap: 10px; + font-weight: 600; + letter-spacing: -0.02em; + font-size: 15px; +} +.brand .mark { + width: 26px; height: 26px; + border-radius: 7px; + background: + radial-gradient(circle at 30% 30%, oklch(0.90 0.14 290), oklch(0.55 0.22 290) 70%); + box-shadow: 0 0 0 1px oklch(1 0 0 / 0.06) inset, 0 4px 14px -4px oklch(0.55 0.22 290 / 0.7); + position: relative; +} +.brand .mark::after { + content: ""; + position: absolute; + inset: 4px; + border-radius: 4px; + background: radial-gradient(circle at 70% 70%, transparent 40%, oklch(0 0 0 / 0.4)); + mix-blend-mode: multiply; +} +.brand .ver { + font-family: var(--font-mono); + font-size: 10px; + color: var(--fg-3); + padding: 2px 6px; + border: 1px solid var(--line); + border-radius: 4px; + margin-left: 4px; + letter-spacing: 0; +} + +.topbar .spacer { flex: 1; } + +.topbar .crumb { + font-size: 12px; + color: var(--fg-2); + display: flex; + gap: 8px; + align-items: center; +} +.topbar .crumb b { + color: var(--fg-1); + font-weight: 500; +} + +.workspace { + display: grid; + grid-template-columns: minmax(440px, 46%) 1fr; + min-height: 0; +} + +.pane { + display: flex; + flex-direction: column; + min-height: 0; + min-width: 0; +} + +.pane.config { + border-right: 1px solid var(--line); + background: var(--bg); + overflow-y: auto; +} + +.pane.monitor { + background: + radial-gradient(80% 60% at 50% -10%, oklch(0.72 0.18 290 / 0.06), transparent 70%), + var(--bg); + overflow: hidden; +} + +/* ───── Form ───── */ +.form { + padding: 20px 24px 90px; + display: flex; + flex-direction: column; + gap: 22px; + max-width: 720px; +} + +.section { + display: flex; + flex-direction: column; + gap: 10px; +} + +.section-head { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 2px; +} +.section-head .num { + width: 18px; height: 18px; + border-radius: 5px; + background: var(--bg-2); + border: 1px solid var(--line); + display: grid; place-items: center; + font-family: var(--font-mono); + font-size: 10px; + color: var(--fg-2); +} +.section-head .label { + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--fg-2); + font-weight: 500; +} +.section-head .hint { + margin-left: auto; + font-size: 11px; + color: var(--fg-3); +} + +/* Field group */ +.field-row { + display: grid; + gap: 8px; +} +.field-row.cols-2 { grid-template-columns: 1fr 1fr; } +.field-row.cols-3 { grid-template-columns: 1fr 1fr 1fr; } +.field-row.split { grid-template-columns: 180px 1fr; } + +.field { + display: flex; + flex-direction: column; + gap: 5px; +} +.field > label { + font-size: 11px; + color: var(--fg-2); + font-weight: 500; + letter-spacing: 0.01em; +} + +/* Control base */ +.ctl { + background: var(--bg-1); + border: 1px solid var(--line); + border-radius: var(--r-md); + padding: 9px 12px; + color: var(--fg); + outline: none; + width: 100%; + transition: border-color .15s, background .15s, box-shadow .15s; + font-size: 13.5px; +} +.ctl:hover { + border-color: var(--line-strong); +} +.ctl:focus, .ctl:focus-within { + border-color: var(--accent-line); + box-shadow: 0 0 0 3px var(--accent-soft); + background: var(--bg-1); +} +.ctl::placeholder { color: var(--fg-3); } + +select.ctl { + appearance: none; + background-image: + linear-gradient(45deg, transparent 50%, var(--fg-2) 50%), + linear-gradient(135deg, var(--fg-2) 50%, transparent 50%); + background-position: + calc(100% - 16px) 16px, + calc(100% - 11px) 16px; + background-size: 5px 5px; + background-repeat: no-repeat; + padding-right: 32px; +} + +textarea.ctl { + font-family: var(--font-sans); + line-height: 1.55; + padding: 11px 13px; +} + +/* Mono input */ +.ctl.mono { + font-family: var(--font-mono); + font-size: 12.5px; + letter-spacing: -0.01em; +} + +/* Composite: select + input */ +.composite { + display: grid; + grid-template-columns: 150px 1fr; + border: 1px solid var(--line); + border-radius: var(--r-md); + overflow: hidden; + background: var(--bg-1); + transition: border-color .15s, box-shadow .15s; +} +.composite:focus-within { + border-color: var(--accent-line); + box-shadow: 0 0 0 3px var(--accent-soft); +} +.composite > select, +.composite > input { + background: transparent; + border: 0; + outline: none; + padding: 10px 12px; + color: var(--fg); + font-size: 13.5px; +} +.composite > select { + border-right: 1px solid var(--line); + appearance: none; + background-image: + linear-gradient(45deg, transparent 50%, var(--fg-2) 50%), + linear-gradient(135deg, var(--fg-2) 50%, transparent 50%); + background-position: + calc(100% - 14px) 17px, + calc(100% - 9px) 17px; + background-size: 5px 5px; + background-repeat: no-repeat; + padding-right: 30px; + color: var(--fg-1); + font-weight: 500; + font-size: 12.5px; +} +.composite > input.mono { + font-family: var(--font-mono); + font-size: 12.5px; +} + +/* Model picker */ +.model-picker { + display: flex; + align-items: stretch; + gap: 8px; +} +.model-picker .composite { + flex: 1; + grid-template-columns: 1fr auto; +} +.model-picker .composite .key-state { + display: flex; + align-items: center; + gap: 6px; + padding: 0 12px; + font-size: 11px; + color: var(--fg-2); + background: var(--bg-2); + border-left: 1px solid var(--line); +} +.model-picker .composite .key-state .dot { + width: 7px; height: 7px; border-radius: 50%; + background: var(--green); + box-shadow: 0 0 0 3px oklch(0.78 0.15 155 / 0.18); +} +.model-picker .composite .key-state.missing .dot { + background: var(--amber); + box-shadow: 0 0 0 3px oklch(0.82 0.14 80 / 0.18); +} + +.btn { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 9px 14px; + border-radius: var(--r-md); + border: 1px solid var(--line); + background: var(--bg-1); + color: var(--fg-1); + font-size: 12.5px; + font-weight: 500; + transition: border-color .15s, background .15s, color .15s, transform .05s; + white-space: nowrap; +} +.btn:hover { border-color: var(--line-strong); background: var(--bg-2); color: var(--fg); } +.btn:active { transform: translateY(1px); } + +.btn.primary { + background: linear-gradient(180deg, oklch(0.78 0.16 290), oklch(0.62 0.20 290)); + border-color: oklch(0.55 0.22 290); + color: white; + font-weight: 600; + box-shadow: 0 4px 14px -4px oklch(0.55 0.22 290 / 0.55), 0 1px 0 oklch(1 0 0 / 0.18) inset; +} +.btn.primary:hover { filter: brightness(1.08); } +.btn.primary:disabled { + background: var(--bg-2); + color: var(--fg-3); + border-color: var(--line); + box-shadow: none; + cursor: not-allowed; +} + +.btn.ghost { + background: transparent; +} +.btn.icon-only { + padding: 8px; + width: 34px; + justify-content: center; +} + +/* Upload zone */ +.dropzone { + border: 1px dashed var(--line-strong); + border-radius: var(--r-md); + padding: 16px; + background: var(--bg-1); + display: flex; + align-items: center; + gap: 12px; + cursor: pointer; + transition: border-color .15s, background .15s; +} +.dropzone:hover { border-color: var(--accent-line); background: var(--accent-soft); } +.dropzone .ico { + width: 36px; height: 36px; + border-radius: 8px; + background: var(--bg-2); + display: grid; place-items: center; + color: var(--fg-2); + font-family: var(--font-mono); + font-size: 14px; +} +.dropzone .label { + flex: 1; + display: flex; + flex-direction: column; + gap: 2px; +} +.dropzone .label .t { font-size: 13px; color: var(--fg); } +.dropzone .label .s { font-size: 11px; color: var(--fg-3); font-family: var(--font-mono); } + +.files { + display: flex; + flex-direction: column; + gap: 4px; + margin-top: 4px; +} +.file-row { + display: flex; + align-items: center; + gap: 8px; + padding: 7px 10px; + background: var(--bg-1); + border: 1px solid var(--line); + border-radius: var(--r-sm); + font-size: 12px; +} +.file-row .name { flex: 1; font-family: var(--font-mono); font-size: 12px; } +.file-row .size { color: var(--fg-3); font-family: var(--font-mono); font-size: 11px; } +.file-row .x { + background: none; border: 0; color: var(--fg-3); padding: 2px 6px; border-radius: 3px; +} +.file-row .x:hover { background: var(--bg-3); color: var(--fg); } + +/* Token chips for connection summary */ +.tokens { + display: flex; flex-wrap: wrap; gap: 6px; + margin-top: 2px; +} +.tok { + font-family: var(--font-mono); + font-size: 10.5px; + padding: 3px 7px; + border-radius: 4px; + background: var(--bg-2); + border: 1px solid var(--line); + color: var(--fg-2); +} + +/* ───── Footer action bar ───── */ +.action-bar { + position: sticky; + bottom: 0; + background: linear-gradient(180deg, transparent, var(--bg) 30%); + padding: 16px 24px 18px; + margin-top: auto; + display: flex; + align-items: center; + gap: 12px; + pointer-events: none; +} +.action-bar > * { pointer-events: auto; } +.action-bar .summary { + flex: 1; + font-size: 11.5px; + color: var(--fg-2); + display: flex; gap: 14px; + font-family: var(--font-mono); +} +.action-bar .summary b { color: var(--fg); font-weight: 500; } + +/* ───── Monitor ───── */ +.monitor-inner { + display: grid; + grid-template-rows: auto auto 1fr auto; + height: 100%; + min-height: 0; + padding: 18px 22px 22px; + gap: 14px; +} + +.monitor-head { + display: flex; + align-items: center; + gap: 10px; +} +.monitor-head .title { + font-size: 13px; + font-weight: 600; +} +.monitor-head .status-pill { + font-family: var(--font-mono); + font-size: 10.5px; + padding: 3px 8px 3px 7px; + border-radius: 999px; + background: var(--bg-2); + border: 1px solid var(--line); + color: var(--fg-2); + display: flex; align-items: center; gap: 6px; +} +.monitor-head .status-pill .dot { + width: 6px; height: 6px; border-radius: 50%; + background: var(--fg-3); +} +.monitor-head .status-pill.running .dot { + background: var(--accent); + box-shadow: 0 0 0 3px var(--accent-soft); + animation: pulse 1.4s ease-in-out infinite; +} +.monitor-head .status-pill.done .dot { background: var(--green); } +.monitor-head .status-pill.error .dot { background: var(--red); } +.monitor-head .spacer { flex: 1; } +.monitor-head .meta { + font-family: var(--font-mono); + font-size: 10.5px; + color: var(--fg-3); +} + +@keyframes pulse { + 0%, 100% { box-shadow: 0 0 0 3px var(--accent-soft); } + 50% { box-shadow: 0 0 0 6px oklch(0.72 0.18 290 / 0.05); } +} + +/* Agents */ +.agents { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 8px; + position: relative; +} +.agent { + position: relative; + border: 1px solid var(--line); + border-radius: var(--r-lg); + padding: 14px 14px 12px; + background: var(--bg-1); + display: flex; + flex-direction: column; + gap: 8px; + transition: border-color .25s, background .25s; + overflow: hidden; +} +.agent.active { + border-color: var(--accent-line); + background: linear-gradient(180deg, var(--accent-soft), var(--bg-1) 70%); + box-shadow: 0 0 0 1px var(--accent-line), 0 8px 24px -10px oklch(0.55 0.22 290 / 0.4); +} +.agent.done { + border-color: oklch(0.78 0.15 155 / 0.35); +} +.agent .agent-head { + display: flex; align-items: center; gap: 10px; +} +.agent .ico { + width: 28px; height: 28px; + border-radius: 7px; + background: var(--bg-2); + border: 1px solid var(--line); + display: grid; place-items: center; + font-family: var(--font-mono); + font-size: 12px; + color: var(--fg-2); +} +.agent.active .ico { color: var(--accent); border-color: var(--accent-line); background: oklch(0.72 0.18 290 / 0.10); } +.agent.done .ico { color: var(--green); border-color: oklch(0.78 0.15 155 / 0.35); } +.agent .name { font-size: 13px; font-weight: 600; letter-spacing: -0.01em; } +.agent .role { font-size: 10.5px; color: var(--fg-3); font-family: var(--font-mono); text-transform: uppercase; letter-spacing: 0.06em; } +.agent .desc { font-size: 11.5px; color: var(--fg-2); line-height: 1.4; } +.agent .progress { + height: 3px; + background: var(--bg-2); + border-radius: 2px; + overflow: hidden; + margin-top: 2px; +} +.agent .progress > span { + display: block; + height: 100%; + background: var(--accent); + width: 0%; + transition: width .3s ease; +} +.agent.done .progress > span { background: var(--green); width: 100% !important; } +.agent .status-tag { + font-family: var(--font-mono); + font-size: 10px; + letter-spacing: 0.05em; + text-transform: uppercase; + color: var(--fg-3); + margin-left: auto; +} +.agent.active .status-tag { color: var(--accent); } +.agent.done .status-tag { color: var(--green); } + +/* Flow line connectors */ +.agents::before, .agents::after { + content: ""; + position: absolute; + top: 32px; + height: 1px; + background: linear-gradient(90deg, transparent, var(--line) 30%, var(--line) 70%, transparent); + z-index: 0; +} +.agents::before { left: 33%; right: 66%; } +.agents::after { left: 66%; right: 33%; } + +/* Body: split between log and output preview */ +.monitor-body { + display: grid; + grid-template-rows: 1fr auto; + gap: 10px; + min-height: 0; +} + +.panel { + background: var(--bg-1); + border: 1px solid var(--line); + border-radius: var(--r-lg); + display: flex; + flex-direction: column; + min-height: 0; + overflow: hidden; +} +.panel .panel-head { + display: flex; + align-items: center; + gap: 8px; + padding: 9px 12px; + border-bottom: 1px solid var(--line); + font-size: 11px; + color: var(--fg-2); + text-transform: uppercase; + letter-spacing: 0.08em; + font-weight: 500; +} +.panel .panel-head .tabs { + display: flex; gap: 2px; margin-left: auto; +} +.panel .panel-head .tab { + padding: 4px 9px; + border-radius: 5px; + font-size: 11px; + background: transparent; + border: 1px solid transparent; + color: var(--fg-3); + text-transform: none; + letter-spacing: 0; + font-weight: 500; +} +.panel .panel-head .tab.active { + background: var(--bg-2); + border-color: var(--line); + color: var(--fg-1); +} + +/* Log */ +.log { + flex: 1; + min-height: 0; + overflow-y: auto; + padding: 10px 12px; + font-family: var(--font-mono); + font-size: 12px; + line-height: 1.55; + color: var(--fg-1); +} +.log .line { + display: grid; + grid-template-columns: 60px 80px 1fr; + gap: 10px; + padding: 2px 0; + align-items: baseline; +} +.log .line .t { color: var(--fg-3); font-size: 10.5px; } +.log .line .a { font-size: 10.5px; } +.log .line .a.conector { color: var(--cyan); } +.log .line .a.localizador { color: var(--accent); } +.log .line .a.organizador { color: oklch(0.82 0.14 80); } +.log .line .a.sistema { color: var(--fg-2); } +.log .line .m { color: var(--fg-1); white-space: pre-wrap; word-break: break-word; } +.log .line.error .m { color: var(--red); } +.log .line.ok .m { color: var(--green); } +.log .empty { + display: grid; place-items: center; + height: 100%; color: var(--fg-3); + font-size: 12px; + font-family: var(--font-mono); +} +.log .empty .big { font-size: 14px; color: var(--fg-2); margin-bottom: 6px; } + +.cursor { + display: inline-block; + width: 7px; height: 12px; + background: var(--accent); + vertical-align: text-bottom; + margin-left: 2px; + animation: blink 1s steps(2, end) infinite; +} +@keyframes blink { 50% { opacity: 0; } } + +/* Output preview */ +.output { + border: 1px solid var(--line); + border-radius: var(--r-lg); + background: var(--bg-1); + overflow: hidden; + display: flex; + flex-direction: column; + max-height: 280px; +} +.output .output-head { + display: flex; + align-items: center; + gap: 10px; + padding: 10px 12px; + border-bottom: 1px solid var(--line); +} +.output .output-head .t { + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--fg-2); + font-weight: 500; +} +.output .output-head .target { + font-family: var(--font-mono); + font-size: 11px; + color: var(--fg-1); + display: flex; align-items: center; gap: 6px; +} +.output .output-head .target .arrow { color: var(--fg-3); } +.output .output-head .actions { margin-left: auto; display: flex; gap: 6px; } +.output .output-body { + flex: 1; + min-height: 0; + overflow: auto; + padding: 0; +} + +/* Result table */ +table.result { + width: 100%; + border-collapse: collapse; + font-size: 12px; +} +table.result th, +table.result td { + text-align: left; + padding: 8px 12px; + border-bottom: 1px solid var(--line); + font-family: var(--font-mono); + font-size: 11.5px; +} +table.result th { + font-weight: 500; + font-size: 10.5px; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--fg-2); + background: var(--bg-2); + position: sticky; top: 0; +} +table.result td .v { + color: var(--fg-1); +} +table.result td .out-of-range { + color: var(--amber); +} +table.result tr:last-child td { border-bottom: 0; } + +/* Empty monitor */ +.monitor-empty { + height: 100%; + display: grid; + place-items: center; + text-align: center; + color: var(--fg-3); + padding: 24px; +} +.monitor-empty .badge { + font-family: var(--font-mono); + font-size: 10px; + padding: 3px 8px; + border-radius: 4px; + background: var(--bg-2); + border: 1px solid var(--line); + color: var(--fg-2); + letter-spacing: 0.06em; + text-transform: uppercase; + margin-bottom: 14px; + display: inline-block; +} +.monitor-empty h2 { + margin: 0 0 6px; + font-size: 18px; + font-weight: 600; + color: var(--fg); + letter-spacing: -0.02em; +} +.monitor-empty p { + margin: 0; + font-size: 12.5px; + max-width: 380px; + line-height: 1.55; + color: var(--fg-2); +} + +/* Toggle group */ +.toggle-group { + display: inline-flex; + background: var(--bg-1); + border: 1px solid var(--line); + border-radius: var(--r-md); + padding: 3px; + gap: 2px; +} +.toggle-group .opt { + padding: 6px 10px; + border-radius: 6px; + background: transparent; + border: 0; + color: var(--fg-2); + font-size: 12px; + display: flex; align-items: center; gap: 6px; +} +.toggle-group .opt[aria-pressed="true"] { + background: var(--bg-3); + color: var(--fg); + box-shadow: 0 1px 0 oklch(1 0 0 / 0.04) inset; +} +.toggle-group .opt svg { width: 12px; height: 12px; } + +/* Modal */ +.modal-back { + position: fixed; inset: 0; + background: oklch(0 0 0 / 0.5); + backdrop-filter: blur(4px); + display: grid; place-items: center; + z-index: 60; + animation: fadein .2s ease; +} +@keyframes fadein { from { opacity: 0; } } +.modal { + background: var(--bg-1); + border: 1px solid var(--line); + border-radius: var(--r-lg); + padding: 22px; + width: min(440px, 92vw); + box-shadow: var(--shadow-2); +} +.modal h3 { + margin: 0 0 4px; + font-size: 15px; + font-weight: 600; +} +.modal p { + margin: 0 0 16px; + font-size: 12.5px; + color: var(--fg-2); + line-height: 1.55; +} +.modal .row { display: flex; gap: 8px; justify-content: flex-end; margin-top: 12px; } +.modal .hint-row { + font-size: 11px; color: var(--fg-3); + display: flex; gap: 6px; align-items: center; + margin-top: 6px; + font-family: var(--font-mono); +} +.modal .hint-row svg { width: 11px; height: 11px; } + +/* Misc */ +.kbd { + font-family: var(--font-mono); + font-size: 10.5px; + padding: 1px 5px; + border-radius: 3px; + background: var(--bg-3); + border: 1px solid var(--line-strong); + color: var(--fg-1); +} + +/* Scrollbars */ +*::-webkit-scrollbar { width: 10px; height: 10px; } +*::-webkit-scrollbar-thumb { background: var(--bg-3); border-radius: 5px; border: 2px solid var(--bg); } +*::-webkit-scrollbar-track { background: transparent; } + +/* SVG icon defaults */ +.icon { width: 14px; height: 14px; flex-shrink: 0; } + +/* ───── Additions for the production app ───── */ +.monitor-head .status-pill.cancelled .dot { background: var(--amber); } +.monitor-head .status-pill.error .dot { background: var(--red); } + +.output .output-head .dl { + display: inline-flex; + align-items: center; + gap: 5px; + padding: 5px 10px; + border-radius: var(--r-sm); + border: 1px solid var(--line); + background: var(--bg-2); + color: var(--fg-1); + font-size: 11px; + font-family: var(--font-mono); + text-decoration: none; +} +.output .output-head .dl:hover { border-color: var(--accent-line); color: var(--fg); } + +.modal .error-row { + margin-top: 10px; + padding: 8px 10px; + border-radius: var(--r-sm); + border: 1px solid oklch(0.70 0.20 25 / 0.4); + background: oklch(0.70 0.20 25 / 0.10); + color: var(--red); + font-size: 12px; + display: none; +} +.modal .error-row.show { display: block; } + +.btn .spin { + width: 11px; height: 11px; + border: 2px solid currentColor; + border-top-color: transparent; + border-radius: 50%; + animation: rotate 0.8s linear infinite; + display: inline-block; +} +@keyframes rotate { to { transform: rotate(360deg); } } diff --git a/spec/webapp/__init__.py b/spec/webapp/__init__.py new file mode 100644 index 0000000..563c979 --- /dev/null +++ b/spec/webapp/__init__.py @@ -0,0 +1 @@ +"""GenIE web application layer: model catalog and job management.""" diff --git a/spec/webapp/catalog.py b/spec/webapp/catalog.py new file mode 100644 index 0000000..1cc78f3 --- /dev/null +++ b/spec/webapp/catalog.py @@ -0,0 +1,61 @@ +"""Catalog of selectable LLM models exposed to the web UI.""" + +from typing import Any, Dict, List, Optional + +MODELS: List[Dict[str, Any]] = [ + { + "id": "gemini-2.5-flash", + "provider": "google", + "provider_label": "Google", + "label": "Gemini 2.5 Flash", + "note": "rápido · multimodal", + }, + { + "id": "gemini-2.5-pro", + "provider": "google", + "provider_label": "Google", + "label": "Gemini 2.5 Pro", + "note": "raciocínio profundo", + }, + { + "id": "gpt-4o-mini", + "provider": "openai", + "provider_label": "OpenAI", + "label": "GPT-4o mini", + "note": "barato · estruturado", + }, + { + "id": "gpt-4o", + "provider": "openai", + "provider_label": "OpenAI", + "label": "GPT-4o", + "note": "multimodal · alta qualidade", + }, + { + "id": "claude-sonnet-4-6", + "provider": "anthropic", + "provider_label": "Anthropic", + "label": "Claude Sonnet 4.6", + "note": "raciocínio + extração", + }, + { + "id": "claude-haiku-4-5-20251001", + "provider": "anthropic", + "provider_label": "Anthropic", + "label": "Claude Haiku 4.5", + "note": "rápido · econômico", + }, +] + + +def find_model(model_id: str) -> Optional[Dict[str, Any]]: + """Look up a catalog entry by model id. + + Args: + model_id: Catalog model id + + Returns: + Optional[dict]: Catalog entry or None + """ + + return next((m for m in MODELS if m["id"] == model_id), None) diff --git a/spec/webapp/jobs.py b/spec/webapp/jobs.py new file mode 100644 index 0000000..fb07595 --- /dev/null +++ b/spec/webapp/jobs.py @@ -0,0 +1,199 @@ +"""In-memory job registry with SSE event fan-out. + +Runs live in process memory: events are kept for replay (reconnects with +Last-Event-ID) and fanned out to any number of SSE subscribers. Credentials +inside the original request are never serialized into events or results. +""" + +import asyncio +import logging +import uuid +from datetime import datetime +from typing import AsyncGenerator, Dict, List, Optional + +from spec.models.webapp import AgentEvent, RunRequest, RunStatus + +logger = logging.getLogger(__name__) + +_TERMINAL: frozenset = frozenset({"done", "error", "cancelled"}) + + +class Job: + """A single extraction run and its event history. + + Attributes: + id: Job identifier + request: Original run request (kept in memory only) + status: Current run status + events: Emitted events, in order + result: Final result payload when finished + error: Error message when failed + task: Asyncio task executing the orchestration + """ + + def __init__(self, request: RunRequest) -> None: + """Initialize a queued job for the given request.""" + + self.id: str = f"genie-{uuid.uuid4().hex[:10]}" + self.request = request + self.status: RunStatus = "queued" + self.events: List[AgentEvent] = [] + self.result: Optional[dict] = None + self.error: Optional[str] = None + self.task: Optional[asyncio.Task] = None + self._subscribers: List[asyncio.Queue] = [] + self._seq = 0 + + @property + def is_finished(self) -> bool: + """Whether the job reached a terminal status.""" + + return self.status in _TERMINAL + + +class JobManager: + """Registry and event bus for extraction runs.""" + + def __init__(self, max_jobs: int = 200) -> None: + """Initialize the manager. + + Args: + max_jobs: Maximum retained jobs before oldest are evicted + """ + + self._jobs: Dict[str, Job] = {} + self._max_jobs = max_jobs + + def create(self, request: RunRequest) -> Job: + """Register a new job. + + Args: + request: Validated run request + + Returns: + Job: Newly created job (status "queued") + """ + + job = Job(request) + self._jobs[job.id] = job + + while len(self._jobs) > self._max_jobs: + oldest_id = next(iter(self._jobs)) + evicted = self._jobs.pop(oldest_id) + if evicted.task and not evicted.task.done(): + evicted.task.cancel() + + return job + + def get(self, job_id: str) -> Optional[Job]: + """Fetch a job by id. + + Args: + job_id: Job identifier + + Returns: + Optional[Job]: Job or None + """ + + return self._jobs.get(job_id) + + def emit( + self, + job: Job, + agent: str, + type: str, + message: str = "", + level: str = "", + progress: Optional[int] = None, + status: Optional[RunStatus] = None, + result: Optional[dict] = None, + ) -> AgentEvent: + """Append an event to a job and fan it out to subscribers. + + Args: + job: Target job + agent: Emitting agent name + type: Event type + message: Human-readable message + level: UI log level hint + progress: Agent progress 0-100 + status: Run status for finish/error events + result: Final result payload for finish events + + Returns: + AgentEvent: The emitted event + """ + + job._seq += 1 + event = AgentEvent( + seq=job._seq, + ts=datetime.now().strftime("%H:%M:%S"), + agent=agent, # type: ignore[arg-type] + type=type, # type: ignore[arg-type] + level=level, + message=message, + progress=progress, + status=status, + result=result, + ) + job.events.append(event) + + for queue in list(job._subscribers): + try: + queue.put_nowait(event) + except asyncio.QueueFull: # pragma: no cover - unbounded queues + logger.warning("Dropping event for slow subscriber on job %s", job.id) + + return event + + async def stream( + self, + job: Job, + last_event_id: Optional[int] = None, + ) -> AsyncGenerator[AgentEvent, None]: + """Yield job events, replaying history then following live updates. + + Args: + job: Job to follow + last_event_id: Replay only events with seq greater than this + + Yields: + AgentEvent: Each event in order + """ + + queue: asyncio.Queue = asyncio.Queue() + job._subscribers.append(queue) + + try: + replay_from = last_event_id or 0 + for event in list(job.events): + if event.seq > replay_from: + yield event + + if job.is_finished: + return + + while True: + event = await queue.get() + yield event + if event.type in ("finish", "error") and event.status in _TERMINAL: + return + finally: + if queue in job._subscribers: + job._subscribers.remove(queue) + + +_manager: Optional[JobManager] = None + + +def get_job_manager() -> JobManager: + """Get the global JobManager singleton. + + Returns: + JobManager: Process-wide job registry + """ + + global _manager + if _manager is None: + _manager = JobManager() + return _manager diff --git a/tests/integration/test_health.py b/tests/integration/test_health.py index 68afe52..42359b8 100644 --- a/tests/integration/test_health.py +++ b/tests/integration/test_health.py @@ -14,15 +14,13 @@ async def client(): yield c -async def test_root_endpoint(client: httpx.AsyncClient): - """Test root endpoint.""" +async def test_root_serves_spa(client: httpx.AsyncClient): + """Root endpoint serves the GenIE single-page application.""" response = await client.get("/") assert response.status_code == 200 - data = response.json() - assert "message" in data - assert "version" in data - assert data["version"] == "0.1.0" + assert "text/html" in response.headers["content-type"] + assert "GenIE" in response.text async def test_health_endpoint(client: httpx.AsyncClient): diff --git a/tests/integration/test_webapp_api.py b/tests/integration/test_webapp_api.py new file mode 100644 index 0000000..b41500d --- /dev/null +++ b/tests/integration/test_webapp_api.py @@ -0,0 +1,240 @@ +"""End-to-end tests for the web app API: models, keys, uploads, runs, downloads.""" + +import asyncio +import json +from typing import Any, Dict + +import httpx +import pytest + +from spec.core.config import get_settings +from spec.core.security import get_key_vault, reset_security_singletons +from spec.extraction.agents import orchestrator as orchestrator_module +from spec.main import app + + +@pytest.fixture +def isolated_storage(tmp_path, monkeypatch): + """Point data dirs at a temp folder and reset security singletons.""" + + settings = get_settings() + monkeypatch.setattr(settings, "data_dir", str(tmp_path)) + monkeypatch.setattr(settings, "db_path", str(tmp_path / "genie.db")) + monkeypatch.setattr(settings, "uploads_dir", str(tmp_path / "uploads")) + monkeypatch.setattr(settings, "outputs_dir", str(tmp_path / "outputs")) + monkeypatch.setattr(settings, "allowed_fs_roots", str(tmp_path)) + monkeypatch.setattr(settings, "master_key", None) + reset_security_singletons() + yield settings + reset_security_singletons() + + +@pytest.fixture +async def client(isolated_storage): + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as c: + yield c + + +class FakeProvider: + """Deterministic stand-in for an LLM provider.""" + + def __init__(self, records=None) -> None: + self.records = records or [ + {"Data": "2026-03-14", "Exame": "Glicose", "Resultado": "118 mg/dL"} + ] + + async def extract(self, content: str, schema: Dict[str, Any], instructions: str = ""): + return {"records": self.records, "confidence": 0.94, "notes": ""} + + +class TestModels: + async def test_catalog_without_keys(self, client): + response = await client.get("/api/v1/models") + + assert response.status_code == 200 + models = response.json() + assert len(models) >= 4 + assert all(m["has_key"] is False for m in models) + assert all("key" not in (m.get("masked_key") or "") for m in models) + + +class TestKeys: + async def test_store_list_delete(self, client): + stored = await client.post( + "/api/v1/keys", + json={"provider": "google", "key": "AIzaSyFakeKey1234", "validate_key": False}, + ) + assert stored.status_code == 200 + body = stored.json() + assert body["masked"].startswith("AIza") + assert "AIzaSyFakeKey1234" not in json.dumps(body) + + listed = await client.get("/api/v1/keys") + assert listed.status_code == 200 + assert listed.json() == [{"provider": "google", "masked": body["masked"]}] + + models = (await client.get("/api/v1/models")).json() + google = [m for m in models if m["provider"] == "google"] + assert all(m["has_key"] for m in google) + + deleted = await client.delete("/api/v1/keys/google") + assert deleted.status_code == 200 + assert (await client.delete("/api/v1/keys/google")).status_code == 404 + + async def test_unknown_provider_rejected(self, client): + response = await client.post( + "/api/v1/keys", + json={"provider": "skynet", "key": "x", "validate_key": False}, + ) + + assert response.status_code == 400 + + +class TestUploads: + async def test_accepts_supported_files(self, client): + response = await client.post( + "/api/v1/uploads", + files=[ + ("files", ("exame.txt", b"Glicose: 118 mg/dL", "text/plain")), + ("files", ("dados.csv", b"a,b\n1,2", "text/csv")), + ], + ) + + assert response.status_code == 200 + body = response.json() + assert len(body["upload_id"]) == 32 + assert {f["name"] for f in body["files"]} == {"exame.txt", "dados.csv"} + + async def test_rejects_unsupported_extension(self, client): + response = await client.post( + "/api/v1/uploads", + files=[("files", ("virus.exe", b"MZ", "application/octet-stream"))], + ) + + assert response.status_code == 400 + + async def test_sanitizes_traversal_names(self, client, isolated_storage): + response = await client.post( + "/api/v1/uploads", + files=[("files", ("../../evil.txt", b"data", "text/plain"))], + ) + + assert response.status_code == 200 + name = response.json()["files"][0]["name"] + assert "/" not in name and ".." not in name + + +class TestRuns: + async def test_unknown_model_rejected(self, client): + response = await client.post( + "/api/v1/runs", + json={ + "model_id": "modelo-x", + "input": {"type": "url", "target": "https://x"}, + "prompt": "extraia", + "output": {"type": "download"}, + }, + ) + + assert response.status_code == 400 + + async def test_missing_key_rejected(self, client): + response = await client.post( + "/api/v1/runs", + json={ + "model_id": "gemini-2.5-flash", + "input": {"type": "url", "target": "https://x"}, + "prompt": "extraia", + "output": {"type": "download"}, + }, + ) + + assert response.status_code == 400 + assert "API Key" in response.json()["detail"] + + async def test_full_pipeline_upload_to_download(self, client, monkeypatch): + get_key_vault().store("google", "AIzaSyFakeKey1234") + monkeypatch.setattr( + orchestrator_module.Orchestrator, + "_resolve_provider", + lambda self, model_id: FakeProvider(), + ) + + upload = await client.post( + "/api/v1/uploads", + files=[("files", ("exames.txt", b"Glicose: 118 mg/dL (ref 70-99)", "text/plain"))], + ) + upload_id = upload.json()["upload_id"] + + created = await client.post( + "/api/v1/runs", + json={ + "model_id": "gemini-2.5-flash", + "input": {"type": "upload", "upload_id": upload_id}, + "prompt": "Extraia exame, resultado e referência", + "output": {"type": "download"}, + "format": "", + }, + ) + assert created.status_code == 201 + job_id = created.json()["job_id"] + + for _ in range(50): + info = (await client.get(f"/api/v1/runs/{job_id}")).json() + if info["status"] in ("done", "error", "cancelled"): + break + await asyncio.sleep(0.1) + + assert info["status"] == "done", info.get("error") + result = info["result"] + assert result["records"] == FakeProvider().records + assert set(result["downloads"]) == {"json", "csv"} + + # SSE replay terminates for finished jobs and carries the finish event. + events = await client.get(f"/api/v1/runs/{job_id}/events") + assert events.status_code == 200 + assert '"type": "finish"' in events.text or '"type":"finish"' in events.text + + # Signed download link works... + download = await client.get(result["downloads"]["json"]) + assert download.status_code == 200 + assert download.json() == FakeProvider().records + + # ...and a tampered signature is refused. + tampered = result["downloads"]["json"].replace("sig=", "sig=ff") + assert (await client.get(tampered)).status_code == 403 + + async def test_credentials_never_leak_into_events(self, client, monkeypatch): + get_key_vault().store("google", "AIzaSyFakeKey1234") + monkeypatch.setattr( + orchestrator_module.Orchestrator, + "_resolve_provider", + lambda self, model_id: FakeProvider(), + ) + + created = await client.post( + "/api/v1/runs", + json={ + "model_id": "gemini-2.5-flash", + "input": { + "type": "db", + "target": "postgres://user:SENHA_SECRETA@host:5432/db", + "password": "SENHA_SECRETA", + }, + "prompt": "extraia tudo", + "output": {"type": "download"}, + }, + ) + job_id = created.json()["job_id"] + + for _ in range(50): + info = (await client.get(f"/api/v1/runs/{job_id}")).json() + if info["status"] in ("done", "error", "cancelled"): + break + await asyncio.sleep(0.1) + + events = await client.get(f"/api/v1/runs/{job_id}/events") + assert "SENHA_SECRETA" not in events.text + assert "SENHA_SECRETA" not in json.dumps(info) diff --git a/tests/unit/test_connector.py b/tests/unit/test_connector.py new file mode 100644 index 0000000..ab3aae7 --- /dev/null +++ b/tests/unit/test_connector.py @@ -0,0 +1,150 @@ +"""Tests for the Connector agent (filesystem safety, content parsing, DB output).""" + +import json +import sqlite3 +from pathlib import Path + +import pytest + +from spec.core.exceptions import ExtractionFailed, InvalidConfig +from spec.extraction.agents.connector import ( + ConnectorAgent, + ensure_path_allowed, + new_upload_id, +) +from spec.extraction.parsers.content import bytes_to_text, html_to_text +from spec.models.webapp import InputSpec, OutputSpec + + +def noop_emit(message: str = "", level: str = "", progress=None) -> None: + """No-op event sink for tests.""" + + +class TestPathSafety: + def test_blocks_system_paths(self) -> None: + with pytest.raises(InvalidConfig): + ensure_path_allowed(Path("/etc/shadow")) + + def test_blocks_traversal(self, tmp_path) -> None: + with pytest.raises(InvalidConfig): + ensure_path_allowed(Path("/usr/../etc/passwd")) + + def test_allows_home(self) -> None: + resolved = ensure_path_allowed(Path.home() / "docs") + + assert resolved == (Path.home() / "docs").resolve() + + +class TestContentParsing: + def test_csv(self) -> None: + text = bytes_to_text(b"a,b\n1,2\n", "data.csv") + + assert "a\tb" in text + assert "1\t2" in text + + def test_json_pretty(self) -> None: + text = bytes_to_text(b'{"x":1}', "data.json") + + assert '"x": 1' in text + + def test_html_strips_tags_and_scripts(self) -> None: + html = "

Título

texto

" + text = html_to_text(html) + + assert "Título" in text + assert "texto" in text + assert "evil" not in text + + def test_docx_rejected_with_guidance(self) -> None: + with pytest.raises(ExtractionFailed, match="docx"): + bytes_to_text(b"PK...", "doc.docx") + + +class TestUploadInput: + @pytest.mark.asyncio + async def test_missing_upload_id_rejected(self) -> None: + connector = ConnectorAgent() + spec = InputSpec(type="upload", upload_id=None) + + with pytest.raises(InvalidConfig): + await connector.open_input(spec, noop_emit) + + @pytest.mark.asyncio + async def test_traversal_upload_id_rejected(self) -> None: + connector = ConnectorAgent() + spec = InputSpec(type="upload", upload_id="../../etc") + + with pytest.raises(InvalidConfig): + await connector.open_input(spec, noop_emit) + + def test_new_upload_id_format(self) -> None: + upload_id = new_upload_id() + + assert len(upload_id) == 32 + assert all(c in "0123456789abcdef" for c in upload_id) + + +class TestPathInput: + @pytest.mark.asyncio + async def test_reads_folder(self, tmp_path, monkeypatch) -> None: + from spec.core.config import get_settings + + monkeypatch.setattr(get_settings(), "allowed_fs_roots", str(tmp_path)) + (tmp_path / "um.txt").write_text("conteúdo um", encoding="utf-8") + (tmp_path / "dois.csv").write_text("a,b\n1,2", encoding="utf-8") + (tmp_path / "ignorado.bin").write_bytes(b"\x00\x01") + + connector = ConnectorAgent() + items = await connector.open_input( + InputSpec(type="path", target=str(tmp_path)), noop_emit + ) + + names = {item["name"] for item in items} + assert names == {"um.txt", "dois.csv"} + + +class TestDbDelivery: + @pytest.mark.asyncio + async def test_writes_sqlite(self, tmp_path, monkeypatch) -> None: + from spec.core.config import get_settings + + monkeypatch.setattr(get_settings(), "allowed_fs_roots", str(tmp_path)) + db_file = tmp_path / "saida.db" + records = [ + {"exame": "Glicose", "resultado": "118 mg/dL"}, + {"exame": "TSH", "resultado": "2.1 µUI/mL", "extra": {"obs": "ok"}}, + ] + + connector = ConnectorAgent() + receipt = await connector.deliver( + OutputSpec(type="db", target=f"sqlite:///{db_file}", table="exames"), + records, + {}, + "genie-test", + noop_emit, + ) + + assert receipt["rows"] == 2 + conn = sqlite3.connect(db_file) + rows = conn.execute("SELECT exame, resultado, extra FROM exames ORDER BY exame").fetchall() + conn.close() + assert rows[0][0] == "Glicose" + assert json.loads(rows[1][2]) == {"obs": "ok"} + + +class TestDownloadDelivery: + @pytest.mark.asyncio + async def test_writes_json_and_csv(self, tmp_path, monkeypatch) -> None: + from spec.core.config import get_settings + + monkeypatch.setattr(get_settings(), "outputs_dir", str(tmp_path)) + records = [{"campo": "valor", "n": 1}] + + connector = ConnectorAgent() + receipt = await connector.deliver( + OutputSpec(type="download"), records, {}, "genie-abc", noop_emit + ) + + assert sorted(receipt["artifacts"]) == ["output.csv", "output.json"] + saved = json.loads((tmp_path / "genie-abc" / "output.json").read_text()) + assert saved == records diff --git a/tests/unit/test_security.py b/tests/unit/test_security.py new file mode 100644 index 0000000..910dee3 --- /dev/null +++ b/tests/unit/test_security.py @@ -0,0 +1,103 @@ +"""Tests for AES-256-GCM encryption and the encrypted key vault.""" + +import os + +import pytest + +from spec.core.exceptions import InvalidConfig, StorageError +from spec.core.security import KeyVault, SecretCipher, mask_secret + + +@pytest.fixture +def cipher() -> SecretCipher: + return SecretCipher(os.urandom(32)) + + +def test_encrypt_decrypt_roundtrip(cipher: SecretCipher) -> None: + secret = "AIzaSyA-example-key-1234567890" + blob = cipher.encrypt(secret) + + assert secret.encode() not in blob + assert cipher.decrypt(blob) == secret + + +def test_encrypt_is_non_deterministic(cipher: SecretCipher) -> None: + blob1 = cipher.encrypt("same-secret") + blob2 = cipher.encrypt("same-secret") + + assert blob1 != blob2 + + +def test_tampered_ciphertext_fails(cipher: SecretCipher) -> None: + blob = bytearray(cipher.encrypt("secret")) + blob[-1] ^= 0xFF + + with pytest.raises(StorageError): + cipher.decrypt(bytes(blob)) + + +def test_wrong_key_fails(cipher: SecretCipher) -> None: + other = SecretCipher(os.urandom(32)) + blob = cipher.encrypt("secret") + + with pytest.raises(StorageError): + other.decrypt(blob) + + +def test_master_key_must_be_32_bytes() -> None: + with pytest.raises(InvalidConfig): + SecretCipher(b"short") + + +def test_sign_and_verify(cipher: SecretCipher) -> None: + signature = cipher.sign("job-1:output.json:123") + + assert cipher.verify("job-1:output.json:123", signature) + assert not cipher.verify("job-1:output.json:124", signature) + assert not cipher.verify("job-1:output.json:123", signature[:-2] + "ff") + + +def test_mask_secret_hides_content() -> None: + masked = mask_secret("sk-ant-veryverysecret") + + assert masked.startswith("sk-a") + assert "secret" not in masked + assert mask_secret("short") == "••••••••" + + +class TestKeyVault: + @pytest.fixture + def vault(self, tmp_path) -> KeyVault: + return KeyVault(str(tmp_path / "vault.db"), SecretCipher(os.urandom(32))) + + def test_store_and_retrieve(self, vault: KeyVault) -> None: + masked = vault.store("google", "AIzaSyExample123456") + + assert masked.startswith("AIza") + assert vault.has("google") + assert vault.masked("google") == masked + assert vault.get_plaintext("google") == "AIzaSyExample123456" + + def test_plaintext_never_on_disk(self, vault: KeyVault, tmp_path) -> None: + vault.store("openai", "sk-supersecret-key-material") + + raw = (tmp_path / "vault.db").read_bytes() + assert b"sk-supersecret-key-material" not in raw + + def test_overwrite_key(self, vault: KeyVault) -> None: + vault.store("google", "first-key-value") + vault.store("google", "second-key-value") + + assert vault.get_plaintext("google") == "second-key-value" + + def test_delete(self, vault: KeyVault) -> None: + vault.store("anthropic", "sk-ant-example") + + assert vault.delete("anthropic") + assert not vault.has("anthropic") + assert vault.get_plaintext("anthropic") is None + assert not vault.delete("anthropic") + + def test_empty_key_rejected(self, vault: KeyVault) -> None: + with pytest.raises(InvalidConfig): + vault.store("google", " ") From 5c11a8f7db55e0059a5fba1e2ed5ed3c7f825c72 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 13:02:25 +0000 Subject: [PATCH 2/3] fix: SSE event dedup, native JSON mode, inline text input; add usage manual MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes and improvements before the PR: Fixes: - JobManager.stream no longer yields duplicate events when emissions race with history replay (tracks last yielded seq); covered by test - Job eviction prefers finished jobs over running ones - Frontend stops SSE reconnection after repeated failures (server restart) instead of retrying forever - pyproject ruff config migrated to [tool.ruff.lint] (deprecation) - health endpoint cleanup (timezone-aware timestamp, version 1.0.0) Reliability improvements: - Native JSON mode on providers: response_format json_object (OpenAI) and response_mime_type application/json (Gemini) - Larger output budgets: Gemini 16384 (thinking tokens), OpenAI/Claude 8192 - Updated provider default models (gemini-2.5-flash, claude-sonnet-4-6) - CSV artifacts written as UTF-8 with BOM so Excel pt-BR renders accents - Stale upload/output batches cleaned up on startup (24h TTL) New: inline 'text' input type — integrating apps (TabEx) send pre-extracted OCR text in a single POST /runs call, no multipart upload needed. Docs: docs/MANUAL.md — full usage manual covering standalone mode (web UI walkthrough), plugin mode (REST contract with curl/Python/JS examples) and the TabEx integration: replaces TabEx's layout-specific regex extraction (extrairResultados) with GenIE via Apps Script polling, enabling layout-independent extraction and automatic new-exam columns. 79 tests passing. https://claude.ai/code/session_01CKjevqGgfWLggV1DG1Tmpq --- README.md | 2 + docs/MANUAL.md | 435 +++++++++++++++++++++++++ pyproject.toml | 4 +- spec/__init__.py | 2 +- spec/api/v1/dependencies.py | 4 +- spec/api/v1/endpoints/health.py | 27 +- spec/api/v1/endpoints/runs.py | 9 +- spec/extraction/agents/connector.py | 17 +- spec/extraction/agents/orchestrator.py | 3 + spec/extraction/engine.py | 8 +- spec/extraction/llm/anthropic.py | 4 +- spec/extraction/llm/factory.py | 8 +- spec/extraction/llm/google.py | 7 +- spec/extraction/llm/openai.py | 3 +- spec/extraction/parsers/pdf.py | 10 +- spec/main.py | 26 +- spec/models/config.py | 11 +- spec/models/extraction.py | 3 +- spec/models/library.py | 5 +- spec/models/output.py | 15 +- spec/models/webapp.py | 9 +- spec/search_library/json_storage.py | 14 +- spec/search_library/matcher.py | 4 +- spec/web/app.js | 14 +- spec/webapp/jobs.py | 18 +- tests/integration/test_webapp_api.py | 84 +++++ 26 files changed, 675 insertions(+), 71 deletions(-) create mode 100644 docs/MANUAL.md diff --git a/README.md b/README.md index 254fd9f..710dcf5 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ curl -s -X POST localhost:8000/api/v1/runs -H 'Content-Type: application/json' - | Banco de dados | SQLite nativo; Postgres/MySQL via SQLAlchemy opcional | SQLite (cria/evolui tabela) | | API REST | GET com Bearer token | POST com Bearer (lote ou por registro) | | Upload / Download | multipart seguro | links assinados (15 min) | +| Texto inline (`text`) | conteúdo enviado no próprio POST — ideal para integração plugin | — | ## Integração TabEx @@ -137,6 +138,7 @@ pytest --cov=spec # cobertura ## Documentação +- **[Manual de uso](./docs/MANUAL.md)** — interface web, API REST (plugin) e integração TabEx - [Arquitetura](./docs/guides/GENIE-ARCHITECTURE.md) - [Especificação v2](./docs/guides/GENIE-SPEC-v2.md) - [Exemplos](./docs/examples/GENIE-EXAMPLES.md) diff --git a/docs/MANUAL.md b/docs/MANUAL.md new file mode 100644 index 0000000..c71aac8 --- /dev/null +++ b/docs/MANUAL.md @@ -0,0 +1,435 @@ +# Manual de Uso — GenIE + +**GenIE** (Generic Extractor of Information Engine) extrai informação estruturada +de fontes heterogêneas — PDFs, planilhas, páginas web, bancos de dados, APIs — +usando LLMs, e entrega o resultado no formato e destino que você definir. + +Ele opera de duas formas: + +| Modo | Para quem | Como | +|---|---|---| +| **Independente** | Pessoas | Interface web em `http://localhost:8000` | +| **Extensão/Plugin** | Outras aplicações (TabEx, scripts, sistemas) | API REST (`/api/v1/...`) | + +Nos dois modos o trabalho é feito pelos mesmos três agentes, em sequência: + +``` +┌──────────┐ ┌─────────────┐ ┌─────────────┐ ┌──────────┐ +│ CONECTOR │ → │ LOCALIZADOR │ → │ ORGANIZADOR │ → │ CONECTOR │ +│ abre a │ │ extrai o que│ │ formata no │ │ entrega │ +│ fonte │ │ foi pedido │ │ padrão dado │ │ no destino│ +└──────────┘ └─────────────┘ └─────────────┘ └──────────┘ + (I/O) (LLM) (LLM) (I/O) +``` + +- **Conector** — toda a entrada e saída. Nunca chama LLM. +- **Localizador** — lê cada documento e extrai os campos pedidos via LLM. +- **Organizador** — reformata os registros conforme sua instrução. Se você não + informar formato, os dados passam direto (custo zero de LLM). + +--- + +## 1. Instalação + +Requisitos: **Python 3.11+**. + +```bash +git clone https://github.com/RMSantista/GenIE.git +cd GenIE +pip install -r requirements.txt + +# opcional, recomendado em produção: +cp .env.example .env +# gere a chave-mestre de criptografia: +# openssl rand -base64 32 → cole em GENIE_MASTER_KEY no .env +# (sem isso, o GenIE gera uma automaticamente em ./data/.master_key) + +uvicorn spec.main:app --port 8000 +``` + +Abra **http://localhost:8000**. A documentação interativa da API fica em +**http://localhost:8000/docs**. + +--- + +## 2. Modo independente (interface web) + +A tela é dividida em **Configuração** (esquerda) e **Monitor de agentes** (direita). + +### 2.1 Seção 01 — Modelo de IA + +Escolha o modelo (Gemini, GPT ou Claude) e clique em **Nova API Key**. + +- A chave é enviada **uma única vez** ao servidor, validada com uma chamada + mínima ao provedor e gravada **cifrada (AES-256-GCM)**. Ela nunca volta ao + navegador — você verá apenas os 4 primeiros caracteres mascarados. +- A chave é **por provedor**: cadastrar a chave do Google libera todos os + modelos Gemini. +- Dica de custo: comece com `Gemini 2.5 Flash` ou `GPT-4o mini` (rápidos e + baratos). Use `Claude Sonnet 4.6` ou `Gemini 2.5 Pro` para documentos + difíceis. + +### 2.2 Seção 02 — Origem dos dados (entrada) + +| Tipo | O que informar | Exemplo | +|---|---|---| +| **URL** | Endereço http(s) de página, PDF ou JSON. Links de *arquivo* do Google Drive são convertidos para download direto | `https://lab.com/laudo.pdf` | +| **Pasta local** | Caminho de pasta ou arquivo no servidor onde o GenIE roda (varre subpastas) | `/home/voce/Documents/exames` | +| **Banco de dados** | URL do banco + usuário/senha. SQLite funciona nativamente; Postgres/MySQL exigem `pip install sqlalchemy` + driver | `sqlite:///dados.db` | +| **API REST** | Endpoint GET + Bearer token (opcional) | `https://api.servico.com/v1/exames` | +| **Upload** | Arraste os arquivos para a área indicada | `.pdf .csv .xlsx .json .txt .html` | + +Formatos de arquivo aceitos: PDF (com texto), CSV/TSV, XLSX, JSON, TXT, MD, +HTML, XML, YAML, LOG. + +### 2.3 Seção 03 — O que extrair + +Instrução em linguagem natural para o Localizador. Seja específico sobre os +**campos** e seus **nomes**: + +> Extraia, para cada exame encontrado, os campos: **data** (YYYY-MM-DD), +> **exame** (nome padronizado), **resultado** (valor com unidade) e +> **referencia** (faixa de referência). Ignore cabeçalhos, rodapés e dados de +> contato do laboratório. + +O botão **Exemplo** preenche o formulário com um caso realista. + +### 2.4 Seção 04 — Destino dos dados (saída) + +| Tipo | O que acontece | +|---|---| +| **URL (webhook)** | POST com o JSON dos registros | +| **Pasta local** | Grava `output.json` + `output.csv` na pasta indicada | +| **Banco de dados** | Insere em tabela SQLite (cria a tabela e novas colunas automaticamente) | +| **API REST** | POST autenticado por Bearer token (em lote ou 1 chamada por registro) | +| **Download** | Gera links assinados de `JSON` e `CSV` válidos por 15 minutos | + +### 2.5 Seção 05 — Formato da saída + +Instrução em linguagem natural para o Organizador. Exemplos: + +- `Envie 1 chamada POST por exame, com o body { "data": "YYYY-MM-DD", "exame": "...", "resultado": "..." }` +- `Agrupe os exames por data e gere um objeto por dia` +- *Vazio* → os registros extraídos são entregues como estão (sem custo extra de LLM). + +### 2.6 Executando + +Clique em **Enviar requisição**. No monitor você acompanha: + +- os **3 cards de agentes** com progresso e status; +- o **log de execução** em tempo real (streaming); +- ao final, a **Saída entregue** com visualização em Tabela/JSON, botão de + copiar e botões de download (quando a saída é Download). + +**Interromper** cancela a execução no servidor (inclusive chamadas de LLM em +andamento). **Limpar** reinicia o monitor. + +--- + +## 3. Modo extensão/plugin (API REST) + +Qualquer aplicação pode usar o GenIE como serviço de extração. O contrato é: + +``` +1. (uma vez) POST /api/v1/keys → cadastra a chave do provedor LLM +2. (opcional) POST /api/v1/uploads → envia arquivos, recebe upload_id +3. POST /api/v1/runs → cria a execução, recebe job_id +4a. GET /api/v1/runs/{id}/events → acompanha por SSE (tempo real) +4b. GET /api/v1/runs/{id} → ou consulta por polling +5. resultado em result.records (+ links de download assinados) +``` + +### 3.1 Entrada `text` — a integração mais simples + +Se a sua aplicação **já possui o texto** (ex.: OCR feito por ela), não é +preciso upload: envie o conteúdo inline em um único POST. + +```bash +curl -s -X POST http://localhost:8000/api/v1/runs \ + -H 'Content-Type: application/json' \ + -d '{ + "model_id": "gemini-2.5-flash", + "input": { "type": "text", "content": "SODIO: 140 mEq/L\nCREATININA: 0,9 mg/dL", "name": "laudo.txt" }, + "prompt": "Extraia exame (minúsculas, sem acento) e resultado (número) de cada análise", + "output": { "type": "download" } + }' +# → {"job_id":"genie-ab12cd34ef","status":"queued"} + +curl -s http://localhost:8000/api/v1/runs/genie-ab12cd34ef +# → {"status":"done","result":{"records":[{"exame":"sodio","resultado":140}, ...]}} +``` + +### 3.2 Exemplo em Python + +```python +import httpx, time + +GENIE = "http://localhost:8000/api/v1" + +def extrair(texto: str, prompt: str) -> list[dict]: + job = httpx.post(f"{GENIE}/runs", json={ + "model_id": "gemini-2.5-flash", + "input": {"type": "text", "content": texto}, + "prompt": prompt, + "output": {"type": "download"}, + }).json() + + while True: + info = httpx.get(f"{GENIE}/runs/{job['job_id']}").json() + if info["status"] in ("done", "error", "cancelled"): + break + time.sleep(1) + + if info["status"] != "done": + raise RuntimeError(info["error"]) + return info["result"]["records"] +``` + +### 3.3 Exemplo em JavaScript (Node/browser) com SSE + +```javascript +const GENIE = "http://localhost:8000/api/v1"; + +async function extrair(texto, prompt) { + const { job_id } = await fetch(`${GENIE}/runs`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model_id: "gemini-2.5-flash", + input: { type: "text", content: texto }, + prompt, + output: { type: "download" }, + }), + }).then((r) => r.json()); + + return new Promise((resolve, reject) => { + const es = new EventSource(`${GENIE}/runs/${job_id}/events`); + es.onmessage = (m) => { + const ev = JSON.parse(m.data); + console.log(`[${ev.agent}] ${ev.message || ""}`); // log em tempo real + if (ev.type === "finish") { es.close(); resolve(ev.result.records); } + if (ev.type === "error") { es.close(); reject(new Error(ev.message)); } + }; + }); +} +``` + +### 3.4 Entrega direta na sua aplicação + +Em vez de buscar o resultado, a sua aplicação pode **recebê-lo**: configure a +saída como `api` (POST com Bearer token) ou `url` (webhook sem autenticação): + +```json +"output": { "type": "api", "target": "https://sua-app.com/v1/importar", "token": "SEU_TOKEN" }, +"format": "Envie 1 POST por registro com o body { \"campo\": ... } esperado pela minha API" +``` + +O Organizador adapta os payloads à sua descrição e o Conector faz as chamadas. + +### 3.5 Upload de arquivos via API + +```bash +UP=$(curl -s -F "files=@laudo.pdf" -F "files=@resultados.csv" \ + http://localhost:8000/api/v1/uploads | python3 -c "import json,sys;print(json.load(sys.stdin)['upload_id'])") + +curl -s -X POST http://localhost:8000/api/v1/runs -H 'Content-Type: application/json' \ + -d "{\"model_id\":\"gemini-2.5-flash\", + \"input\":{\"type\":\"upload\",\"upload_id\":\"$UP\"}, + \"prompt\":\"Extraia ...\", + \"output\":{\"type\":\"download\"}}" +``` + +### 3.6 Referência rápida de endpoints + +| Método | Rota | Descrição | +|---|---|---| +| `GET` | `/api/v1/models` | Modelos disponíveis + `has_key` | +| `POST` | `/api/v1/keys` | `{provider, key, validate_key}` → cifra e guarda | +| `DELETE` | `/api/v1/keys/{provider}` | Remove a chave | +| `POST` | `/api/v1/uploads` | multipart → `{upload_id, files}` | +| `POST` | `/api/v1/runs` | Cria execução → `{job_id}` | +| `GET` | `/api/v1/runs/{id}` | Estado + resultado final | +| `GET` | `/api/v1/runs/{id}/events` | SSE (suporta `Last-Event-ID` para reconexão) | +| `POST` | `/api/v1/runs/{id}/cancel` | Interrompe a execução | +| `GET` | `/api/v1/downloads/{id}/{arquivo}?exp=&sig=` | Artefatos (link assinado) | + +Esquema completo (request/response) em `/docs` (OpenAPI). + +--- + +## 4. Integração com o TabEx + +### 4.1 O que o TabEx faz hoje + +O [TabEx](https://github.com/RMSantista/TabEx) (Google Apps Script) automatiza +a tabulação de exames de sangue do SUS de Ribeirão Preto: + +1. Um gatilho roda `processarNovosExames()` a cada 5 minutos; +2. `extrairTextoPDF()` faz OCR dos PDFs novos da pasta do Drive (Drive API); +3. `extrairData()` e `extrairResultados()` aplicam **regex fixas** para achar a + data de coleta e **8 exames pré-definidos** (sódio, potássio, cálcio, + magnésio, fósforo, ureia, creatinina, TFG); +4. `atualizarPlanilha()` grava na Google Sheets conforme o mapa `COLUNAS`; +5. O PDF é arquivado em subpastas por data. + +**Onde ele quebra:** o passo 3. As regex assumem um único layout de laudo +(SUS-RP). Laudos de outros laboratórios, variações de formato ou OCR imperfeito +fazem a extração falhar — e cada exame novo exige programar mais regex. + +### 4.2 Onde o GenIE entra + +O GenIE substitui exatamente o passo 3 (a "inteligência"), mantendo o que o +TabEx já faz bem (monitorar o Drive, OCR nativo do Google e escrever na +planilha): + +``` +TabEx (Apps Script) GenIE (servidor) +─────────────────── ──────────────── +1. detecta PDF novo no Drive +2. OCR via Drive API ──── texto ────────▶ 3. POST /api/v1/runs + input: { type: "text", content: } + prompt: "extraia data, exame, resultado…" +4. polling GET /runs/{id} ◀── records ─── Localizador extrai de QUALQUER layout +5. atualizarPlanilha(records) Organizador padroniza nomes/números +6. arquiva o PDF +``` + +Ganhos imediatos: + +- **Independência de layout** — laudos de qualquer laboratório/formato; +- **Novos exames sem código** — o Localizador devolve *todos* os exames do + laudo; o TabEx pode criar colunas novas dinamicamente em vez de limitar-se + aos 8 fixos; +- **Resiliência a OCR imperfeito** — o LLM entende `S0DIO`, `Sódio:`, quebras + de linha etc., onde a regex falha. + +### 4.3 Código de integração (Apps Script) + +Substitua a chamada a `extrairResultados(texto)` por: + +```javascript +// URL pública do seu servidor GenIE (ver requisito no item 4.4) +const GENIE_URL = 'https://seu-genie.exemplo.com'; + +function extrairComGenIE(textoOcr) { + const criacao = UrlFetchApp.fetch(GENIE_URL + '/api/v1/runs', { + method: 'post', + contentType: 'application/json', + payload: JSON.stringify({ + model_id: 'gemini-2.5-flash', + input: { type: 'text', content: textoOcr, name: 'laudo-sus.txt' }, + prompt: + 'Este é o texto OCR de um laudo de exames de sangue do SUS. ' + + 'Extraia, para cada análise presente, os campos: ' + + 'data_coleta (YYYY-MM-DD), exame (nome em minúsculas sem acento, ex.: ' + + 'sodio, potassio, calcio, magnesio, fosforo, ureia, creatinina, tfg) ' + + 'e resultado (apenas o número). Inclua também exames fora dessa lista.', + output: { type: 'download' } + }) + }); + const jobId = JSON.parse(criacao.getContentText()).job_id; + + // Apps Script não suporta SSE — usar polling: + for (let i = 0; i < 30; i++) { + Utilities.sleep(2000); + const info = JSON.parse( + UrlFetchApp.fetch(GENIE_URL + '/api/v1/runs/' + jobId).getContentText() + ); + if (info.status === 'done') return info.result.records; + if (info.status === 'error') throw new Error('GenIE: ' + info.error); + } + throw new Error('GenIE: tempo esgotado'); +} + +// Em processarNovosExames(), troque: +// const resultados = extrairResultados(texto); +// por: +// const registros = extrairComGenIE(texto); +// → registros = [{data_coleta:'2025-12-08', exame:'sodio', resultado:140}, …] +``` + +Para aproveitar os exames novos, faça `atualizarPlanilha()` procurar a coluna +pelo nome do exame e **criar a coluna se não existir** — o GenIE passa a ditar +o schema, e a planilha cresce sozinha (decisão de projeto nº 3 do GenIE, +*Auto Schema Adaptation*). + +### 4.4 Requisito de rede + +O Apps Script roda nos servidores do Google, então o GenIE precisa estar +acessível por **HTTPS público**: um VPS, Cloud Run, ou um túnel +(`cloudflared tunnel`, `ngrok`) apontando para o seu GenIE local. Para testar +sem expor nada, cole o texto OCR direto na interface web do GenIE (entrada +Upload com um `.txt`) e confira a tabela extraída. + +### 4.5 Fallback + +Recomenda-se manter as regex atuais do TabEx como *fallback*: se o GenIE +estiver fora do ar (`try/catch` na `UrlFetchApp.fetch`), o TabEx volta ao +comportamento atual — alinhado ao princípio do GenIE de sempre haver +alternativa quando a IA está indisponível. + +--- + +## 5. Outros cenários de uso (plugin ou standalone) + +| Cenário | Entrada | Saída | +|---|---|---| +| Tabular exames de qualquer formato (TabEx) | `text` (OCR do app) ou `upload` | `api`/`download` | +| Garimpar dados em uma pasta e gerar base para apresentação | `path` | `path` (CSV/JSON) | +| Migrar dados entre bancos ajustando formato | `db` (origem) | `db` (destino) + *formato da saída* descrevendo o schema alvo | +| Carga em outro sistema a partir de sites | `url` (uma execução por site) | `api` do sistema alvo | +| Extração pontual de um arquivo | `upload` | `download` | + +--- + +## 6. Segurança + +- **Chaves de LLM**: cifradas em repouso com AES-256-GCM; chave-mestre via + `GENIE_MASTER_KEY` (produção) ou arquivo local `data/.master_key` (0600). + Nenhum endpoint devolve a chave em claro. +- **Credenciais por execução** (senha de banco, token de API): só em memória, + nunca em logs, eventos ou resultados; URLs exibidas têm credenciais redigidas. +- **Filesystem**: leitura/escrita restritas ao home do usuário, ao diretório do + projeto e a `ALLOWED_FS_ROOTS` — caminhos fora disso são recusados. +- **Uploads**: allowlist de extensões, nomes sanitizados, limites de tamanho e + quantidade; lotes antigos são limpos automaticamente após 24 h. +- **Downloads**: links assinados (HMAC-SHA256) com validade de 15 minutos. +- **CORS**: allowlist explícita (`CORS_ORIGINS`). +- **Exposição pública**: o GenIE não tem autenticação de usuários embutida — + ao publicá-lo na internet (caso TabEx), coloque-o atrás de um proxy com + autenticação (Basic Auth/Nginx, Cloudflare Access, etc.). + +## 7. Configuração (variáveis de ambiente) + +| Variável | Padrão | Descrição | +|---|---|---| +| `GENIE_MASTER_KEY` | *(auto)* | Chave-mestre base64 de 32 bytes (`openssl rand -base64 32`) | +| `API_PORT` | `8000` | Porta do servidor | +| `CORS_ORIGINS` | localhost | Origens permitidas, separadas por vírgula | +| `ALLOWED_FS_ROOTS` | *(vazio)* | Raízes extras de filesystem, separadas por `:` | +| `MAX_UPLOAD_MB` | `50` | Tamanho máximo por arquivo | +| `MAX_FILES_PER_UPLOAD` | `20` | Arquivos por lote | +| `GOOGLE_API_KEY` / `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` | *(vazio)* | Fallback por env (prefira cadastrar pela interface) | +| `DATA_DIR`, `UPLOADS_DIR`, `OUTPUTS_DIR`, `DB_PATH` | `./data/...` | Caminhos de dados | + +## 8. Solução de problemas + +| Sintoma | Causa provável / solução | +|---|---| +| "A chave informada foi recusada pelo provedor" | Chave inválida/expirada. Gere outra no console do provedor | +| "PDF não contém texto extraível (provavelmente digitalizado)" | O GenIE ainda não faz OCR. Use o OCR do Google Drive (como o TabEx) e mande o texto, ou um PDF nativo | +| "Acesso negado ao caminho ..." | Caminho fora das raízes permitidas — ajuste `ALLOWED_FS_ROOTS` | +| "Pastas do Google Drive exigem credenciais..." | Use link direto de arquivo, Upload ou Pasta local | +| "Para conectar a este banco instale sqlalchemy..." | `pip install sqlalchemy psycopg2-binary` (Postgres) ou `pymysql` (MySQL) | +| Extração veio vazia (`records: []`) | Refine o prompt: liste os campos com nomes explícitos e diga o que ignorar | +| Erro 429/503 no log | Limite de taxa do provedor — o GenIE tenta 3x com backoff; aguarde ou troque o modelo | +| Link de download "expirado" | Links valem 15 min — rode novamente ou use saída Pasta local | + +## 9. Limitações conhecidas + +- OCR de PDFs escaneados ainda não é nativo (planejado; contorno: OCR externo + entrada `text`). +- Arquivos `.docx` não são lidos (converta para PDF/TXT). +- Pastas do Google Drive exigem service account (não configurado). +- Saída direta em banco suporta SQLite nativamente (outros via SQLAlchemy). +- Execuções vivem em memória: reiniciar o servidor limpa o histórico de jobs + (as chaves cifradas persistem). diff --git a/pyproject.toml b/pyproject.toml index f3b0d33..2ca8822 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,9 +36,11 @@ target-version = ["py311"] [tool.ruff] line-length = 88 + +[tool.ruff.lint] select = ["E", "F", "I", "N", "W"] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "__init__.py" = ["F401"] [tool.mypy] diff --git a/spec/__init__.py b/spec/__init__.py index ccf4077..0c69301 100644 --- a/spec/__init__.py +++ b/spec/__init__.py @@ -3,6 +3,6 @@ A Python framework for intelligent data extraction using LLMs. """ -__version__ = "0.1.0" +__version__ = "1.0.0" __author__ = "Rodrigo" __license__ = "MIT" diff --git a/spec/api/v1/dependencies.py b/spec/api/v1/dependencies.py index c433942..8d0f7bf 100644 --- a/spec/api/v1/dependencies.py +++ b/spec/api/v1/dependencies.py @@ -48,7 +48,9 @@ def get_search_library(settings: Settings = Depends(get_app_settings)) -> JSONSt return JSONStorage(storage_path=settings.search_library_path) -def get_llm_factory(settings: Settings = Depends(get_app_settings)) -> LLMProviderFactory: +def get_llm_factory( + settings: Settings = Depends(get_app_settings), +) -> LLMProviderFactory: """Get LLM provider factory. Args: diff --git a/spec/api/v1/endpoints/health.py b/spec/api/v1/endpoints/health.py index d556296..5b7b964 100644 --- a/spec/api/v1/endpoints/health.py +++ b/spec/api/v1/endpoints/health.py @@ -1,6 +1,6 @@ """Health check endpoint for API status verification.""" -from datetime import datetime +from datetime import datetime, timezone from fastapi import APIRouter, Depends @@ -10,20 +10,7 @@ router = APIRouter() -class HealthResponse(dict): - """Health check response model. - - Attributes: - status: Health status ("healthy" or "unhealthy") - timestamp: Check timestamp - version: API version - environment: Current environment - """ - - pass - - -@router.get("/health", response_model=dict[str, str | datetime]) +@router.get("/health") async def health_check(settings: Settings = Depends(get_app_settings)) -> dict: """Health check endpoint. @@ -33,21 +20,21 @@ async def health_check(settings: Settings = Depends(get_app_settings)) -> dict: settings: Application settings (injected via Depends) Returns: - dict: Health status information including timestamp, version, and environment + dict: Health status with timestamp, version and environment Example: GET /api/v1/health Response: { "status": "healthy", - "version": "0.1.0", - "timestamp": "2026-03-05T10:30:45.123456", + "version": "1.0.0", + "timestamp": "2026-06-11T10:30:45.123456+00:00", "environment": "development" } """ return { "status": "healthy", - "version": "0.1.0", - "timestamp": datetime.utcnow().isoformat(), + "version": "1.0.0", + "timestamp": datetime.now(timezone.utc).isoformat(), "environment": settings.environment, } diff --git a/spec/api/v1/endpoints/runs.py b/spec/api/v1/endpoints/runs.py index 0b3a8f9..8fff91e 100644 --- a/spec/api/v1/endpoints/runs.py +++ b/spec/api/v1/endpoints/runs.py @@ -73,7 +73,14 @@ async def create_run(request: RunRequest) -> RunCreated: raise HTTPException( status_code=400, detail="Envie os arquivos antes de executar (upload)." ) - if request.input.type != "upload" and not request.input.target.strip(): + if request.input.type == "text" and not request.input.content.strip(): + raise HTTPException( + status_code=400, detail="Entrada 'text' requer o campo 'content'." + ) + if ( + request.input.type not in ("upload", "text") + and not request.input.target.strip() + ): raise HTTPException( status_code=400, detail="Informe o endereço da origem dos dados." ) diff --git a/spec/extraction/agents/connector.py b/spec/extraction/agents/connector.py index 9a9ef8a..a58433a 100644 --- a/spec/extraction/agents/connector.py +++ b/spec/extraction/agents/connector.py @@ -138,10 +138,24 @@ async def open_input(self, spec: InputSpec, emit: EmitFn) -> List[Dict[str, Any] return await self._open_api(spec, emit) if spec.type == "db": return self._open_db(spec, emit) + if spec.type == "text": + return self._open_text(spec, emit) raise InvalidConfig(f"Tipo de entrada não suportado: {spec.type}") # ── Inputs ──────────────────────────────────────────────────────────── + def _open_text(self, spec: InputSpec, emit: EmitFn) -> List[Dict[str, Any]]: + """Wrap inline text content (sent by integrating apps) as a single item.""" + + content = spec.content.strip() + if not content: + raise InvalidConfig( + "Entrada 'text' requer o campo 'content' com o texto a analisar." + ) + name = spec.name.strip() or "conteudo.txt" + emit(message=f"Recebido conteúdo inline ({len(content)} caracteres)") + return [{"id": "text-1", "name": name, "content": content[:400_000]}] + def _open_upload(self, spec: InputSpec, emit: EmitFn) -> List[Dict[str, Any]]: """Read previously uploaded files for this run.""" @@ -480,7 +494,8 @@ def _write_artifacts(self, records: List[Any], directory: Path) -> Dict[str, str columns.append(key) if columns: csv_path = directory / "output.csv" - with open(csv_path, "w", encoding="utf-8", newline="") as f: + # utf-8-sig (BOM) so Excel pt-BR opens accents correctly. + with open(csv_path, "w", encoding="utf-8-sig", newline="") as f: writer = csv.DictWriter( f, fieldnames=columns, extrasaction="ignore" ) diff --git a/spec/extraction/agents/orchestrator.py b/spec/extraction/agents/orchestrator.py index afefa0f..017d248 100644 --- a/spec/extraction/agents/orchestrator.py +++ b/spec/extraction/agents/orchestrator.py @@ -30,6 +30,7 @@ "db": "Banco de dados", "api": "API REST", "upload": "Upload", + "text": "Texto inline", } _OUT_LABELS = { "url": "URL (webhook)", @@ -46,6 +47,8 @@ def _display_target(spec: InputSpec | OutputSpec) -> str: target = re.sub(r"//[^/@]+@", "//••••@", spec.target or "") if isinstance(spec, InputSpec) and spec.type == "upload": return "arquivos enviados" + if isinstance(spec, InputSpec) and spec.type == "text": + return spec.name or "texto inline" if not target: return "saida.json" if getattr(spec, "type", "") == "download" else "—" return target diff --git a/spec/extraction/engine.py b/spec/extraction/engine.py index f5e02a9..c4aa1be 100644 --- a/spec/extraction/engine.py +++ b/spec/extraction/engine.py @@ -74,7 +74,9 @@ async def extract(self, request: ExtractionRequest) -> ExtractionResponse: extraction_id = self._generate_extraction_id() try: - logger.info(f"Starting extraction {extraction_id} with config {request.config_id}") + logger.info( + f"Starting extraction {extraction_id} with config {request.config_id}" + ) # 1. Read content logger.debug("Step 1: Reading content from source") @@ -127,7 +129,9 @@ async def extract(self, request: ExtractionRequest) -> ExtractionResponse: method_used = "llm" # 5. Auto-save pattern if configured - if request.options and request.options.get("auto_create_patterns", True): + if request.options and request.options.get( + "auto_create_patterns", True + ): logger.debug("Step 5: Auto-saving pattern") try: new_pattern = self._generate_pattern_from_extraction( diff --git a/spec/extraction/llm/anthropic.py b/spec/extraction/llm/anthropic.py index a602578..c85ef90 100644 --- a/spec/extraction/llm/anthropic.py +++ b/spec/extraction/llm/anthropic.py @@ -24,8 +24,8 @@ class AnthropicProvider(BaseLLMProvider): def __init__( self, api_key: str, - model: str = "claude-sonnet-4-20250514", - max_tokens: int = 4096, + model: str = "claude-sonnet-4-6", + max_tokens: int = 8192, temperature: float = 0.0, ) -> None: """Initialize Anthropic provider. diff --git a/spec/extraction/llm/factory.py b/spec/extraction/llm/factory.py index 3aba439..e79487c 100644 --- a/spec/extraction/llm/factory.py +++ b/spec/extraction/llm/factory.py @@ -69,7 +69,9 @@ def get_provider( resolved_key = api_key or self._get_api_key_for_provider(provider_name) if not resolved_key: - raise InvalidConfig(f"API key for provider '{provider_name}' is not configured") + raise InvalidConfig( + f"API key for provider '{provider_name}' is not configured" + ) # Never put key material (even a prefix) in cache keys: hash it. key_digest = hashlib.sha256(resolved_key.encode("utf-8")).hexdigest()[:16] @@ -134,7 +136,9 @@ def set_provider_config( self._providers.clear() - logger.info(f"Provider configured: {provider}, model: {model or _DEFAULT_MODELS[provider]}") + logger.info( + f"Provider configured: {provider}, model: {model or _DEFAULT_MODELS[provider]}" + ) @staticmethod def list_providers() -> list[dict]: diff --git a/spec/extraction/llm/google.py b/spec/extraction/llm/google.py index 5033f53..ba5812e 100644 --- a/spec/extraction/llm/google.py +++ b/spec/extraction/llm/google.py @@ -32,8 +32,8 @@ class GoogleProvider(BaseLLMProvider): def __init__( self, api_key: str, - model: str = "gemini-1.5-pro", - max_tokens: int = 4096, + model: str = "gemini-2.5-flash", + max_tokens: int = 16384, temperature: float = 0.0, ) -> None: """Initialize Google Gemini provider. @@ -55,10 +55,13 @@ def __init__( self.max_tokens = max_tokens self.temperature = temperature self.client = genai.Client(api_key=api_key) + # response_mime_type forces native JSON mode; the generous token limit + # accounts for Gemini 2.5 "thinking" tokens that also consume output. self._generate_config = types.GenerateContentConfig( system_instruction=_SYSTEM_INSTRUCTION, temperature=temperature, max_output_tokens=max_tokens, + response_mime_type="application/json", ) async def extract( diff --git a/spec/extraction/llm/openai.py b/spec/extraction/llm/openai.py index 07179fc..939791f 100644 --- a/spec/extraction/llm/openai.py +++ b/spec/extraction/llm/openai.py @@ -32,7 +32,7 @@ def __init__( self, api_key: str, model: str = "gpt-4o", - max_tokens: int = 4096, + max_tokens: int = 8192, temperature: float = 0.0, ) -> None: """Initialize OpenAI provider. @@ -84,6 +84,7 @@ async def extract( model=self.model, max_tokens=self.max_tokens, temperature=self.temperature, + response_format={"type": "json_object"}, messages=[ {"role": "system", "content": _SYSTEM_MESSAGE}, {"role": "user", "content": prompt}, diff --git a/spec/extraction/parsers/pdf.py b/spec/extraction/parsers/pdf.py index 6190499..743a520 100644 --- a/spec/extraction/parsers/pdf.py +++ b/spec/extraction/parsers/pdf.py @@ -42,7 +42,9 @@ async def extract_text( source_type = source.get("type", "").lower() if source_type not in ("file", "pdf"): - raise InvalidConfig(f"PDFParser does not support source type: {source_type}") + raise InvalidConfig( + f"PDFParser does not support source type: {source_type}" + ) path = source.get("path") if not path: @@ -70,9 +72,9 @@ async def extract_text( page_text = "" # Check if page has enough text (not scanned) - if ( - detect_scanned - and (not page_text or len(page_text) < PDFParser.SCANNED_PDF_THRESHOLD) + if detect_scanned and ( + not page_text + or len(page_text) < PDFParser.SCANNED_PDF_THRESHOLD ): raise ExtractionFailed( f"PDF appears to be scanned (no text on page {page_num}). " diff --git a/spec/main.py b/spec/main.py index d02fa11..2487969 100644 --- a/spec/main.py +++ b/spec/main.py @@ -1,5 +1,7 @@ """FastAPI application entry point for GENIE framework.""" +import shutil +import time from contextlib import asynccontextmanager from pathlib import Path from typing import AsyncGenerator @@ -18,6 +20,26 @@ logger = get_logger(__name__) +def _cleanup_stale_dirs(root: Path, max_age_hours: int) -> None: + """Remove stale upload/output batch directories left by old runs. + + Args: + root: Directory containing per-batch subdirectories + max_age_hours: Age threshold for removal + """ + + if not root.is_dir(): + return + cutoff = time.time() - max_age_hours * 3600 + for batch_dir in root.iterdir(): + try: + if batch_dir.is_dir() and batch_dir.stat().st_mtime < cutoff: + shutil.rmtree(batch_dir, ignore_errors=True) + logger.debug(f"Removed stale directory: {batch_dir}") + except OSError: # pragma: no cover - best-effort housekeeping + continue + + @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: """Lifespan context manager for app startup and shutdown. @@ -35,6 +57,8 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: logger.info(f"Starting GENIE API in {settings.environment} mode") logger.debug(f"Log level: {settings.log_level}") logger.debug(f"API: {settings.api_host}:{settings.api_port}") + _cleanup_stale_dirs(Path(settings.uploads_dir), max_age_hours=24) + _cleanup_stale_dirs(Path(settings.outputs_dir), max_age_hours=24) yield @@ -46,7 +70,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: app = FastAPI( title="GENIE - Generic Extractor of Information Engine", description="LLM-powered data extraction framework", - version="0.1.0", + version="1.0.0", lifespan=lifespan, ) diff --git a/spec/models/config.py b/spec/models/config.py index e0304ab..7db0475 100644 --- a/spec/models/config.py +++ b/spec/models/config.py @@ -94,6 +94,7 @@ class ExtractionConfig(BaseModel): class Config: """Pydantic configuration.""" + json_schema_extra = { "example": { "extraction_id": "config_001", @@ -101,25 +102,25 @@ class Config: "input": { "type": "pdf", "source": "/uploads/reports", - "access_mode": "local_secure" + "access_mode": "local_secure", }, "output": { "type": "json", "destination": "/outputs", - "auto_adapt": True + "auto_adapt": True, }, "llm": { "provider": "anthropic", "model": "claude-sonnet-4-20250514", "temperature": 0.0, - "max_tokens": 4096 + "max_tokens": 4096, }, "behavior": { "use_search_library": True, "auto_create_patterns": True, "layout_independent": True, - "update_on_change": True + "update_on_change": True, }, - "extraction_instructions": "Extract patient name, age, and test results." + "extraction_instructions": "Extract patient name, age, and test results.", } } diff --git a/spec/models/extraction.py b/spec/models/extraction.py index d3739ef..9b72710 100644 --- a/spec/models/extraction.py +++ b/spec/models/extraction.py @@ -46,6 +46,7 @@ class ExtractionResponse(BaseModel): class Config: """Pydantic configuration.""" + json_schema_extra = { "example": { "extraction_id": "ext_123456", @@ -55,6 +56,6 @@ class Config: "confidence": 0.95, "processing_time_ms": 1250, "layout_fingerprint": "a1b2c3d4e5f6g7h8", - "error": None + "error": None, } } diff --git a/spec/models/library.py b/spec/models/library.py index 7d10692..fff7f8f 100644 --- a/spec/models/library.py +++ b/spec/models/library.py @@ -51,6 +51,7 @@ class SearchPattern(BaseModel): class Config: """Pydantic configuration.""" + json_schema_extra = { "example": { "layout_id": "layout_abc123", @@ -66,9 +67,9 @@ class Config: "extraction_method": "regex", "pattern": r"Patient:\s*([^\n]+)", "validation": r".{2,}", - "post_process": None + "post_process": None, } - ] + ], } } diff --git a/spec/models/output.py b/spec/models/output.py index 635b36e..f0dcbc2 100644 --- a/spec/models/output.py +++ b/spec/models/output.py @@ -32,30 +32,27 @@ class OutputSchema(BaseModel): class Config: """Pydantic configuration.""" + json_schema_extra = { "example": { "fields": { "patient_id": { "name": "patient_id", "type": "string", - "required": True + "required": True, }, "patient_name": { "name": "patient_name", "type": "string", - "required": True + "required": True, }, "exam_date": { "name": "exam_date", "type": "date", - "required": True + "required": True, }, - "result": { - "name": "result", - "type": "string", - "required": False - } + "result": {"name": "result", "type": "string", "required": False}, }, - "primary_key": "patient_id" + "primary_key": "patient_id", } } diff --git a/spec/models/webapp.py b/spec/models/webapp.py index 386b98b..9a47b78 100644 --- a/spec/models/webapp.py +++ b/spec/models/webapp.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, Field -InputType = Literal["url", "path", "db", "api", "upload"] +InputType = Literal["url", "path", "db", "api", "upload", "text"] OutputType = Literal["url", "path", "db", "api", "download"] AgentName = Literal["conector", "localizador", "organizador", "sistema"] EventType = Literal["progress", "log", "done", "error", "finish"] @@ -15,13 +15,16 @@ class InputSpec(BaseModel): """Source specification for the Connector agent. Attributes: - type: Input kind (url, path, db, api, upload) + type: Input kind (url, path, db, api, upload, text) target: Address (URL, filesystem path, DB URL, API endpoint) user: Username for db connections password: Password for db connections (transient, never persisted) token: Bearer token for api connections (transient, never persisted) query: Optional SQL query for db inputs upload_id: Upload batch id for upload inputs + content: Inline text content for text inputs (plugin-friendly: + other apps can send pre-extracted/OCR text in a single POST) + name: Optional display name for text inputs """ type: InputType @@ -31,6 +34,8 @@ class InputSpec(BaseModel): token: str = "" query: str = "" upload_id: Optional[str] = None + content: str = "" + name: str = "" class OutputSpec(BaseModel): diff --git a/spec/search_library/json_storage.py b/spec/search_library/json_storage.py index e97e9c1..845f678 100644 --- a/spec/search_library/json_storage.py +++ b/spec/search_library/json_storage.py @@ -24,7 +24,9 @@ class JSONStorage(BaseStorage): - Pattern success rate tracking """ - def __init__(self, storage_path: str = "./data/search_library/patterns.json") -> None: + def __init__( + self, storage_path: str = "./data/search_library/patterns.json" + ) -> None: """Initialize JSON storage. Args: @@ -178,9 +180,7 @@ async def save_pattern( # Add to storage self._cache["patterns"].append(new_pattern) - self._cache["metadata"]["total_patterns"] = len( - self._cache["patterns"] - ) + self._cache["metadata"]["total_patterns"] = len(self._cache["patterns"]) self._cache["metadata"]["last_updated"] = now # Persist @@ -221,9 +221,9 @@ async def update_success_rate( new_rate = (old_rate * (use_count - 1) + success_value) / use_count pattern["success_rate"] = round(new_rate, 3) - self._cache["metadata"]["last_updated"] = ( - datetime.utcnow().isoformat() - ) + self._cache["metadata"][ + "last_updated" + ] = datetime.utcnow().isoformat() self._save_storage(self._cache) diff --git a/spec/search_library/matcher.py b/spec/search_library/matcher.py index e96e17f..e993a9a 100644 --- a/spec/search_library/matcher.py +++ b/spec/search_library/matcher.py @@ -63,7 +63,9 @@ async def extract_with_pattern( elif extraction_method in ("instruction", "query"): # Phase 2+ features - logger.warning(f"Extraction method '{extraction_method}' not yet supported") + logger.warning( + f"Extraction method '{extraction_method}' not yet supported" + ) extracted[field_name] = None else: diff --git a/spec/web/app.js b/spec/web/app.js index 02fa2f3..65ece69 100644 --- a/spec/web/app.js +++ b/spec/web/app.js @@ -535,8 +535,10 @@ async function startRun() { function subscribe(jobId) { const source = new EventSource(`${API}/runs/${jobId}/events`); state.eventSource = source; + let consecutiveFailures = 0; source.onmessage = (message) => { + consecutiveFailures = 0; let event; try { event = JSON.parse(message.data); } catch { return; } applyEvent(event); @@ -546,6 +548,7 @@ function subscribe(jobId) { // EventSource auto-reconnects with Last-Event-ID; double-check job state. try { const info = await apiJson(`/runs/${jobId}`); + consecutiveFailures = 0; if (info.status !== "running" && info.status !== "queued") { closeStream(); state.run.status = info.status; @@ -553,7 +556,16 @@ function subscribe(jobId) { if (info.error) pushLocalLog("sistema", info.error, "error"); renderActionBar(); renderMonitor(); } - } catch { /* transient; let EventSource retry */ } + } catch { + // Job gone (e.g. server restarted) or network down: give up after a few tries. + consecutiveFailures += 1; + if (consecutiveFailures >= 4) { + closeStream(); + state.run.status = "error"; + pushLocalLog("sistema", "Conexão com o servidor perdida — execução interrompida no monitor.", "error"); + renderActionBar(); renderMonitor(); + } + } }; } diff --git a/spec/webapp/jobs.py b/spec/webapp/jobs.py index fb07595..930835a 100644 --- a/spec/webapp/jobs.py +++ b/spec/webapp/jobs.py @@ -78,8 +78,12 @@ def create(self, request: RunRequest) -> Job: self._jobs[job.id] = job while len(self._jobs) > self._max_jobs: - oldest_id = next(iter(self._jobs)) - evicted = self._jobs.pop(oldest_id) + # Prefer evicting finished jobs; fall back to the oldest one. + evict_id = next( + (jid for jid, j in self._jobs.items() if j.is_finished), + next(iter(self._jobs)), + ) + evicted = self._jobs.pop(evict_id) if evicted.task and not evicted.task.done(): evicted.task.cancel() @@ -165,17 +169,23 @@ async def stream( job._subscribers.append(queue) try: - replay_from = last_event_id or 0 + # Events emitted while we replay history also land in the queue; + # track the highest seq yielded so far to skip those duplicates. + last_seq = last_event_id or 0 for event in list(job.events): - if event.seq > replay_from: + if event.seq > last_seq: yield event + last_seq = event.seq if job.is_finished: return while True: event = await queue.get() + if event.seq <= last_seq: + continue yield event + last_seq = event.seq if event.type in ("finish", "error") and event.status in _TERMINAL: return finally: diff --git a/tests/integration/test_webapp_api.py b/tests/integration/test_webapp_api.py index b41500d..5d0d08d 100644 --- a/tests/integration/test_webapp_api.py +++ b/tests/integration/test_webapp_api.py @@ -206,6 +206,90 @@ async def test_full_pipeline_upload_to_download(self, client, monkeypatch): tampered = result["downloads"]["json"].replace("sig=", "sig=ff") assert (await client.get(tampered)).status_code == 403 + async def test_text_input_rejected_without_content(self, client): + get_key_vault().store("google", "AIzaSyFakeKey1234") + response = await client.post( + "/api/v1/runs", + json={ + "model_id": "gemini-2.5-flash", + "input": {"type": "text", "content": " "}, + "prompt": "extraia", + "output": {"type": "download"}, + }, + ) + + assert response.status_code == 400 + assert "content" in response.json()["detail"] + + async def test_text_input_pipeline(self, client, monkeypatch): + """Plugin-style integration: inline OCR text in, records out (TabEx case).""" + + get_key_vault().store("google", "AIzaSyFakeKey1234") + monkeypatch.setattr( + orchestrator_module.Orchestrator, + "_resolve_provider", + lambda self, model_id: FakeProvider(), + ) + + created = await client.post( + "/api/v1/runs", + json={ + "model_id": "gemini-2.5-flash", + "input": { + "type": "text", + "content": "SODIO: 140 mEq/L\nGlicose: 118 mg/dL", + "name": "ocr-sus.txt", + }, + "prompt": "Extraia exame e resultado", + "output": {"type": "download"}, + }, + ) + assert created.status_code == 201 + job_id = created.json()["job_id"] + + for _ in range(50): + info = (await client.get(f"/api/v1/runs/{job_id}")).json() + if info["status"] in ("done", "error", "cancelled"): + break + await asyncio.sleep(0.1) + + assert info["status"] == "done", info.get("error") + assert info["result"]["records"] == FakeProvider().records + assert info["result"]["source"] == {"type": "text", "target": "ocr-sus.txt"} + + async def test_events_have_no_duplicate_seq(self, client, monkeypatch): + get_key_vault().store("google", "AIzaSyFakeKey1234") + monkeypatch.setattr( + orchestrator_module.Orchestrator, + "_resolve_provider", + lambda self, model_id: FakeProvider(), + ) + + created = await client.post( + "/api/v1/runs", + json={ + "model_id": "gemini-2.5-flash", + "input": {"type": "text", "content": "Glicose: 118 mg/dL"}, + "prompt": "Extraia exame e resultado", + "output": {"type": "download"}, + }, + ) + job_id = created.json()["job_id"] + + for _ in range(50): + info = (await client.get(f"/api/v1/runs/{job_id}")).json() + if info["status"] in ("done", "error", "cancelled"): + break + await asyncio.sleep(0.1) + + events = await client.get(f"/api/v1/runs/{job_id}/events") + seqs = [ + json.loads(line[len("data: "):])["seq"] + for line in events.text.splitlines() + if line.startswith("data: ") + ] + assert seqs == sorted(set(seqs)) + async def test_credentials_never_leak_into_events(self, client, monkeypatch): get_key_vault().store("google", "AIzaSyFakeKey1234") monkeypatch.setattr( From 33128f8a0d932f8df83b2c0e97dfb69befad338c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 22:04:36 +0000 Subject: [PATCH 3/3] feat: close Phase 1 plan gaps and restructure repo per GENIE-TODO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the remaining Phase 1 items from docs/guides/GENIE-TODO.md (Quality Gate 1.4) after reviewing the original implementation plan: - Config CRUD endpoints (POST/GET/PUT/DELETE /api/v1/configs) backed by a new file-based ConfigStore (data/configs/{id}.json, safe id validation) - Search Library endpoints (GET /api/v1/library/patterns, /patterns/{id}, /stats with usage aggregates) - ExtractionEngine now loads the stored ExtractionConfig: uses its extraction_instructions, output schema and provider/model, and honors behavior.use_search_library / auto_create_patterns flags (decision nº 1: generic, not general — previously config_id was accepted but ignored) - scripts/test_llm_connection.py (planned Stage 1.2.2 utility) Test coverage raised from 65% to 81% (plan target: >= 80%), 127 tests: - tests/unit/test_llm_providers.py (mocked clients; planned Stage 1.2.2) - tests/integration/test_end_to_end.py (config -> LLM -> pattern saved -> library hit with zero LLM cost -> force_llm bypass; planned Stage 1.4.4) - tests/unit/test_search_library.py (matcher + success-rate moving average) - connector sqlite input, inline text input, HTTP delivery, XLSX parsing - providers endpoint tests Restructuring (alignment with the plan): - Removed empty spec/api/v1/routes/ and spec/output/formatters/ (not in plan) - Moved setup.sh and test-genie.py into scripts/ - Moved root status docs (IMPLEMENTATION-SUMMARY, HOMOLOGATION-CHECKLIST, FINAL-RELEASE-NOTES, TESTE-RAPIDO, PHASE-1-STATUS) to docs/history/ - Updated living docs: GENIE-TODO.md checkboxes (Phase 1 complete, dated) + addendum for the web app delivery; PROJECT-AUDIT.md status appendix - Updated stale default model ids in models/config.py https://claude.ai/code/session_01CKjevqGgfWLggV1DG1Tmpq --- docs/guides/GENIE-TODO.md | 157 ++++++++------- docs/guides/PROJECT-AUDIT.md | 24 +++ .../history/FINAL-RELEASE-NOTES.md | 0 .../history/HOMOLOGATION-CHECKLIST.md | 0 .../history/IMPLEMENTATION-SUMMARY.md | 0 .../history/PHASE-1-STATUS.md | 0 .../history/TESTE-RAPIDO.md | 0 setup.sh => scripts/setup.sh | 0 test-genie.py => scripts/test-genie.py | 0 scripts/test_llm_connection.py | 47 +++++ spec/api/v1/dependencies.py | 15 ++ spec/api/v1/endpoints/config.py | 148 +++++++++++++++ spec/api/v1/endpoints/library.py | 94 +++++++++ spec/api/v1/router.py | 4 + spec/api/v1/routes/__init__.py | 0 spec/core/config_store.py | 127 +++++++++++++ spec/extraction/engine.py | 84 +++++++-- spec/models/config.py | 4 +- spec/output/formatters/__init__.py | 0 tests/conftest.py | 6 +- tests/integration/test_config_library_api.py | 126 +++++++++++++ tests/integration/test_end_to_end.py | 178 ++++++++++++++++++ tests/integration/test_health.py | 2 +- tests/integration/test_providers_api.py | 91 +++++++++ tests/unit/test_config.py | 1 - tests/unit/test_connector.py | 165 ++++++++++++++++ tests/unit/test_exceptions.py | 5 +- tests/unit/test_fingerprint.py | 1 - tests/unit/test_llm_providers.py | 170 +++++++++++++++++ tests/unit/test_models.py | 10 +- tests/unit/test_parsers.py | 6 +- tests/unit/test_search_library.py | 126 +++++++++++++ 32 files changed, 1485 insertions(+), 106 deletions(-) rename FINAL-RELEASE-NOTES.md => docs/history/FINAL-RELEASE-NOTES.md (100%) rename HOMOLOGATION-CHECKLIST.md => docs/history/HOMOLOGATION-CHECKLIST.md (100%) rename IMPLEMENTATION-SUMMARY.md => docs/history/IMPLEMENTATION-SUMMARY.md (100%) rename PHASE-1-STATUS.md => docs/history/PHASE-1-STATUS.md (100%) rename TESTE-RAPIDO.md => docs/history/TESTE-RAPIDO.md (100%) rename setup.sh => scripts/setup.sh (100%) rename test-genie.py => scripts/test-genie.py (100%) create mode 100644 scripts/test_llm_connection.py create mode 100644 spec/api/v1/endpoints/config.py create mode 100644 spec/api/v1/endpoints/library.py delete mode 100644 spec/api/v1/routes/__init__.py create mode 100644 spec/core/config_store.py delete mode 100644 spec/output/formatters/__init__.py create mode 100644 tests/integration/test_config_library_api.py create mode 100644 tests/integration/test_end_to_end.py create mode 100644 tests/integration/test_providers_api.py create mode 100644 tests/unit/test_llm_providers.py create mode 100644 tests/unit/test_search_library.py diff --git a/docs/guides/GENIE-TODO.md b/docs/guides/GENIE-TODO.md index 01681d6..64e2612 100644 --- a/docs/guides/GENIE-TODO.md +++ b/docs/guides/GENIE-TODO.md @@ -2,9 +2,9 @@ ## Living Document -> **Status:** Pre-development (no code written yet) +> **Status:** Phase 1 COMPLETE ✅ + Web App (3 agentes) entregue — ver Adendo 2026-06-11 > **Created:** 2026-03-05 -> **Last Updated:** 2026-03-05 +> **Last Updated:** 2026-06-11 > > **How to maintain:** Update checkboxes as items are completed. Add dates in `(YYYY-MM-DD)` after completed items. Add new items as requirements emerge. Never delete completed items — they serve as project history. @@ -20,17 +20,17 @@ #### Stage 1.1.1 — Repository & Tooling -- [ ] `[S]` Initialize Git repository with `.gitignore` (Python, IDE, .env) -- [ ] `[S]` Create `pyproject.toml` with Poetry (Python ^3.11) -- [ ] `[S]` Add core dependencies: `fastapi`, `uvicorn`, `pydantic`, `pydantic-settings`, `anthropic` -- [ ] `[S]` Add dev dependencies: `pytest`, `pytest-asyncio`, `pytest-cov`, `ruff`, `mypy`, `black` -- [ ] `[S]` Create `.env.example` with all required environment variables -- [ ] `[S]` Create `config/development.yaml` with default settings -- [ ] `[S]` Setup `ruff` and `mypy` configuration in `pyproject.toml` +- [x] `[S]` Initialize Git repository with `.gitignore` (Python, IDE, .env) ✅ (2026-06-11) +- [x] `[S]` Create `pyproject.toml` with Poetry (Python ^3.11) ✅ (2026-06-11) +- [x] `[S]` Add core dependencies: `fastapi`, `uvicorn`, `pydantic`, `pydantic-settings`, `anthropic` ✅ (2026-06-11) +- [x] `[S]` Add dev dependencies: `pytest`, `pytest-asyncio`, `pytest-cov`, `ruff`, `mypy`, `black` ✅ (2026-06-11) +- [x] `[S]` Create `.env.example` with all required environment variables ✅ (2026-06-11) +- [x] `[S]` Create `config/development.yaml` with default settings ✅ (2026-06-11) +- [x] `[S]` Setup `ruff` and `mypy` configuration in `pyproject.toml` ✅ (2026-06-11) #### Stage 1.1.2 — Folder Structure -- [ ] `[M]` Create full package structure under `genie/`: +- [x] `[M]` Create full package structure under `genie/`: ✅ (2026-06-11) - `api/v1/endpoints/`, `api/v1/dependencies.py` - `core/` (`config.py`, `exceptions.py`, `security.py`, `logging_config.py`) - `models/` (`extraction.py`, `config.py`, `library.py`, `output.py`) @@ -38,22 +38,22 @@ - `search_library/` (`base.py`, `json_storage.py`) - `output/` (`manager.py`, `adapters/`, `schema_adapter.py`) - `mcp/`, `utils/` -- [ ] `[S]` Create `__init__.py` files for all packages -- [ ] `[S]` Create `tests/` structure: `unit/`, `integration/`, `fixtures/` -- [ ] `[S]` Create `data/search_library/` and `data/uploads/` with `.gitkeep` -- [ ] `[S]` Create `scripts/` directory with `setup.sh` placeholder +- [x] `[S]` Create `__init__.py` files for all packages ✅ (2026-06-11) +- [x] `[S]` Create `tests/` structure: `unit/`, `integration/`, `fixtures/` ✅ (2026-06-11) +- [x] `[S]` Create `data/search_library/` and `data/uploads/` with `.gitkeep` ✅ (2026-06-11) +- [x] `[S]` Create `scripts/` directory with `setup.sh` placeholder ✅ (2026-06-11) #### Stage 1.1.3 — Core Infrastructure -- [ ] `[M]` Implement `genie/core/config.py` — Pydantic Settings model (env vars, paths, API config) -- [ ] `[M]` Implement `genie/core/exceptions.py` — `GenieException` hierarchy: +- [x] `[M]` Implement `genie/core/config.py` — Pydantic Settings model (env vars, paths, API config) ✅ (2026-06-11) +- [x] `[M]` Implement `genie/core/exceptions.py` — `GenieException` hierarchy: ✅ (2026-06-11) - `LayoutNotRecognized`, `ExtractionFailed`, `LLMProviderError`, `InvalidConfig`, `StorageError` -- [ ] `[M]` Implement `genie/core/logging_config.py` — structured logging setup (console + file handlers) -- [ ] `[S]` Implement `genie/core/security.py` — `SecureKeyStore` with Fernet encryption (placeholder) -- [ ] `[M]` Implement `genie/main.py` — FastAPI app creation, CORS, lifespan events -- [ ] `[S]` Implement `genie/api/v1/endpoints/health.py` — `GET /health` endpoint -- [ ] `[S]` Write test: `tests/unit/test_config.py` — validate settings loading -- [ ] `[S]` Write test: `tests/integration/test_health.py` — health endpoint responds 200 +- [x] `[M]` Implement `genie/core/logging_config.py` — structured logging setup (console + file handlers) ✅ (2026-06-11) +- [x] `[S]` Implement `genie/core/security.py` — `SecureKeyStore` with Fernet encryption (placeholder) ✅ (2026-06-11) +- [x] `[M]` Implement `genie/main.py` — FastAPI app creation, CORS, lifespan events ✅ (2026-06-11) +- [x] `[S]` Implement `genie/api/v1/endpoints/health.py` — `GET /health` endpoint ✅ (2026-06-11) +- [x] `[S]` Write test: `tests/unit/test_config.py` — validate settings loading ✅ (2026-06-11) +- [x] `[S]` Write test: `tests/integration/test_health.py` — health endpoint responds 200 ✅ (2026-06-11) > **Quality Gate 1.1:** Server starts with `uvicorn genie.main:app --reload`, health endpoint returns `{"status": "healthy"}`, all tests pass. @@ -65,44 +65,44 @@ #### Stage 1.2.1 — Pydantic Models -- [ ] `[M]` Implement `genie/models/extraction.py`: +- [x] `[M]` Implement `genie/models/extraction.py`: ✅ (2026-06-11) - `ExtractionRequest` (config_id, source, force_llm, options) - `ExtractionResponse` (extraction_id, status, method_used, data, confidence, processing_time_ms, layout_fingerprint) -- [ ] `[M]` Implement `genie/models/config.py`: +- [x] `[M]` Implement `genie/models/config.py`: ✅ (2026-06-11) - `ExtractionConfig` (extraction_id, input, output, llm, behavior) - `InputConfig`, `OutputConfig`, `LLMConfig`, `BehaviorConfig` -- [ ] `[S]` Implement `genie/models/library.py`: +- [x] `[S]` Implement `genie/models/library.py`: ✅ (2026-06-11) - `SearchPattern`, `PatternField`, `LibraryMetadata` -- [ ] `[S]` Implement `genie/models/output.py`: +- [x] `[S]` Implement `genie/models/output.py`: ✅ (2026-06-11) - `OutputSchema`, `FieldDefinition` -- [ ] `[S]` Write tests: `tests/unit/test_models.py` — validation for all models +- [x] `[S]` Write tests: `tests/unit/test_models.py` — validation for all models ✅ (2026-06-11) #### Stage 1.2.2 — LLM Provider Interface & Anthropic -- [ ] `[M]` Implement `genie/extraction/llm/base.py`: +- [x] `[M]` Implement `genie/extraction/llm/base.py`: ✅ (2026-06-11) - `BaseLLMProvider` ABC with `extract()`, `_build_prompt()`, `_parse_response()` -- [ ] `[L]` Implement `genie/extraction/llm/anthropic.py`: +- [x] `[L]` Implement `genie/extraction/llm/anthropic.py`: ✅ (2026-06-11) - `AnthropicProvider` with async Claude API calls - Prompt engineering for structured extraction - JSON response parsing with markdown cleanup -- [ ] `[M]` Implement `genie/extraction/llm/factory.py`: +- [x] `[M]` Implement `genie/extraction/llm/factory.py`: ✅ (2026-06-11) - `LLMProviderFactory` — creates provider instances by name -- [ ] `[M]` Write tests: `tests/unit/test_llm_providers.py` — mock API calls, validate prompt building and response parsing -- [ ] `[S]` Create `scripts/test_llm_connection.py` — manual LLM connectivity test +- [x] `[M]` Write tests: `tests/unit/test_llm_providers.py` — mock API calls, validate prompt building and response parsing ✅ (2026-06-11) +- [x] `[S]` Create `scripts/test_llm_connection.py` — manual LLM connectivity test ✅ (2026-06-11) #### Stage 1.2.3 — Text Parser & Basic Extraction -- [ ] `[M]` Implement `genie/extraction/parsers/text.py`: +- [x] `[M]` Implement `genie/extraction/parsers/text.py`: ✅ (2026-06-11) - `TextParser` — plain text content reading -- [ ] `[S]` Implement `genie/extraction/engine.py` — initial `ExtractionEngine` skeleton: +- [x] `[S]` Implement `genie/extraction/engine.py` — initial `ExtractionEngine` skeleton: ✅ (2026-06-11) - `extract()` method with LLM-only flow (no Search Library yet) - `_read_content()` dispatching to parsers -- [ ] `[M]` Implement `genie/api/v1/endpoints/extract.py`: +- [x] `[M]` Implement `genie/api/v1/endpoints/extract.py`: ✅ (2026-06-11) - `POST /api/v1/extract` — accepts text source, returns extracted data -- [ ] `[M]` Implement `genie/api/v1/dependencies.py`: +- [x] `[M]` Implement `genie/api/v1/dependencies.py`: ✅ (2026-06-11) - Dependency injection for `ExtractionEngine`, `LLMProviderFactory` -- [ ] `[M]` Write tests: `tests/integration/test_api.py` — extract endpoint with text input -- [ ] `[S]` Write tests: `tests/unit/test_parsers.py` — text parser +- [x] `[M]` Write tests: `tests/integration/test_api.py` — extract endpoint with text input ✅ (2026-06-11) +- [x] `[S]` Write tests: `tests/unit/test_parsers.py` — text parser ✅ (2026-06-11) > **Quality Gate 1.2:** `POST /api/v1/extract` with text source returns structured JSON via LLM. All unit and integration tests pass. @@ -114,23 +114,23 @@ #### Stage 1.3.1 — PDF Parser -- [ ] `[S]` Add dependency: `PyPDF2` -- [ ] `[M]` Implement `genie/extraction/parsers/pdf.py`: +- [x] `[S]` Add dependency: `PyPDF2` ✅ (2026-06-11) +- [x] `[M]` Implement `genie/extraction/parsers/pdf.py`: ✅ (2026-06-11) - `PDFParser` — text extraction from native PDFs (page-by-page) - Scanned PDF detection (fallback flag for OCR) -- [ ] `[S]` Update `ExtractionEngine._read_content()` to dispatch PDF sources to `PDFParser` -- [ ] `[S]` Update `POST /api/v1/extract` to accept `"type": "file"` sources with path +- [x] `[S]` Update `ExtractionEngine._read_content()` to dispatch PDF sources to `PDFParser` ✅ (2026-06-11) +- [x] `[S]` Update `POST /api/v1/extract` to accept `"type": "file"` sources with path ✅ (2026-06-11) - [ ] `[S]` Add sample PDFs to `tests/fixtures/sample_pdfs/` -- [ ] `[M]` Write tests: `tests/unit/test_parsers.py` — PDF text extraction +- [x] `[M]` Write tests: `tests/unit/test_parsers.py` — PDF text extraction ✅ (2026-06-11) #### Stage 1.3.2 — Layout Fingerprint Algorithm -- [ ] `[L]` Implement `genie/extraction/layout/fingerprint.py`: +- [x] `[L]` Implement `genie/extraction/layout/fingerprint.py`: ✅ (2026-06-11) - `LayoutFingerprint.generate()` — structure extraction (remove variable data, keep labels/formatting) - `LayoutFingerprint.similarity()` — fingerprint comparison (Hamming distance) - Configurable sensitivity levels (low/medium/high) -- [ ] `[M]` Integrate fingerprinting into `ExtractionEngine.extract()` — generate fingerprint on every extraction -- [ ] `[M]` Write tests: `tests/unit/test_fingerprint.py`: +- [x] `[M]` Integrate fingerprinting into `ExtractionEngine.extract()` — generate fingerprint on every extraction ✅ (2026-06-11) +- [x] `[M]` Write tests: `tests/unit/test_fingerprint.py`: ✅ (2026-06-11) - Same layout with different data produces same fingerprint - Different layouts produce different fingerprints - Similarity scoring works correctly @@ -145,28 +145,28 @@ #### Stage 1.4.1 — JSON Storage Implementation -- [ ] `[M]` Implement `genie/search_library/base.py`: +- [x] `[M]` Implement `genie/search_library/base.py`: ✅ (2026-06-11) - `BaseStorage` ABC with `find_pattern()`, `save_pattern()`, `update_success_rate()`, `list_patterns()` -- [ ] `[L]` Implement `genie/search_library/json_storage.py`: +- [x] `[L]` Implement `genie/search_library/json_storage.py`: ✅ (2026-06-11) - `JSONStorage` implementing `BaseStorage` - File-based CRUD with in-memory cache - Thread-safe read/write operations - Pattern matching by fingerprint + config_id - Success rate tracking (moving average) -- [ ] `[M]` Implement `genie/search_library/matcher.py`: +- [x] `[M]` Implement `genie/search_library/matcher.py`: ✅ (2026-06-11) - `PatternMatcher` — execute REGEX patterns against content - Validation of extracted data against pattern rules -- [ ] `[M]` Write tests: `tests/unit/test_search_library.py` — CRUD, pattern lookup, success rate +- [x] `[M]` Write tests: `tests/unit/test_search_library.py` — CRUD, pattern lookup, success rate ✅ (2026-06-11) #### Stage 1.4.2 — ExtractionEngine Full Flow -- [ ] `[L]` Complete `genie/extraction/engine.py`: +- [x] `[L]` Complete `genie/extraction/engine.py`: ✅ (2026-06-11) - Full extraction flow: fingerprint → library lookup → LLM fallback → save pattern - Confidence calculation based on extraction method - `force_llm` option support - Pattern auto-save after successful LLM extraction -- [ ] `[M]` Update `genie/api/v1/dependencies.py` — inject `SearchLibrary` into engine -- [ ] `[M]` Write tests: `tests/unit/test_extraction_engine.py`: +- [x] `[M]` Update `genie/api/v1/dependencies.py` — inject `SearchLibrary` into engine ✅ (2026-06-11) +- [x] `[M]` Write tests: `tests/unit/test_extraction_engine.py`: ✅ (2026-06-11) - Library hit path (pattern found) - Library miss path (LLM fallback) - Pattern saved after LLM extraction @@ -174,25 +174,25 @@ #### Stage 1.4.3 — REST API Completion -- [ ] `[M]` Implement `genie/api/v1/endpoints/config.py`: +- [x] `[M]` Implement `genie/api/v1/endpoints/config.py`: ✅ (2026-06-11) - `POST /api/v1/configs` — create extraction configuration - `GET /api/v1/configs/{config_id}` — retrieve configuration - `PUT /api/v1/configs/{config_id}` — update configuration - `DELETE /api/v1/configs/{config_id}` — delete configuration -- [ ] `[M]` Implement `genie/api/v1/endpoints/library.py`: +- [x] `[M]` Implement `genie/api/v1/endpoints/library.py`: ✅ (2026-06-11) - `GET /api/v1/library/patterns` — list all patterns - `GET /api/v1/library/patterns/{layout_id}` — get pattern details - `GET /api/v1/library/stats` — library statistics -- [ ] `[S]` Implement API router aggregation in `genie/api/v1/router.py` -- [ ] `[M]` Write tests: `tests/integration/test_api.py` — config CRUD, library endpoints +- [x] `[S]` Implement API router aggregation in `genie/api/v1/router.py` ✅ (2026-06-11) +- [x] `[M]` Write tests: `tests/integration/test_api.py` — config CRUD, library endpoints ✅ (2026-06-11) #### Stage 1.4.4 — End-to-End Validation -- [ ] `[L]` Write `tests/integration/test_end_to_end.py`: +- [x] `[L]` Write `tests/integration/test_end_to_end.py`: ✅ (2026-06-11) - Full flow: create config → extract from text → verify pattern saved → re-extract same layout → verify library hit - Full flow: extract from PDF → verify fingerprint → verify pattern storage - [ ] `[M]` Manual validation: extract from 3+ different document layouts, verify Search Library grows -- [ ] `[S]` Run full test suite, verify 80%+ coverage +- [x] `[S]` Run full test suite, verify 80%+ coverage ✅ (2026-06-11) > **Quality Gate 1.4:** Complete extraction flow works (library lookup → LLM fallback → pattern save). Config CRUD and Library endpoints work. End-to-end test passes. Test coverage >= 80%. @@ -320,7 +320,7 @@ #### Stage 3.2.1 — XLSX & CSV Parsers -- [ ] `[S]` Add dependency: `openpyxl` +- [x] `[S]` Add dependency: `openpyxl` ✅ (2026-06-11) - [ ] `[M]` Implement `genie/extraction/parsers/spreadsheet.py`: - `SpreadsheetParser` — XLSX and CSV reading - Sheet selection, header detection, data type inference @@ -420,14 +420,14 @@ - [ ] `[M]` Generate `docs/api/openapi.yaml` from FastAPI auto-docs - [ ] `[M]` Add detailed endpoint descriptions, examples, and error responses -- [ ] `[S]` Verify Swagger UI works at `/docs` +- [x] `[S]` Verify Swagger UI works at `/docs` ✅ (2026-06-11) #### Stage 4.2.2 — TabEx Integration - [ ] `[L]` Create TabEx integration example using JS SDK -- [ ] `[M]` Implement `genie/api/v1/endpoints/extract.py` — file upload support (multipart) +- [x] `[M]` File upload support (multipart) — entregue como `POST /api/v1/uploads` ✅ (2026-06-11) - [ ] `[M]` Validate: TabEx JS app extracts medical reports via GENIE API -- [ ] `[S]` Document integration guide at `docs/examples/tabex_integration.md` +- [x] `[S]` Document integration guide — entregue em `docs/MANUAL.md` (seção 4, TabEx) ✅ (2026-06-11) #### Stage 4.2.3 — Load Testing @@ -453,9 +453,9 @@ - [ ] `[M]` Implement `genie/api/middleware/auth.py`: - API key validation middleware - Key generation and rotation -- [ ] `[M]` Complete `genie/core/security.py`: - - `SecureKeyStore` — encrypted API key storage (Fernet) - - `SecureFileAccess` — sandboxed file reading (allowed paths) +- [x] `[M]` Complete `genie/core/security.py`: ✅ (2026-06-11) + - `KeyVault` — encrypted API key storage (AES-256-GCM; substitui o plano original com Fernet) + - Sandboxed file reading via `ensure_path_allowed()` (allowlist de raízes) - [ ] `[S]` Write tests: auth middleware, key management #### Stage 5.1.2 — Authorization & Rate Limiting @@ -528,7 +528,7 @@ #### Stage 5.3.3 — Documentation Completion -- [ ] `[M]` Create `README.md` — project overview, quickstart, architecture diagram +- [x] `[M]` Create `README.md` — project overview, quickstart, architecture diagram ✅ (2026-06-11) - [ ] `[M]` Create `docs/guides/quickstart.md` — step-by-step setup guide - [ ] `[M]` Create `docs/guides/configuration.md` — complete configuration reference - [ ] `[M]` Create `docs/guides/deployment.md` — production deployment guide @@ -567,6 +567,29 @@ The **GenIE 10 Code Standards** must be verified at every Phase completion: --- +## Adendo — 2026-06-11: Aplicação Web GenIE (handoff de design) + +Itens entregues a partir do handoff de design (claude.ai/design), fora do roadmap +original mas alinhados às decisões de projeto 1, 2 e 5: + +- [x] SPA estática (`spec/web/`) servida pelo FastAPI — porte fiel do protótipo ✅ (2026-06-11) +- [x] Pipeline de 3 agentes: Conector (I/O), Localizador (extração LLM), Organizador (formato) — `spec/extraction/agents/` ✅ (2026-06-11) +- [x] Conectores de entrada: URL, pasta local, banco (SQLite/SQLAlchemy), API REST, upload, texto inline ✅ (2026-06-11) +- [x] Destinos de saída: webhook, pasta, SQLite, API REST (TabEx), download assinado ✅ (2026-06-11) +- [x] Cofre cifrado de API keys (AES-256-GCM) + endpoints `/api/v1/keys` ✅ (2026-06-11) +- [x] Execuções com streaming SSE (`/api/v1/runs/{id}/events`), cancelamento e replay ✅ (2026-06-11) +- [x] Multi-provider operacional: Google Gemini, OpenAI, Anthropic (modo JSON nativo) ✅ (2026-06-11) +- [x] Manual de uso completo (`docs/MANUAL.md`): standalone + plugin + integração TabEx ✅ (2026-06-11) +- [x] Suíte com 127 testes, cobertura 81% (meta: 80%) ✅ (2026-06-11) + +Reestruturações aplicadas: +- `spec/api/v1/routes/` e `spec/output/formatters/` (vazios, fora do plano) removidos +- `setup.sh` e `test-genie.py` movidos para `scripts/`; criado `scripts/test_llm_connection.py` +- Documentos de status da raiz movidos para `docs/history/` +- `data/` integralmente fora do versionamento (chave-mestre e segredos) + +--- + ## Summary | Phase | Sub-Phases | Stages | Items | Timeline | diff --git a/docs/guides/PROJECT-AUDIT.md b/docs/guides/PROJECT-AUDIT.md index 23451ad..1d8768e 100644 --- a/docs/guides/PROJECT-AUDIT.md +++ b/docs/guides/PROJECT-AUDIT.md @@ -245,3 +245,27 @@ Quando aprovado, começaremos com **Stage 1.1.1** criando: 4. Código em `spec/` (novos arquivos) Nenhum arquivo existente será sobrescrito. + + +--- + +# Atualização do Audit — 2026-06-11 + +O cenário "zero código implementado" acima é histórico. Status atual: + +- ✅ **Fase 1 (MVP Core): concluída** — config/exceptions/logging/security, modelos + Pydantic, providers LLM (Google/OpenAI/Anthropic), parsers (texto/PDF/CSV/XLSX/HTML/JSON), + fingerprint, Search Library (JSON) + matcher, ExtractionEngine com fluxo completo + (config → fingerprint → library → LLM → pattern), endpoints `/extract`, `/configs` (CRUD), + `/library` (patterns/stats), `/providers`, `/health`. +- ✅ **Aplicação web (handoff de design)** — SPA em `spec/web/` + pipeline de 3 agentes + (Conector/Localizador/Organizador) em `spec/extraction/agents/`, com `/models`, `/keys` + (cofre AES-256-GCM), `/uploads`, `/runs` (SSE) e `/downloads` (links assinados). +- ✅ **Testes:** 127 passando · cobertura 81% (meta ≥80%). +- ✅ **Docs:** `README.md`, `docs/MANUAL.md` (standalone + plugin + TabEx). +- 📁 **Reestruturação:** `routes/` e `output/formatters/` removidos; scripts em `scripts/`; + docs de status da raiz em `docs/history/`; `data/` fora do versionamento. +- ⏭️ **Próximos (Fase 2):** geração automática de REGEX (PatternGenerator), fingerprint + avançado, SQLite storage para a Search Library, correção manual de padrões via API. + +Rastreamento item a item: `docs/guides/GENIE-TODO.md` (checkboxes atualizados). diff --git a/FINAL-RELEASE-NOTES.md b/docs/history/FINAL-RELEASE-NOTES.md similarity index 100% rename from FINAL-RELEASE-NOTES.md rename to docs/history/FINAL-RELEASE-NOTES.md diff --git a/HOMOLOGATION-CHECKLIST.md b/docs/history/HOMOLOGATION-CHECKLIST.md similarity index 100% rename from HOMOLOGATION-CHECKLIST.md rename to docs/history/HOMOLOGATION-CHECKLIST.md diff --git a/IMPLEMENTATION-SUMMARY.md b/docs/history/IMPLEMENTATION-SUMMARY.md similarity index 100% rename from IMPLEMENTATION-SUMMARY.md rename to docs/history/IMPLEMENTATION-SUMMARY.md diff --git a/PHASE-1-STATUS.md b/docs/history/PHASE-1-STATUS.md similarity index 100% rename from PHASE-1-STATUS.md rename to docs/history/PHASE-1-STATUS.md diff --git a/TESTE-RAPIDO.md b/docs/history/TESTE-RAPIDO.md similarity index 100% rename from TESTE-RAPIDO.md rename to docs/history/TESTE-RAPIDO.md diff --git a/setup.sh b/scripts/setup.sh similarity index 100% rename from setup.sh rename to scripts/setup.sh diff --git a/test-genie.py b/scripts/test-genie.py similarity index 100% rename from test-genie.py rename to scripts/test-genie.py diff --git a/scripts/test_llm_connection.py b/scripts/test_llm_connection.py new file mode 100644 index 0000000..81c6ae8 --- /dev/null +++ b/scripts/test_llm_connection.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Manual LLM connectivity test (Phase 1, Stage 1.2.2). + +Usage: + python3 scripts/test_llm_connection.py [provider] + +Resolves the API key from the encrypted vault first, then from env vars +(GOOGLE_API_KEY / OPENAI_API_KEY / ANTHROPIC_API_KEY). +""" + +import asyncio +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from spec.extraction.llm.factory import LLMProviderFactory # noqa: E402 + + +async def main() -> int: + """Run a minimal extraction against the chosen provider. + + Returns: + int: Process exit code (0 = success) + """ + + provider_name = sys.argv[1] if len(sys.argv) > 1 else "google" + factory = LLMProviderFactory() + + print(f"Testing provider: {provider_name}") + try: + provider = factory.get_provider(provider_name=provider_name) + result = await provider.extract( + content="ping", + schema={"ok": True}, + instructions='Responda apenas com o JSON {"ok": true}.', + ) + except Exception as e: # noqa: BLE001 - CLI feedback + print(f"✗ FALHOU: {e}") + return 1 + + print(f"✓ OK — resposta: {result}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/spec/api/v1/dependencies.py b/spec/api/v1/dependencies.py index 8d0f7bf..bbc5f7e 100644 --- a/spec/api/v1/dependencies.py +++ b/spec/api/v1/dependencies.py @@ -9,6 +9,7 @@ from fastapi import Depends from spec.core.config import Settings, get_settings +from spec.core.config_store import ConfigStore from spec.extraction.engine import ExtractionEngine from spec.extraction.llm.factory import LLMProviderFactory from spec.output.manager import OutputManager @@ -71,10 +72,23 @@ def get_output_manager() -> OutputManager: return OutputManager() +def get_config_store(settings: Settings = Depends(get_app_settings)) -> ConfigStore: + """Get extraction configuration store. + + Args: + settings: Application settings (injected) + + Returns: + ConfigStore: File-backed configuration store + """ + return ConfigStore(config_dir=settings.config_dir) + + def get_extraction_engine( search_library: JSONStorage = Depends(get_search_library), llm_factory: LLMProviderFactory = Depends(get_llm_factory), output_manager: OutputManager = Depends(get_output_manager), + config_store: ConfigStore = Depends(get_config_store), ) -> ExtractionEngine: """Get extraction engine instance. @@ -92,4 +106,5 @@ def get_extraction_engine( search_library=search_library, llm_factory=llm_factory, output_manager=output_manager, + config_store=config_store, ) diff --git a/spec/api/v1/endpoints/config.py b/spec/api/v1/endpoints/config.py new file mode 100644 index 0000000..ff2d055 --- /dev/null +++ b/spec/api/v1/endpoints/config.py @@ -0,0 +1,148 @@ +"""CRUD endpoints for extraction configurations (Phase 1, Stage 1.4.3).""" + +import logging + +from fastapi import APIRouter, Depends, HTTPException + +from spec.api.v1.dependencies import get_config_store +from spec.core.config_store import ConfigStore +from spec.core.exceptions import InvalidConfig +from spec.models.config import ExtractionConfig + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +@router.post("", response_model=ExtractionConfig, status_code=201) +async def create_config( + config: ExtractionConfig, + store: ConfigStore = Depends(get_config_store), +) -> ExtractionConfig: + """Create an extraction configuration. + + Args: + config: Configuration document + store: Config store (injected) + + Returns: + ExtractionConfig: Stored configuration + + Raises: + HTTPException: 409 if the id already exists, 400 for invalid ids + """ + + try: + if store.get(config.extraction_id) is not None: + raise HTTPException( + status_code=409, + detail=f"Configuração '{config.extraction_id}' já existe (use PUT).", + ) + return store.save(config) + except InvalidConfig as e: + raise HTTPException(status_code=400, detail=str(e)) + + +@router.get("", response_model=list[ExtractionConfig]) +async def list_configs( + store: ConfigStore = Depends(get_config_store), +) -> list[ExtractionConfig]: + """List all extraction configurations. + + Args: + store: Config store (injected) + + Returns: + list[ExtractionConfig]: Stored configurations + """ + + return store.list() + + +@router.get("/{config_id}", response_model=ExtractionConfig) +async def get_config( + config_id: str, + store: ConfigStore = Depends(get_config_store), +) -> ExtractionConfig: + """Retrieve a configuration by id. + + Args: + config_id: Configuration identifier + store: Config store (injected) + + Returns: + ExtractionConfig: Stored configuration + + Raises: + HTTPException: 404 when absent, 400 for invalid ids + """ + + try: + config = store.get(config_id) + except InvalidConfig as e: + raise HTTPException(status_code=400, detail=str(e)) + if config is None: + raise HTTPException( + status_code=404, detail=f"Configuração '{config_id}' não encontrada" + ) + return config + + +@router.put("/{config_id}", response_model=ExtractionConfig) +async def update_config( + config_id: str, + config: ExtractionConfig, + store: ConfigStore = Depends(get_config_store), +) -> ExtractionConfig: + """Create or update a configuration. + + Args: + config_id: Configuration identifier (must match the body) + config: Configuration document + store: Config store (injected) + + Returns: + ExtractionConfig: Stored configuration + + Raises: + HTTPException: 400 when ids mismatch or are invalid + """ + + if config.extraction_id != config_id: + raise HTTPException( + status_code=400, + detail="extraction_id do corpo difere do id da URL", + ) + try: + return store.save(config) + except InvalidConfig as e: + raise HTTPException(status_code=400, detail=str(e)) + + +@router.delete("/{config_id}") +async def delete_config( + config_id: str, + store: ConfigStore = Depends(get_config_store), +) -> dict[str, bool]: + """Delete a configuration. + + Args: + config_id: Configuration identifier + store: Config store (injected) + + Returns: + dict: {"deleted": true} + + Raises: + HTTPException: 404 when absent + """ + + try: + deleted = store.delete(config_id) + except InvalidConfig as e: + raise HTTPException(status_code=400, detail=str(e)) + if not deleted: + raise HTTPException( + status_code=404, detail=f"Configuração '{config_id}' não encontrada" + ) + return {"deleted": True} diff --git a/spec/api/v1/endpoints/library.py b/spec/api/v1/endpoints/library.py new file mode 100644 index 0000000..c5a125e --- /dev/null +++ b/spec/api/v1/endpoints/library.py @@ -0,0 +1,94 @@ +"""Search Library inspection endpoints (Phase 1, Stage 1.4.3).""" + +import logging +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException + +from spec.api.v1.dependencies import get_search_library +from spec.search_library.json_storage import JSONStorage + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +@router.get("/patterns") +async def list_patterns( + config_id: Optional[str] = None, + library: JSONStorage = Depends(get_search_library), +) -> list[dict[str, Any]]: + """List stored extraction patterns. + + Args: + config_id: Optional filter by configuration id + library: Search library storage (injected) + + Returns: + list[dict]: Stored patterns + """ + + return await library.list_patterns(config_id=config_id) + + +@router.get("/patterns/{layout_id}") +async def get_pattern( + layout_id: str, + library: JSONStorage = Depends(get_search_library), +) -> dict[str, Any]: + """Get a single pattern by layout id. + + Args: + layout_id: Pattern layout identifier + library: Search library storage (injected) + + Returns: + dict: Pattern details + + Raises: + HTTPException: 404 when the pattern does not exist + """ + + patterns = await library.list_patterns() + pattern = next((p for p in patterns if p.get("layout_id") == layout_id), None) + if pattern is None: + raise HTTPException( + status_code=404, detail=f"Padrão '{layout_id}' não encontrado" + ) + return pattern + + +@router.get("/stats") +async def library_stats( + library: JSONStorage = Depends(get_search_library), +) -> dict[str, Any]: + """Aggregate statistics about the Search Library. + + Args: + library: Search library storage (injected) + + Returns: + dict: Metadata plus usage aggregates + """ + + metadata = await library.get_metadata() + patterns = await library.list_patterns() + + total_uses = sum(p.get("use_count", 0) for p in patterns) + avg_success = ( + round(sum(p.get("success_rate", 0.0) for p in patterns) / len(patterns), 3) + if patterns + else 0.0 + ) + by_config: dict[str, int] = {} + for pattern in patterns: + key = pattern.get("config_id", "?") + by_config[key] = by_config.get(key, 0) + 1 + + return { + "metadata": metadata, + "total_patterns": len(patterns), + "total_uses": total_uses, + "average_success_rate": avg_success, + "patterns_by_config": by_config, + } diff --git a/spec/api/v1/router.py b/spec/api/v1/router.py index 61166de..447b622 100644 --- a/spec/api/v1/router.py +++ b/spec/api/v1/router.py @@ -6,10 +6,12 @@ from fastapi import APIRouter +from spec.api.v1.endpoints.config import router as config_router from spec.api.v1.endpoints.downloads import router as downloads_router from spec.api.v1.endpoints.extract import router as extract_router from spec.api.v1.endpoints.health import router as health_router from spec.api.v1.endpoints.keys import router as keys_router +from spec.api.v1.endpoints.library import router as library_router from spec.api.v1.endpoints.models import router as models_router from spec.api.v1.endpoints.providers import router as providers_router from spec.api.v1.endpoints.runs import router as runs_router @@ -22,6 +24,8 @@ router.include_router(health_router, tags=["health"]) router.include_router(extract_router, tags=["extraction"]) router.include_router(providers_router, prefix="/providers", tags=["providers"]) +router.include_router(config_router, prefix="/configs", tags=["configs"]) +router.include_router(library_router, prefix="/library", tags=["library"]) router.include_router(models_router, prefix="/models", tags=["models"]) router.include_router(keys_router, prefix="/keys", tags=["keys"]) router.include_router(uploads_router, prefix="/uploads", tags=["uploads"]) diff --git a/spec/api/v1/routes/__init__.py b/spec/api/v1/routes/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/spec/core/config_store.py b/spec/core/config_store.py new file mode 100644 index 0000000..196308a --- /dev/null +++ b/spec/core/config_store.py @@ -0,0 +1,127 @@ +"""Persistent store for extraction configurations (one JSON file per config). + +Implements the configuration layer of GenIE's design decision nº 1 +("Generic, not General"): each use case is described by an +ExtractionConfig that tells the engine WHAT to extract. +""" + +import logging +import re +from pathlib import Path +from typing import List, Optional + +from spec.core.exceptions import InvalidConfig, StorageError +from spec.models.config import ExtractionConfig + +logger = logging.getLogger(__name__) + +_CONFIG_ID_RE = re.compile(r"^[A-Za-z0-9_\-]{1,64}$") + + +class ConfigStore: + """File-backed CRUD for ExtractionConfig documents.""" + + def __init__(self, config_dir: str) -> None: + """Initialize the store. + + Args: + config_dir: Directory where config JSON files live + """ + + self._dir = Path(config_dir) + self._dir.mkdir(parents=True, exist_ok=True) + + def _path_for(self, config_id: str) -> Path: + """Resolve the safe file path for a config id. + + Args: + config_id: Configuration identifier + + Returns: + Path: JSON file path + + Raises: + InvalidConfig: If the id contains unsafe characters + """ + + if not _CONFIG_ID_RE.match(config_id): + raise InvalidConfig( + f"config_id inválido: '{config_id}' (use letras, números, '-' e '_')" + ) + return self._dir / f"{config_id}.json" + + def save(self, config: ExtractionConfig) -> ExtractionConfig: + """Create or replace a configuration. + + Args: + config: Validated configuration + + Returns: + ExtractionConfig: The stored configuration + + Raises: + StorageError: If the file cannot be written + """ + + path = self._path_for(config.extraction_id) + try: + path.write_text( + config.model_dump_json(indent=2), + encoding="utf-8", + ) + except OSError as e: + raise StorageError(f"Falha ao gravar configuração: {e}") + logger.info("Saved extraction config: %s", config.extraction_id) + return config + + def get(self, config_id: str) -> Optional[ExtractionConfig]: + """Load a configuration by id. + + Args: + config_id: Configuration identifier + + Returns: + Optional[ExtractionConfig]: Config or None when absent + """ + + path = self._path_for(config_id) + if not path.is_file(): + return None + try: + return ExtractionConfig.model_validate_json(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as e: + raise StorageError(f"Configuração corrompida '{config_id}': {e}") + + def list(self) -> List[ExtractionConfig]: + """List all stored configurations. + + Returns: + list[ExtractionConfig]: Configs ordered by id + """ + + configs: List[ExtractionConfig] = [] + for path in sorted(self._dir.glob("*.json")): + try: + configs.append( + ExtractionConfig.model_validate_json(path.read_text(encoding="utf-8")) + ) + except ValueError: + logger.warning("Skipping corrupt config file: %s", path) + return configs + + def delete(self, config_id: str) -> bool: + """Delete a configuration. + + Args: + config_id: Configuration identifier + + Returns: + bool: True if a config was removed + """ + + path = self._path_for(config_id) + if not path.is_file(): + return False + path.unlink() + logger.info("Deleted extraction config: %s", config_id) + return True diff --git a/spec/extraction/engine.py b/spec/extraction/engine.py index c4aa1be..3431459 100644 --- a/spec/extraction/engine.py +++ b/spec/extraction/engine.py @@ -3,13 +3,15 @@ import logging import uuid from time import time -from typing import Any, Dict +from typing import Any, Dict, Optional +from spec.core.config_store import ConfigStore from spec.core.exceptions import ExtractionFailed, InvalidConfig from spec.extraction.layout.fingerprint import LayoutFingerprint from spec.extraction.llm.factory import LLMProviderFactory from spec.extraction.parsers.pdf import PDFParser from spec.extraction.parsers.text import TextParser +from spec.models.config import ExtractionConfig from spec.models.extraction import ExtractionRequest, ExtractionResponse from spec.output.manager import OutputManager from spec.search_library.base import BaseStorage @@ -41,6 +43,7 @@ def __init__( search_library: BaseStorage, llm_factory: LLMProviderFactory, output_manager: OutputManager, + config_store: Optional[ConfigStore] = None, ) -> None: """Initialize extraction engine. @@ -48,11 +51,13 @@ def __init__( search_library: Search library storage instance llm_factory: LLM provider factory output_manager: Output manager instance + config_store: Store of extraction configurations (optional) """ self.search_library = search_library self.llm_factory = llm_factory self.output_manager = output_manager + self.config_store = config_store self.fingerprint_generator = LayoutFingerprint() logger.debug("Initialized ExtractionEngine") @@ -78,6 +83,9 @@ async def extract(self, request: ExtractionRequest) -> ExtractionResponse: f"Starting extraction {extraction_id} with config {request.config_id}" ) + # 0. Load stored configuration (decision nº 1: generic, not general) + config = self._load_config(request.config_id) + # 1. Read content logger.debug("Step 1: Reading content from source") content = await self._read_content(request.source) @@ -88,12 +96,15 @@ async def extract(self, request: ExtractionRequest) -> ExtractionResponse: layout_fingerprint = self.fingerprint_generator.generate(content) logger.debug(f"Generated fingerprint: {layout_fingerprint}") - # 3. Search library lookup + # 3. Search library lookup (decision nº 2: library first, LLM second) logger.debug("Step 3: Searching library for matching pattern") - pattern = await self.search_library.find_pattern( - layout_fingerprint, - request.config_id, - ) + use_library = config.behavior.use_search_library if config else True + pattern = None + if use_library: + pattern = await self.search_library.find_pattern( + layout_fingerprint, + request.config_id, + ) method_used = "unknown" extracted_data = {} @@ -124,14 +135,19 @@ async def extract(self, request: ExtractionRequest) -> ExtractionResponse: extracted_data, confidence = await self._extract_with_llm( content, request.config_id, - request.source, + config, ) method_used = "llm" # 5. Auto-save pattern if configured - if request.options and request.options.get( - "auto_create_patterns", True - ): + auto_create = ( + config.behavior.auto_create_patterns if config else True + ) + if request.options is not None: + auto_create = request.options.get( + "auto_create_patterns", auto_create + ) + if auto_create: logger.debug("Step 5: Auto-saving pattern") try: new_pattern = self._generate_pattern_from_extraction( @@ -219,18 +235,40 @@ async def _read_content(self, source: Dict[str, Any]) -> str: else: raise InvalidConfig(f"Unsupported source type: {source_type}") + def _load_config(self, config_id: str) -> Optional[ExtractionConfig]: + """Load the stored configuration for this extraction, if any. + + Args: + config_id: Configuration identifier + + Returns: + Optional[ExtractionConfig]: Config or None (unknown id keeps the + engine backwards-compatible with ad-hoc extractions) + """ + + if self.config_store is None: + return None + try: + config = self.config_store.get(config_id) + except Exception as e: # noqa: BLE001 - config errors must not abort + logger.warning(f"Could not load config '{config_id}': {e}") + return None + if config is None: + logger.debug(f"No stored config for '{config_id}', using defaults") + return config + async def _extract_with_llm( self, content: str, config_id: str, - source: Dict[str, Any], + config: Optional[ExtractionConfig] = None, ) -> tuple[Dict[str, Any], float]: - """Extract data using LLM. + """Extract data using LLM, honoring the stored configuration. Args: content: Document content config_id: Configuration ID - source: Source specification + config: Stored configuration (instructions, schema, provider) Returns: tuple: (extracted_data dict, confidence score) @@ -240,16 +278,24 @@ async def _extract_with_llm( """ try: - llm_provider = self.llm_factory.get_default_provider() - - # Basic schema for Phase 1 - schema = {"fields": {}} + if config is not None: + llm_provider = self.llm_factory.get_provider( + provider_name=config.llm.provider, + model=config.llm.model, + ) + schema = {"fields": config.output.schema or {}} + instructions = config.extraction_instructions + else: + llm_provider = self.llm_factory.get_default_provider() + schema = {"fields": {}} + instructions = ( + f"Extract structured data from this document for config {config_id}" + ) - # Call LLM extracted_data = await llm_provider.extract( content=content, schema=schema, - instructions=f"Extract structured data from this document for config {config_id}", + instructions=instructions, ) confidence = 0.90 diff --git a/spec/models/config.py b/spec/models/config.py index 7db0475..f1532ce 100644 --- a/spec/models/config.py +++ b/spec/models/config.py @@ -48,7 +48,7 @@ class LLMConfig(BaseModel): """ provider: str = Field("anthropic", description="LLM provider") - model: str = Field("claude-sonnet-4-20250514", description="Model ID") + model: str = Field("claude-sonnet-4-6", description="Model ID") api_key_ref: Optional[str] = Field(None, description="API key reference") fallback_to_ocr: bool = Field(False, description="Fallback to OCR") temperature: float = Field(0.0, ge=0.0, le=1.0, description="Temperature") @@ -111,7 +111,7 @@ class Config: }, "llm": { "provider": "anthropic", - "model": "claude-sonnet-4-20250514", + "model": "claude-sonnet-4-6", "temperature": 0.0, "max_tokens": 4096, }, diff --git a/spec/output/formatters/__init__.py b/spec/output/formatters/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/conftest.py b/tests/conftest.py index b5b366f..f74c16d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,12 @@ """Pytest configuration and fixtures for GENIE tests.""" -import pytest import asyncio -from pathlib import Path + +import pytest from fastapi.testclient import TestClient -from spec.main import app from spec.core.config import Settings +from spec.main import app @pytest.fixture(scope="session") diff --git a/tests/integration/test_config_library_api.py b/tests/integration/test_config_library_api.py new file mode 100644 index 0000000..2c3ffa9 --- /dev/null +++ b/tests/integration/test_config_library_api.py @@ -0,0 +1,126 @@ +"""Tests for config CRUD and Search Library endpoints (Phase 1, Stage 1.4.3).""" + +import httpx +import pytest + +from spec.core.config import get_settings +from spec.main import app + + +@pytest.fixture +def isolated_dirs(tmp_path, monkeypatch): + """Point config and library storage at a temp folder.""" + + settings = get_settings() + monkeypatch.setattr(settings, "config_dir", str(tmp_path / "configs")) + monkeypatch.setattr( + settings, "search_library_path", str(tmp_path / "patterns.json") + ) + return settings + + +@pytest.fixture +async def client(isolated_dirs): + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as c: + yield c + + +def _sample_config(config_id: str = "medical_reports_v1") -> dict: + return { + "extraction_id": config_id, + "name": "Exames médicos", + "input": {"type": "pdf", "source": "/uploads", "access_mode": "local_secure"}, + "output": {"type": "json", "auto_adapt": True}, + "llm": {"provider": "google", "model": "gemini-2.5-flash"}, + "behavior": { + "use_search_library": True, + "auto_create_patterns": True, + "layout_independent": True, + "update_on_change": True, + }, + "extraction_instructions": "Extraia data, exame, resultado e referência.", + } + + +class TestConfigCrud: + async def test_create_get_update_delete(self, client): + created = await client.post("/api/v1/configs", json=_sample_config()) + assert created.status_code == 201 + + fetched = await client.get("/api/v1/configs/medical_reports_v1") + assert fetched.status_code == 200 + assert fetched.json()["name"] == "Exames médicos" + + listed = await client.get("/api/v1/configs") + assert [c["extraction_id"] for c in listed.json()] == ["medical_reports_v1"] + + updated_body = _sample_config() + updated_body["name"] = "Exames v2" + updated = await client.put( + "/api/v1/configs/medical_reports_v1", json=updated_body + ) + assert updated.status_code == 200 + assert updated.json()["name"] == "Exames v2" + + deleted = await client.delete("/api/v1/configs/medical_reports_v1") + assert deleted.status_code == 200 + assert (await client.get("/api/v1/configs/medical_reports_v1")).status_code == 404 + + async def test_duplicate_create_conflicts(self, client): + assert ( + await client.post("/api/v1/configs", json=_sample_config()) + ).status_code == 201 + assert ( + await client.post("/api/v1/configs", json=_sample_config()) + ).status_code == 409 + + async def test_id_mismatch_rejected(self, client): + response = await client.put( + "/api/v1/configs/outro_id", json=_sample_config("medical_reports_v1") + ) + assert response.status_code == 400 + + async def test_unsafe_id_rejected(self, client): + response = await client.get("/api/v1/configs/..%2F..%2Fetc") + assert response.status_code in (400, 404) + + +class TestLibraryEndpoints: + async def test_empty_library(self, client): + patterns = await client.get("/api/v1/library/patterns") + assert patterns.status_code == 200 + assert patterns.json() == [] + + stats = await client.get("/api/v1/library/stats") + assert stats.status_code == 200 + assert stats.json()["total_patterns"] == 0 + + async def test_patterns_and_stats_after_save(self, client, isolated_dirs): + from spec.search_library.json_storage import JSONStorage + + storage = JSONStorage(storage_path=isolated_dirs.search_library_path) + await storage.save_pattern( + "fp-123", "medical_reports_v1", {"fields": [{"field_name": "exame"}]} + ) + + patterns = (await client.get("/api/v1/library/patterns")).json() + assert len(patterns) == 1 + layout_id = patterns[0]["layout_id"] + + single = await client.get(f"/api/v1/library/patterns/{layout_id}") + assert single.status_code == 200 + assert single.json()["config_id"] == "medical_reports_v1" + + missing = await client.get("/api/v1/library/patterns/layout_nao_existe") + assert missing.status_code == 404 + + stats = (await client.get("/api/v1/library/stats")).json() + assert stats["total_patterns"] == 1 + assert stats["patterns_by_config"] == {"medical_reports_v1": 1} + + filtered = await client.get( + "/api/v1/library/patterns", params={"config_id": "outro"} + ) + assert filtered.json() == [] diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py new file mode 100644 index 0000000..a7d368b --- /dev/null +++ b/tests/integration/test_end_to_end.py @@ -0,0 +1,178 @@ +"""End-to-end extraction flow tests (Phase 1, Stage 1.4.4). + +Flow under test: stored config → extract text via (mocked) LLM → +pattern auto-saved → re-extract same layout → Search Library consulted +first (decision nº 2). With a hand-crafted regex pattern, the library +path extracts with zero LLM cost. +""" + +from typing import Any, Dict +from unittest.mock import AsyncMock + +import pytest + +from spec.core.config_store import ConfigStore +from spec.extraction.engine import ExtractionEngine +from spec.extraction.llm.factory import LLMProviderFactory +from spec.models.config import ExtractionConfig +from spec.models.extraction import ExtractionRequest +from spec.output.manager import OutputManager +from spec.search_library.json_storage import JSONStorage + +_DOCUMENT = """Laudo de exame +Paciente: Maria Silva +Exame: Glicose +Resultado: 118 mg/dL +""" + + +def _config(config_id: str = "exames_v1") -> ExtractionConfig: + return ExtractionConfig.model_validate( + { + "extraction_id": config_id, + "name": "Exames", + "input": {"type": "text"}, + "output": {"type": "json", "schema": {"exame": "string", "resultado": "string"}}, + "llm": {"provider": "google", "model": "gemini-2.5-flash"}, + "behavior": { + "use_search_library": True, + "auto_create_patterns": True, + "layout_independent": True, + "update_on_change": True, + }, + "extraction_instructions": "Extraia exame e resultado.", + } + ) + + +def _engine(tmp_path, llm_result: Dict[str, Any]) -> tuple[ExtractionEngine, AsyncMock]: + store = ConfigStore(config_dir=str(tmp_path / "configs")) + store.save(_config()) + + fake_provider = AsyncMock() + fake_provider.extract = AsyncMock(return_value=llm_result) + factory = LLMProviderFactory() + factory.get_provider = lambda **kwargs: fake_provider # type: ignore[method-assign] + + engine = ExtractionEngine( + search_library=JSONStorage(storage_path=str(tmp_path / "patterns.json")), + llm_factory=factory, + output_manager=OutputManager(), + config_store=store, + ) + return engine, fake_provider + + +class TestEndToEndFlow: + @pytest.mark.asyncio + async def test_llm_extraction_uses_config_and_saves_pattern(self, tmp_path): + engine, provider = _engine( + tmp_path, {"exame": "Glicose", "resultado": "118 mg/dL"} + ) + + response = await engine.extract( + ExtractionRequest( + config_id="exames_v1", + source={"type": "text", "content": _DOCUMENT}, + ) + ) + + assert response.status == "success" + assert response.method_used == "llm" + assert response.data == {"exame": "Glicose", "resultado": "118 mg/dL"} + assert response.layout_fingerprint + + # Config-driven prompt: instructions and schema came from the store. + kwargs = provider.extract.call_args.kwargs + assert kwargs["instructions"] == "Extraia exame e resultado." + assert kwargs["schema"] == {"fields": {"exame": "string", "resultado": "string"}} + + # Pattern was auto-saved, indexed by the fingerprint. + patterns = await engine.search_library.list_patterns(config_id="exames_v1") + assert len(patterns) == 1 + assert patterns[0]["fingerprint"] == response.layout_fingerprint + + @pytest.mark.asyncio + async def test_library_hit_extracts_without_llm(self, tmp_path): + engine, provider = _engine(tmp_path, {"never": "called"}) + + # Seed the library with a working regex pattern for this layout. + fingerprint = engine.fingerprint_generator.generate(_DOCUMENT) + await engine.search_library.save_pattern( + fingerprint, + "exames_v1", + { + "fields": [ + { + "field_name": "exame", + "extraction_method": "regex", + "pattern": r"Exame:\s*([^\n]+)", + }, + { + "field_name": "resultado", + "extraction_method": "regex", + "pattern": r"Resultado:\s*([^\n]+)", + }, + ] + }, + ) + + response = await engine.extract( + ExtractionRequest( + config_id="exames_v1", + source={"type": "text", "content": _DOCUMENT}, + ) + ) + + assert response.status == "success" + assert response.method_used == "search_library" + assert response.data["exame"] == "Glicose" + assert response.data["resultado"] == "118 mg/dL" + provider.extract.assert_not_called() + + @pytest.mark.asyncio + async def test_force_llm_bypasses_library(self, tmp_path): + engine, provider = _engine(tmp_path, {"exame": "Glicose"}) + fingerprint = engine.fingerprint_generator.generate(_DOCUMENT) + await engine.search_library.save_pattern( + fingerprint, + "exames_v1", + { + "fields": [ + { + "field_name": "exame", + "extraction_method": "regex", + "pattern": r"Exame:\s*([^\n]+)", + } + ] + }, + ) + + response = await engine.extract( + ExtractionRequest( + config_id="exames_v1", + source={"type": "text", "content": _DOCUMENT}, + force_llm=True, + ) + ) + + assert response.method_used == "llm" + provider.extract.assert_called_once() + + @pytest.mark.asyncio + async def test_unknown_config_falls_back_to_defaults(self, tmp_path): + engine, provider = _engine(tmp_path, {"campo": "valor"}) + + response = await engine.extract( + ExtractionRequest( + config_id="config_inexistente", + source={"type": "text", "content": _DOCUMENT}, + options={"auto_create_patterns": False}, + ) + ) + + assert response.status == "success" + assert response.method_used == "llm" + # No pattern saved when auto_create_patterns is off. + patterns = await engine.search_library.list_patterns() + assert patterns == [] diff --git a/tests/integration/test_health.py b/tests/integration/test_health.py index 42359b8..fb6aba7 100644 --- a/tests/integration/test_health.py +++ b/tests/integration/test_health.py @@ -1,7 +1,7 @@ """Tests for health check endpoint.""" -import pytest import httpx +import pytest from spec.main import app diff --git a/tests/integration/test_providers_api.py b/tests/integration/test_providers_api.py new file mode 100644 index 0000000..1f3403f --- /dev/null +++ b/tests/integration/test_providers_api.py @@ -0,0 +1,91 @@ +"""Tests for the legacy provider management endpoints.""" + +from unittest.mock import AsyncMock + +import httpx +import pytest + +from spec.extraction.llm.factory import LLMProviderFactory +from spec.main import app + + +@pytest.fixture +async def client(): + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as c: + yield c + + +class TestProvidersEndpoints: + async def test_list_providers(self, client): + response = await client.get("/api/v1/providers") + + assert response.status_code == 200 + providers = response.json() + assert {p["name"] for p in providers} == {"google", "openai", "anthropic"} + assert sum(1 for p in providers if p["is_active"]) == 1 + + async def test_active_provider(self, client): + response = await client.get("/api/v1/providers/active") + + assert response.status_code == 200 + assert response.json()["is_active"] is True + + async def test_configure_unknown_provider(self, client): + response = await client.post( + "/api/v1/providers/configure", + json={"provider": "skynet", "api_key": "x"}, + ) + + assert response.status_code == 400 + + async def test_configure_validates_and_persists(self, client, tmp_path, monkeypatch): + from spec.core.config import get_settings + from spec.core.security import reset_security_singletons + + settings = get_settings() + monkeypatch.setattr(settings, "data_dir", str(tmp_path)) + monkeypatch.setattr(settings, "db_path", str(tmp_path / "genie.db")) + monkeypatch.setattr(settings, "master_key", None) + reset_security_singletons() + + fake_provider = AsyncMock() + fake_provider.extract = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr( + LLMProviderFactory, + "get_provider", + lambda self, **kwargs: fake_provider, + ) + + app.dependency_overrides.clear() + try: + response = await client.post( + "/api/v1/providers/configure", + json={"provider": "google", "api_key": "AIzaSyTest123", "model": None}, + ) + finally: + reset_security_singletons() + + assert response.status_code == 200 + body = response.json() + assert body["success"] is True + assert "AIzaSyTest123" not in str(body) + + async def test_configure_rejected_when_validation_fails(self, client, monkeypatch): + from spec.core.exceptions import LLMProviderError + + fake_provider = AsyncMock() + fake_provider.extract = AsyncMock(side_effect=LLMProviderError("chave inválida")) + monkeypatch.setattr( + LLMProviderFactory, + "get_provider", + lambda self, **kwargs: fake_provider, + ) + + response = await client.post( + "/api/v1/providers/configure", + json={"provider": "openai", "api_key": "sk-bad"}, + ) + + assert response.status_code == 400 diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 0c491e3..fd0b955 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1,6 +1,5 @@ """Tests for configuration management.""" -import pytest from pathlib import Path from spec.core.config import Settings, get_settings diff --git a/tests/unit/test_connector.py b/tests/unit/test_connector.py index ab3aae7..afe7d02 100644 --- a/tests/unit/test_connector.py +++ b/tests/unit/test_connector.py @@ -148,3 +148,168 @@ async def test_writes_json_and_csv(self, tmp_path, monkeypatch) -> None: assert sorted(receipt["artifacts"]) == ["output.csv", "output.json"] saved = json.loads((tmp_path / "genie-abc" / "output.json").read_text()) assert saved == records + + +class TestSqliteInput: + @pytest.mark.asyncio + async def test_reads_tables_as_items(self, tmp_path, monkeypatch) -> None: + from spec.core.config import get_settings + + monkeypatch.setattr(get_settings(), "allowed_fs_roots", str(tmp_path)) + db_file = tmp_path / "origem.db" + conn = sqlite3.connect(db_file) + conn.execute("CREATE TABLE exames (nome TEXT, valor TEXT)") + conn.execute("INSERT INTO exames VALUES ('Glicose', '118')") + conn.commit() + conn.close() + + connector = ConnectorAgent() + items = await connector.open_input( + InputSpec(type="db", target=f"sqlite:///{db_file}"), noop_emit + ) + + assert len(items) == 1 + assert "Glicose" in items[0]["content"] + + @pytest.mark.asyncio + async def test_custom_query(self, tmp_path, monkeypatch) -> None: + from spec.core.config import get_settings + + monkeypatch.setattr(get_settings(), "allowed_fs_roots", str(tmp_path)) + db_file = tmp_path / "origem.db" + conn = sqlite3.connect(db_file) + conn.execute("CREATE TABLE t (a INT)") + conn.executemany("INSERT INTO t VALUES (?)", [(1,), (2,)]) + conn.commit() + conn.close() + + connector = ConnectorAgent() + items = await connector.open_input( + InputSpec(type="db", target=str(db_file), query="SELECT a FROM t WHERE a > 1"), + noop_emit, + ) + + assert len(items) == 1 + assert '"a": 2' in items[0]["content"] + + @pytest.mark.asyncio + async def test_missing_db_rejected(self, tmp_path, monkeypatch) -> None: + from spec.core.config import get_settings + + monkeypatch.setattr(get_settings(), "allowed_fs_roots", str(tmp_path)) + connector = ConnectorAgent() + + with pytest.raises(InvalidConfig): + await connector.open_input( + InputSpec(type="db", target=str(tmp_path / "nao_existe.db")), noop_emit + ) + + +class TestTextInput: + @pytest.mark.asyncio + async def test_inline_text_item(self) -> None: + connector = ConnectorAgent() + items = await connector.open_input( + InputSpec(type="text", content="Glicose: 118", name="ocr.txt"), noop_emit + ) + + assert items == [{"id": "text-1", "name": "ocr.txt", "content": "Glicose: 118"}] + + @pytest.mark.asyncio + async def test_blank_content_rejected(self) -> None: + connector = ConnectorAgent() + + with pytest.raises(InvalidConfig): + await connector.open_input(InputSpec(type="text", content=" "), noop_emit) + + +class TestHttpDelivery: + class _FakeResponse: + def __init__(self, status_code: int = 200) -> None: + self.status_code = status_code + + class _FakeClient: + def __init__(self, *args, **kwargs) -> None: + self.calls = [] + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return False + + async def request(self, method, url, json=None, headers=None): + TestHttpDelivery.last_call = { + "method": method, "url": url, "json": json, "headers": headers, + } + return TestHttpDelivery._FakeResponse(TestHttpDelivery.status) + + status = 200 + last_call: dict = {} + + @pytest.mark.asyncio + async def test_batch_post_with_bearer(self, monkeypatch) -> None: + import spec.extraction.agents.connector as connector_module + + monkeypatch.setattr(connector_module.httpx, "AsyncClient", self._FakeClient) + TestHttpDelivery.status = 200 + + connector = ConnectorAgent() + receipt = await connector.deliver( + OutputSpec(type="api", target="https://tabex.test/v2/exames", token="TBX_1"), + [{"exame": "Glicose"}], + {}, + "genie-test", + noop_emit, + ) + + assert receipt == {"mode": "api", "calls": 1, "status": [200]} + assert TestHttpDelivery.last_call["headers"]["Authorization"] == "Bearer TBX_1" + assert TestHttpDelivery.last_call["json"] == [{"exame": "Glicose"}] + + @pytest.mark.asyncio + async def test_error_status_raises(self, monkeypatch) -> None: + import spec.extraction.agents.connector as connector_module + + monkeypatch.setattr(connector_module.httpx, "AsyncClient", self._FakeClient) + TestHttpDelivery.status = 500 + + connector = ConnectorAgent() + with pytest.raises(ExtractionFailed): + await connector.deliver( + OutputSpec(type="url", target="https://hooks.test/x"), + [{"a": 1}], + {}, + "genie-test", + noop_emit, + ) + + @pytest.mark.asyncio + async def test_invalid_scheme_rejected(self) -> None: + connector = ConnectorAgent() + with pytest.raises(InvalidConfig): + await connector.deliver( + OutputSpec(type="api", target="ftp://x"), [{}], {}, "j", noop_emit + ) + + +class TestXlsxParsing: + def test_xlsx_roundtrip(self) -> None: + import io + + from openpyxl import Workbook + + from spec.extraction.parsers.content import xlsx_bytes_to_text + + workbook = Workbook() + sheet = workbook.active + sheet.title = "Exames" + sheet.append(["exame", "resultado"]) + sheet.append(["Glicose", 118]) + buffer = io.BytesIO() + workbook.save(buffer) + + text = xlsx_bytes_to_text(buffer.getvalue()) + + assert "# Planilha: Exames" in text + assert "Glicose\t118" in text diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index a2b948a..4996ff5 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -1,13 +1,12 @@ """Tests for custom exceptions.""" -import pytest from spec.core.exceptions import ( + ExtractionFailed, GenieException, + InvalidConfig, LayoutNotRecognized, - ExtractionFailed, LLMProviderError, - InvalidConfig, StorageError, ) diff --git a/tests/unit/test_fingerprint.py b/tests/unit/test_fingerprint.py index 27bbc2a..14009e5 100644 --- a/tests/unit/test_fingerprint.py +++ b/tests/unit/test_fingerprint.py @@ -1,6 +1,5 @@ """Tests for layout fingerprinting.""" -import pytest from spec.extraction.layout.fingerprint import LayoutFingerprint diff --git a/tests/unit/test_llm_providers.py b/tests/unit/test_llm_providers.py new file mode 100644 index 0000000..d95e8b1 --- /dev/null +++ b/tests/unit/test_llm_providers.py @@ -0,0 +1,170 @@ +"""Tests for LLM providers with mocked API clients (Phase 1, Stage 1.2.2).""" + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from spec.core.exceptions import InvalidConfig, LLMProviderError +from spec.extraction.llm.anthropic import AnthropicProvider +from spec.extraction.llm.factory import LLMProviderFactory +from spec.extraction.llm.google import GoogleProvider +from spec.extraction.llm.openai import OpenAIProvider + +_SCHEMA = {"fields": {"exame": "string", "resultado": "string"}} +_PAYLOAD = {"exame": "Glicose", "resultado": "118 mg/dL"} + + +class TestPromptBuilding: + def test_prompt_contains_content_schema_and_instructions(self) -> None: + provider = AnthropicProvider(api_key="sk-ant-test") + prompt = provider._build_prompt("DOCUMENTO XYZ", _SCHEMA, "extraia exames") + + assert "DOCUMENTO XYZ" in prompt + assert '"exame"' in prompt + assert "extraia exames" in prompt + + def test_prompt_has_default_instructions(self) -> None: + provider = OpenAIProvider(api_key="sk-test") + prompt = provider._build_prompt("doc", _SCHEMA) + + assert "Extract all fields" in prompt + + +class TestResponseParsing: + def test_anthropic_parses_markdown_wrapped_json(self) -> None: + provider = AnthropicProvider(api_key="sk-ant-test") + response = SimpleNamespace( + content=[SimpleNamespace(text=f"```json\n{json.dumps(_PAYLOAD)}\n```")] + ) + + assert provider._parse_response(response) == _PAYLOAD + + def test_openai_parses_plain_json(self) -> None: + provider = OpenAIProvider(api_key="sk-test") + response = SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content=json.dumps(_PAYLOAD)))] + ) + + assert provider._parse_response(response) == _PAYLOAD + + def test_google_parses_code_fenced_json(self) -> None: + provider = GoogleProvider(api_key="AIza-test") + response = SimpleNamespace(text=f"```\n{json.dumps(_PAYLOAD)}\n```") + + assert provider._parse_response(response) == _PAYLOAD + + def test_invalid_json_raises_provider_error(self) -> None: + provider = GoogleProvider(api_key="AIza-test") + response = SimpleNamespace(text="isto não é JSON") + + with pytest.raises(LLMProviderError): + provider._parse_response(response) + + +class TestExtractWithMockedClients: + @pytest.mark.asyncio + async def test_anthropic_extract(self) -> None: + provider = AnthropicProvider(api_key="sk-ant-test") + provider.client = SimpleNamespace( + messages=SimpleNamespace( + create=AsyncMock( + return_value=SimpleNamespace( + content=[SimpleNamespace(text=json.dumps(_PAYLOAD))] + ) + ) + ) + ) + + result = await provider.extract("doc", _SCHEMA, "extraia") + + assert result == _PAYLOAD + kwargs = provider.client.messages.create.call_args.kwargs + assert kwargs["model"] == provider.model + assert "doc" in kwargs["messages"][0]["content"] + + @pytest.mark.asyncio + async def test_openai_extract_uses_json_mode(self) -> None: + provider = OpenAIProvider(api_key="sk-test") + provider.client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=AsyncMock( + return_value=SimpleNamespace( + choices=[ + SimpleNamespace( + message=SimpleNamespace(content=json.dumps(_PAYLOAD)) + ) + ] + ) + ) + ) + ) + ) + + result = await provider.extract("doc", _SCHEMA) + + assert result == _PAYLOAD + kwargs = provider.client.chat.completions.create.call_args.kwargs + assert kwargs["response_format"] == {"type": "json_object"} + + @pytest.mark.asyncio + async def test_google_extract(self) -> None: + provider = GoogleProvider(api_key="AIza-test") + provider.client = SimpleNamespace( + aio=SimpleNamespace( + models=SimpleNamespace( + generate_content=AsyncMock( + return_value=SimpleNamespace(text=json.dumps(_PAYLOAD)) + ) + ) + ) + ) + + result = await provider.extract("doc", _SCHEMA) + + assert result == _PAYLOAD + + @pytest.mark.asyncio + async def test_api_error_wrapped_as_provider_error(self) -> None: + provider = AnthropicProvider(api_key="sk-ant-test") + provider.client = SimpleNamespace( + messages=SimpleNamespace(create=AsyncMock(side_effect=RuntimeError("boom"))) + ) + + with pytest.raises(LLMProviderError, match="boom"): + await provider.extract("doc", _SCHEMA) + + def test_missing_key_rejected(self) -> None: + with pytest.raises(LLMProviderError): + AnthropicProvider(api_key="") + with pytest.raises(LLMProviderError): + OpenAIProvider(api_key="") + with pytest.raises(LLMProviderError): + GoogleProvider(api_key="") + + +class TestFactory: + def test_unknown_provider_rejected(self) -> None: + with pytest.raises(InvalidConfig): + LLMProviderFactory().get_provider("skynet", api_key="x") + + def test_provider_instances_are_cached(self) -> None: + factory = LLMProviderFactory() + a = factory.get_provider("openai", api_key="sk-test") + b = factory.get_provider("openai", api_key="sk-test") + + assert a is b + + def test_cache_key_never_contains_key_material(self) -> None: + factory = LLMProviderFactory() + factory.get_provider("openai", api_key="sk-supersecret123") + + assert all("supersecret" not in k for k in factory._providers) + + def test_defaults_per_provider(self) -> None: + factory = LLMProviderFactory() + provider = factory.get_provider("google", api_key="AIza-test") + + assert provider.model == "gemini-2.5-flash" diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index 321c5a5..11ed917 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -1,17 +1,15 @@ """Tests for Pydantic models.""" -import pytest from datetime import datetime -from spec.models.extraction import ExtractionRequest, ExtractionResponse +import pytest + from spec.models.config import ( InputConfig, - OutputConfig, LLMConfig, - BehaviorConfig, - ExtractionConfig, ) -from spec.models.library import PatternField, SearchPattern, LibraryMetadata +from spec.models.extraction import ExtractionRequest, ExtractionResponse +from spec.models.library import PatternField, SearchPattern from spec.models.output import FieldDefinition, OutputSchema diff --git a/tests/unit/test_parsers.py b/tests/unit/test_parsers.py index 914a33f..84758bb 100644 --- a/tests/unit/test_parsers.py +++ b/tests/unit/test_parsers.py @@ -1,10 +1,10 @@ """Tests for content parsers.""" -import pytest -import asyncio from pathlib import Path -from spec.core.exceptions import InvalidConfig, ExtractionFailed +import pytest + +from spec.core.exceptions import InvalidConfig from spec.extraction.parsers.text import TextParser diff --git a/tests/unit/test_search_library.py b/tests/unit/test_search_library.py new file mode 100644 index 0000000..a130ef9 --- /dev/null +++ b/tests/unit/test_search_library.py @@ -0,0 +1,126 @@ +"""Tests for PatternMatcher and JSONStorage behaviors (Phase 1, Stage 1.4.1).""" + +import pytest + +from spec.search_library.json_storage import JSONStorage +from spec.search_library.matcher import PatternMatcher + +_CONTENT = """Paciente: Maria Silva +Exame: Glicose +Resultado: 118 mg/dL +""" + +_PATTERN = { + "fields": [ + { + "field_name": "exame", + "extraction_method": "regex", + "pattern": r"Exame:\s*([^\n]+)", + "validation": r".{2,}", + }, + { + "field_name": "resultado", + "extraction_method": "regex", + "pattern": r"Resultado:\s*([^\n]+)", + }, + ] +} + + +class TestPatternMatcher: + @pytest.mark.asyncio + async def test_extracts_fields_with_regex(self) -> None: + data = await PatternMatcher.extract_with_pattern(_CONTENT, _PATTERN) + + assert data == {"exame": "Glicose", "resultado": "118 mg/dL"} + + @pytest.mark.asyncio + async def test_missing_match_yields_none(self) -> None: + data = await PatternMatcher.extract_with_pattern("sem nada aqui", _PATTERN) + + assert data == {"exame": None, "resultado": None} + + @pytest.mark.asyncio + async def test_empty_or_invalid_patterns_are_safe(self) -> None: + pattern = { + "fields": [ + {"field_name": "vazio", "extraction_method": "regex", "pattern": ""}, + {"field_name": "quebrado", "extraction_method": "regex", "pattern": "(["}, + {"field_name": "futuro", "extraction_method": "instruction"}, + {"field_name": "estranho", "extraction_method": "telepatia"}, + ] + } + + data = await PatternMatcher.extract_with_pattern(_CONTENT, pattern) + + assert data == { + "vazio": None, + "quebrado": None, + "futuro": None, + "estranho": None, + } + + @pytest.mark.asyncio + async def test_validation_passes_and_fails(self) -> None: + ok = await PatternMatcher.validate_extraction( + {"exame": "Glicose", "resultado": "118 mg/dL"}, _PATTERN + ) + assert ok is True + + bad = await PatternMatcher.validate_extraction({"exame": "X"}, _PATTERN) + assert bad is False # validation r".{2,}" rejects 1-char value + + @pytest.mark.asyncio + async def test_validation_skips_none_values(self) -> None: + ok = await PatternMatcher.validate_extraction( + {"exame": None, "resultado": "118"}, _PATTERN + ) + assert ok is True + + +class TestJsonStorage: + @pytest.fixture + def storage(self, tmp_path) -> JSONStorage: + return JSONStorage(storage_path=str(tmp_path / "patterns.json")) + + @pytest.mark.asyncio + async def test_find_increments_use_count(self, storage: JSONStorage) -> None: + await storage.save_pattern("fp-1", "cfg", {"fields": []}) + + first = await storage.find_pattern("fp-1", "cfg") + assert first is not None + count_after_first = first["use_count"] + + second = await storage.find_pattern("fp-1", "cfg") + assert second["use_count"] == count_after_first + 1 + assert await storage.find_pattern("fp-2", "cfg") is None + assert await storage.find_pattern("fp-1", "outra_cfg") is None + + @pytest.mark.asyncio + async def test_success_rate_moving_average(self, storage: JSONStorage) -> None: + await storage.save_pattern("fp-1", "cfg", {"fields": []}) + await storage.find_pattern("fp-1", "cfg") # use_count -> 2 + + await storage.update_success_rate("fp-1", success=False) + + patterns = await storage.list_patterns() + assert patterns[0]["success_rate"] < 1.0 + + @pytest.mark.asyncio + async def test_metadata_tracks_totals(self, storage: JSONStorage) -> None: + await storage.save_pattern("fp-1", "cfg-a", {"fields": []}) + await storage.save_pattern("fp-2", "cfg-b", {"fields": []}) + + metadata = await storage.get_metadata() + assert metadata["total_patterns"] == 2 + + only_a = await storage.list_patterns(config_id="cfg-a") + assert len(only_a) == 1 + + @pytest.mark.asyncio + async def test_persistence_across_instances(self, storage: JSONStorage, tmp_path) -> None: + await storage.save_pattern("fp-1", "cfg", {"fields": []}) + + reopened = JSONStorage(storage_path=str(tmp_path / "patterns.json")) + patterns = await reopened.list_patterns() + assert len(patterns) == 1