diff --git a/.env.template b/.env.template index 58601f4..65c7db6 100644 --- a/.env.template +++ b/.env.template @@ -15,8 +15,6 @@ AZURE_LOCATION=eastus2 COSMOS_DB_ENDPOINT= COSMOS_DB_NAME=contentflow STORAGE_ACCOUNT_NAME= -STORAGE_ACCOUNT_WORKER_QUEUE_URL= -STORAGE_WORKER_QUEUE_NAME=contentflow-execution-requests BLOB_STORAGE_CONTAINER_NAME=content APP_CONFIG_ENDPOINT= APPLICATIONINSIGHTS_CONNECTION_STRING= @@ -33,26 +31,6 @@ API_SERVER_PORT=8090 DEBUG=false LOG_LEVEL=INFO -# ============================================== -# Worker Configuration -# ============================================== -NUM_PROCESSING_WORKERS=4 -NUM_SOURCE_WORKERS=2 -WORKER_NAME=contentflow-worker -LOG_LEVEL=INFO - -# Queue Settings -QUEUE_POLL_INTERVAL_SECONDS=5 -QUEUE_VISIBILITY_TIMEOUT_SECONDS=300 -QUEUE_MAX_MESSAGES=32 -DEFAULT_POLLING_INTERVAL_SECONDS=300 -SCHEDULER_SLEEP_INTERVAL_SECONDS=5 -LOCK_TTL_SECONDS=300 - -# Processing Settings -MAX_TASK_RETRIES=3 -TASK_TIMEOUT_SECONDS=600 - # ============================================== # Web Configuration # ============================================== diff --git a/.gitignore b/.gitignore index 6e1d706..28df913 100644 --- a/.gitignore +++ b/.gitignore @@ -139,6 +139,7 @@ celerybeat.pid .venv env/ venv/ +temp-venv/ ENV/ env.bak/ venv.bak/ diff --git a/azure.yaml b/azure.yaml index 6f82a38..bb9e689 100644 --- a/azure.yaml +++ b/azure.yaml @@ -22,18 +22,7 @@ services: path: ./Dockerfile context: .. target: "" - remoteBuild: true - - # ContentFlow Worker - Container App - worker: - project: ./contentflow-worker - language: python - host: containerapp - docker: - path: ./Dockerfile - context: .. - target: "" - remoteBuild: true + remoteBuild: false # ContentFlow Web - Container App web: @@ -44,23 +33,23 @@ services: path: ./Dockerfile context: .. target: "" - remoteBuild: true + remoteBuild: false # Global hooks -# hooks: -# # Run before provisioning infrastructure -# preprovision: -# shell: sh -# run: ./infra/scripts/pre-provision.sh -# interactive: false -# continueOnError: false +hooks: + # Import placeholder image to ACR before provisioning (needed for AILZ internal environments) + preprovision: + shell: sh + run: ./infra/scripts/pre-provision.sh + interactive: false + continueOnError: false -# # Run after infrastructure is provisioned -# # postprovision: -# # shell: sh -# # run: ./infra/scripts/post-provision.sh -# # interactive: false -# # continueOnError: false + # Run after infrastructure is provisioned + postprovision: + shell: sh + run: ./infra/scripts/post-provision.sh + interactive: false + continueOnError: false # # # Run before deploying all services # # predeploy: @@ -69,9 +58,9 @@ services: # # interactive: false # # continueOnError: false -# # Run after all services are deployed -# postdeploy: -# shell: sh -# run: ./infra/scripts/post-deploy.sh -# interactive: true -# continueOnError: false + # Run after all services are deployed + postdeploy: + shell: sh + run: ./infra/scripts/post-deploy.sh + interactive: true + continueOnError: false diff --git a/contentflow-api/app/dependencies.py b/contentflow-api/app/dependencies.py index 898672f..4e8743e 100644 --- a/contentflow-api/app/dependencies.py +++ b/contentflow-api/app/dependencies.py @@ -20,6 +20,7 @@ EXECUTOR_CATALOG_FILE_PATH = f'{Path(__file__).parent.parent.parent}/contentflow-lib/executor_catalog.yaml' +SEED_PIPELINES_DIR = f'{Path(__file__).parent.parent}/seed/pipelines' def get_config_provider(refresh: bool = False) -> ConfigurationProvider: """Get a singleton instance of ConfigurationProvider""" @@ -54,10 +55,7 @@ def get_health_service(): cosmos_db_name=app_settings.COSMOS_DB_NAME, cosmos_db_containers=app_settings.get_cosmos_db_containers(), blob_storage_account=app_settings.BLOB_STORAGE_ACCOUNT_NAME, - blob_storage_container=app_settings.BLOB_STORAGE_CONTAINER_NAME, - storage_account_worker_queue_url=app_settings.STORAGE_ACCOUNT_WORKER_QUEUE_URL, - storage_worker_queue_name=app_settings.STORAGE_WORKER_QUEUE_NAME, - worker_engine_api_endpoint=app_settings.WORKER_ENGINE_API_ENDPOINT) + blob_storage_container=app_settings.BLOB_STORAGE_CONTAINER_NAME) @ttl_cache(ttl=__cache_ttl) # Cache for 10 minutes async def get_pipeline_service(): @@ -89,6 +87,20 @@ async def get_pipeline_execution_service(): from app.services.pipeline_execution_service import PipelineExecutionService return PipelineExecutionService(await get_cosmos_client()) + +async def get_ingest_service(): + """Dependency to get IngestService (uses global blob storage singleton)""" + from app.settings import get_settings + from app.utils.blob_storage import get_blob_storage_service + from app.services.ingest_service import IngestService + + app_settings = get_settings() + blob_service = await get_blob_storage_service( + account_name=app_settings.BLOB_STORAGE_ACCOUNT_NAME, + container_name=app_settings.BLOB_STORAGE_CONTAINER_NAME, + ) + return IngestService(blob_service) + async def initialize_executor_catalog(): """Initialize executor catalog""" logger.info(f"{'>'*70}") @@ -158,6 +170,116 @@ async def initialize_blob_storage(): logger.exception(e) raise +async def initialize_seed_pipelines(): + """Seed pipelines from YAML files in seed/pipelines/ on startup. + + For each YAML file, checks if a pipeline with the same name already exists. + If not, creates it. If it already exists, skips it. + """ + import yaml + import glob + + logger.info(f"{'>'*70}") + logger.info("contentflow.api: Seeding pipelines from seed/pipelines/...") + logger.info(f"{'-'*70}") + try: + pipeline_service = await get_pipeline_service() + seed_dir = str(SEED_PIPELINES_DIR) + yaml_files = glob.glob(f"{seed_dir}/*.yaml") + glob.glob(f"{seed_dir}/*.yml") + + if not yaml_files: + logger.info("No pipeline YAML files found in seed/pipelines/. Skipping.") + logger.info(f"{'<'*70}") + return True + + created = 0 + skipped = 0 + for yaml_path in yaml_files: + try: + with open(yaml_path, "r") as f: + raw = yaml.safe_load(f) + + pipeline_def = raw.get("pipeline", raw) + name = pipeline_def.get("name") + if not name: + logger.warning(f"Skipping {yaml_path}: no 'name' field found.") + continue + + # Check if already exists + existing = await pipeline_service.get_pipeline_by_name(name) + if existing: + logger.info(f" Pipeline '{name}' already exists (id={existing.id}). Skipping.") + skipped += 1 + continue + + # Build the full YAML string for storage + with open(yaml_path, "r") as f: + yaml_str = f.read() + + # Parse executors from YAML into nodes for canvas visualization + executors = pipeline_def.get("executors", []) + nodes = [] + for executor in executors: + nodes.append({ + "id": executor.get("id", ""), + "type": "executor", + "position": executor.get("position", {"x": 0, "y": 0}), + "data": { + "label": executor.get("name", ""), + "executor": { + "id": executor.get("type", ""), + "name": executor.get("name", ""), + "category": "unknown", + "description": executor.get("description", ""), + }, + "config": { + "name": executor.get("name", ""), + "description": executor.get("description", ""), + "settings": executor.get("settings", {}), + }, + }, + }) + + # Parse edges from YAML for canvas connections + yaml_edges = pipeline_def.get("edges", []) + edges = [] + for edge_def in yaml_edges: + src = edge_def.get("from", "") + tgt = edge_def.get("to", "") + if isinstance(src, str) and isinstance(tgt, str): + edges.append({ + "id": f"{src}-{tgt}-{len(edges)}", + "source": src, + "target": tgt, + "type": "default", + "animated": True, + }) + + pipeline_data = { + "name": name, + "description": pipeline_def.get("description", ""), + "yaml": yaml_str, + "nodes": nodes, + "edges": edges, + "tags": pipeline_def.get("tags", []), + "enabled": True, + } + + result = await pipeline_service.create_pipeline(pipeline_data) + logger.info(f" Created pipeline '{name}' with id={result.id}") + created += 1 + + except Exception as e: + logger.warning(f" Failed to seed pipeline from {yaml_path}: {e}") + + logger.info(f"Pipeline seeding complete: {created} created, {skipped} skipped.") + logger.info(f"{'<'*70}") + return True + except Exception as e: + logger.error(f"Error during pipeline seeding: {str(e)}") + logger.exception(e) + raise + async def close_all(): """Close all dependencies""" global __cosmos_client diff --git a/contentflow-api/app/models/__init__.py b/contentflow-api/app/models/__init__.py index 508b1ce..b8c5980 100644 --- a/contentflow-api/app/models/__init__.py +++ b/contentflow-api/app/models/__init__.py @@ -17,6 +17,7 @@ VaultExecution, VaultCrawlCheckpoint, ) +from ._ingest import IngestPayload, IngestResponse, IngestResultsResponse try: __version__ = importlib.metadata.version(__name__) @@ -39,4 +40,7 @@ "VaultUpdateRequest", "VaultExecution", "VaultCrawlCheckpoint", + "IngestPayload", + "IngestResponse", + "IngestResultsResponse", ] \ No newline at end of file diff --git a/contentflow-api/app/models/_ingest.py b/contentflow-api/app/models/_ingest.py new file mode 100644 index 0000000..426872b --- /dev/null +++ b/contentflow-api/app/models/_ingest.py @@ -0,0 +1,33 @@ +""" +Ingest API models for multipart document submission. +""" +from typing import Optional +from pydantic import BaseModel + + +class IngestPayload(BaseModel): + """Form fields submitted by the client that become ProvidedDetails.json""" + caseId: str + firstName: str + lastName: str + mailingAddress: str + dateOfBirth: str + + +class IngestResponse(BaseModel): + """Response returned to client on successful ingest (202 Accepted)""" + execution_id: str + case_id: str + status: str + files_uploaded: int + blob_prefix: str + pipeline_id: str + pipeline_name: str + + +class IngestResultsResponse(BaseModel): + """Response for GET /ingest/{execution_id}/results""" + execution_id: str + status: str + results: Optional[dict] = None + error: Optional[str] = None diff --git a/contentflow-api/app/routers/__init__.py b/contentflow-api/app/routers/__init__.py index da1967a..97611d0 100644 --- a/contentflow-api/app/routers/__init__.py +++ b/contentflow-api/app/routers/__init__.py @@ -4,6 +4,7 @@ from .pipelines import router as pipelines_router from .executors import router as executors_router from .vaults import router as vaults_router +from .ingest import router as ingest_router try: __version__ = importlib.metadata.version(__name__) @@ -15,4 +16,5 @@ "pipelines_router", "executors_router", "vaults_router", + "ingest_router", ] \ No newline at end of file diff --git a/contentflow-api/app/routers/ingest.py b/contentflow-api/app/routers/ingest.py new file mode 100644 index 0000000..3a7397f --- /dev/null +++ b/contentflow-api/app/routers/ingest.py @@ -0,0 +1,281 @@ +""" +Ingest router for multipart document submission API. +""" +import json +import logging +import os +from typing import List, Optional + +from fastapi import APIRouter, BackgroundTasks, Depends, File, Form, HTTPException, UploadFile +from fastapi.responses import JSONResponse +from azure.core.exceptions import ResourceNotFoundError + +from app.models import IngestPayload, IngestResponse, IngestResultsResponse, ExecutionStatus +from app.services.pipeline_service import PipelineService +from app.services.pipeline_execution_service import PipelineExecutionService +from app.services.ingest_service import IngestService +from app.dependencies import ( + get_pipeline_service, + get_pipeline_execution_service, + get_ingest_service, +) +from app.settings import get_settings +from app.utils.blob_storage import BlobStorageService, get_blob_storage_service + +logger = logging.getLogger("contentflow.api.routers.ingest") + +router = APIRouter(prefix="/ingest", tags=["ingest"]) + + +@router.post("/", response_model=IngestResponse, status_code=202) +async def ingest_documents( + background_tasks: BackgroundTasks, + # Payload fields + case_id: str = Form(..., alias="caseId", description="Client's unique case identifier"), + first_name: str = Form(..., alias="firstName"), + last_name: str = Form(..., alias="lastName"), + mailing_address: str = Form(..., alias="mailingAddress"), + date_of_birth: str = Form(..., alias="dateOfBirth"), + # Files + files: List[UploadFile] = File(..., description="Document files to process"), + # Dependencies + pipeline_service: PipelineService = Depends(get_pipeline_service), + execution_service: PipelineExecutionService = Depends(get_pipeline_execution_service), + ingest_service: IngestService = Depends(get_ingest_service), +): + """ + Submit documents for processing through a ContentFlow pipeline. + + Accepts multipart form data with case details and document files. + Generates ProvidedDetails.json from the payload, uploads everything + to blob storage, and triggers the pipeline. + + Returns 202 Accepted with an execution_id for polling. + """ + settings = get_settings() + + # --- Validate case_id --- + try: + ingest_service.validate_case_id(case_id) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + + # --- Validate files --- + if not files: + raise HTTPException(status_code=400, detail="At least one file is required.") + + if len(files) > settings.INGEST_MAX_FILE_COUNT: + raise HTTPException( + status_code=400, + detail=f"Too many files. Maximum allowed: {settings.INGEST_MAX_FILE_COUNT}", + ) + + allowed_extensions = settings.INGEST_ALLOWED_EXTENSIONS + max_file_bytes = settings.INGEST_MAX_FILE_SIZE_MB * 1024 * 1024 + max_total_bytes = settings.INGEST_MAX_TOTAL_SIZE_MB * 1024 * 1024 + total_size = 0 + + for f in files: + # Validate extension + fname = f.filename or "" + ext = os.path.splitext(fname)[1].lower() + if ext not in allowed_extensions: + raise HTTPException( + status_code=400, + detail=f"File type '{ext}' not allowed for file '{fname}'. Allowed: {allowed_extensions}", + ) + # Validate size (if known) + if f.size is not None: + if f.size > max_file_bytes: + raise HTTPException( + status_code=413, + detail=f"File '{fname}' exceeds maximum size of {settings.INGEST_MAX_FILE_SIZE_MB} MB.", + ) + total_size += f.size + + if total_size > max_total_bytes: + raise HTTPException( + status_code=413, + detail=f"Total upload size exceeds maximum of {settings.INGEST_MAX_TOTAL_SIZE_MB} MB.", + ) + + # --- Resolve pipeline by name (abstracted from client) --- + pipeline_name = settings.INGEST_PIPELINE_NAME + pipeline = await pipeline_service.get_pipeline_by_name(pipeline_name) + if not pipeline: + logger.error(f"Ingest pipeline not found by name: '{pipeline_name}'") + raise HTTPException( + status_code=503, + detail="Processing pipeline is not available. Please contact support.", + ) + if not pipeline.enabled: + raise HTTPException( + status_code=503, + detail="Processing pipeline is temporarily disabled. Please try again later.", + ) + + # --- Create execution record first (to get execution_id for folder name) --- + try: + execution = await execution_service.create_execution( + pipeline=pipeline, + inputs={}, # will be updated after folder name is determined + created_by=None, + ) + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Failed to create execution record: {e}" + ) + + execution_id = execution.id + folder_prefix = f"input/{case_id}_{execution_id}/" + + # --- Build payload model --- + payload = IngestPayload( + caseId=case_id, + firstName=first_name, + lastName=last_name, + mailingAddress=mailing_address, + dateOfBirth=date_of_birth, + ) + + # --- Upload files and ProvidedDetails.json --- + try: + # Generate and upload ProvidedDetails.json + provided_details_bytes = ingest_service.generate_provided_details(payload) + await ingest_service.upload_provided_details(folder_prefix, provided_details_bytes) + + # Upload document files + uploaded_paths = await ingest_service.upload_files(folder_prefix, files) + except Exception as e: + # Mark execution as failed if upload fails + await execution_service.update_execution_status( + execution_id, ExecutionStatus.FAILED, error=f"File upload failed: {e}" + ) + raise HTTPException(status_code=500, detail=f"File upload failed: {e}") + + # --- Update execution inputs with case_prefix --- + try: + exec_record = await execution_service.get_execution(execution_id) + if exec_record: + exec_record.inputs = { + "case_id": case_id, + "case_prefix": folder_prefix, + "execution_id": execution_id, + } + await execution_service.update( + exec_record.model_dump(by_alias=True, exclude_none=False) + ) + except Exception as e: + logger.warning(f"Failed to update execution inputs: {e}") + + # --- Trigger pipeline execution in background --- + background_tasks.add_task( + execution_service.start_execution, + execution_id=execution_id, + pipeline=pipeline, + ) + + return IngestResponse( + execution_id=execution_id, + case_id=case_id, + status="pending", + files_uploaded=len(uploaded_paths), + blob_prefix=folder_prefix, + pipeline_id=pipeline.id, + pipeline_name=pipeline.name, + ) + + +@router.get("/{execution_id}/results", response_model=IngestResultsResponse) +async def get_ingest_results( + execution_id: str, + execution_service: PipelineExecutionService = Depends(get_pipeline_execution_service), +): + """ + Retrieve validation results for a completed ingest execution. + + - **200**: Execution completed — returns results.json content. + - **202**: Execution is still running — retry later. + - **404**: Execution ID not found. + - **422**: Execution failed — returns error details. + """ + # --- Validate execution_id format --- + if not execution_id or not execution_id.strip(): + raise HTTPException(status_code=400, detail="execution_id is required.") + + # --- Fetch execution record from Cosmos DB --- + try: + execution = await execution_service.get_execution(execution_id) + except Exception as e: + logger.error(f"Failed to look up execution {execution_id}: {e}") + raise HTTPException(status_code=500, detail="Unable to retrieve execution record.") + + if not execution: + raise HTTPException(status_code=404, detail="Execution not found.") + + # --- Handle non-terminal states --- + status = execution.status + if isinstance(status, ExecutionStatus): + status = status.value + + if status in (ExecutionStatus.PENDING.value, ExecutionStatus.RUNNING.value): + return JSONResponse( + status_code=202, + content=IngestResultsResponse( + execution_id=execution_id, + status=status, + ).model_dump(), + ) + + # --- Handle failed / cancelled --- + if status in (ExecutionStatus.FAILED.value, ExecutionStatus.CANCELLED.value): + raise HTTPException( + status_code=422, + detail={ + "execution_id": execution_id, + "status": status, + "error": execution.error or "Execution did not complete successfully.", + }, + ) + + # --- Execution completed — download results.json from blob --- + case_prefix = (execution.inputs or {}).get("case_prefix") + if not case_prefix: + raise HTTPException( + status_code=500, + detail="Execution record is missing case_prefix — cannot locate results.", + ) + + results_blob_path = f"{case_prefix}results.json" + + settings = get_settings() + try: + blob_service = await get_blob_storage_service( + account_name=settings.BLOB_STORAGE_ACCOUNT_NAME, + container_name=settings.BLOB_STORAGE_CONTAINER_NAME, + ) + raw_bytes = await blob_service.download_file(results_blob_path) + results_data = json.loads(raw_bytes) + except ResourceNotFoundError: + raise HTTPException( + status_code=404, + detail=f"Results file not found at {results_blob_path}. The pipeline may not have produced results.", + ) + except json.JSONDecodeError: + logger.error(f"Malformed results.json at {results_blob_path}") + raise HTTPException( + status_code=500, + detail="Results file exists but contains invalid JSON.", + ) + except Exception as e: + logger.error(f"Error downloading results for {execution_id}: {e}") + raise HTTPException( + status_code=500, + detail="Failed to retrieve results from storage.", + ) + + return IngestResultsResponse( + execution_id=execution_id, + status=status, + results=results_data, + ) diff --git a/contentflow-api/app/services/health_service.py b/contentflow-api/app/services/health_service.py index de17923..e41c23a 100644 --- a/contentflow-api/app/services/health_service.py +++ b/contentflow-api/app/services/health_service.py @@ -10,7 +10,6 @@ from pydantic import BaseModel from azure.cosmos import CosmosClient -from azure.storage.queue import QueueServiceClient from azure.appconfiguration import AzureAppConfigurationClient from azure.core.exceptions import ResourceNotFoundError @@ -44,22 +43,15 @@ def __init__(self, cosmos_db_name: str = None, cosmos_db_containers: List[str] = None, blob_storage_account: str = None, - blob_storage_container: str = None, - storage_account_worker_queue_url: str = None, - storage_worker_queue_name: str = None, - worker_engine_api_endpoint: str = None): + blob_storage_container: str = None): self.cosmos_endpoint = cosmos_endpoint self.cosmos_db_name = cosmos_db_name self.cosmos_db_containers = cosmos_db_containers self.blob_storage_account = blob_storage_account self.blob_storage_container = blob_storage_container - self.storage_account_worker_queue_url = storage_account_worker_queue_url - self.storage_worker_queue_name = storage_worker_queue_name - self.worker_engine_api_endpoint = worker_engine_api_endpoint self.cosmos_client: Optional[CosmosClient] = None - self.queue_client: Optional[QueueServiceClient] = None self.app_config_client: Optional[AzureAppConfigurationClient] = None credential, token_details = get_azure_credential_with_details() @@ -75,12 +67,10 @@ async def check_all_services(self) -> SystemHealth: self._check_app_config_health(), self._check_cosmos_db_health(), self._check_blob_storage_health(), - self._check_storage_queue_health(), - self._check_worker_health(), return_exceptions=True ) - service_names = ["app_config", "cosmos_db", "blob_storage", "storage_queue", "worker"] + service_names = ["app_config", "cosmos_db", "blob_storage"] for i, result in enumerate(results): if isinstance(result, Exception): @@ -272,85 +262,6 @@ async def _check_blob_storage_health(self) -> ServiceHealth: endpoint=f"https://{self.blob_storage_account}.blob.core.windows.net" ) - async def _check_storage_queue_health(self) -> ServiceHealth: - """Check Azure Storage Queue connection health""" - start_time = datetime.now(timezone.utc) - - try: - - logger.debug(f"Checking storage queue health with URL: {self.storage_account_worker_queue_url} and queue name: {self.storage_worker_queue_name}") - - if not self.storage_account_worker_queue_url: - return ServiceHealth( - name="storage_queue", - status="error", - message="Storage queue URL not configured", - error="Storage queue URL not configured in Azure App Configuration. Ensure 'STORAGE_ACCOUNT_WORKER_QUEUE_URL' is set using the correct key prefix.", - last_checked=datetime.now(timezone.utc).isoformat(), - endpoint="Not configured" - ) - - credential, token_details = (self.credential, self.token_details) - if not credential or not token_details: - credential, token_details = get_azure_credential_with_details() - - # Create client if not exists - if not self.queue_client: - # Extract account URL from queue URL - account_url = self.storage_account_worker_queue_url - - self.queue_client = QueueServiceClient( - account_url=account_url, - credential=credential - ) - - # Try to get queue properties - queue_client = self.queue_client.get_queue_client(self.storage_worker_queue_name) - properties = queue_client.get_queue_properties() - - response_time = int((datetime.now(timezone.utc) - start_time).total_seconds() * 1000) - - return ServiceHealth( - name="storage_queue", - status="connected", - message="Connected successfully", - details={ - "queue_name": self.storage_worker_queue_name, - "approximate_message_count": properties.approximate_message_count, - "metadata": properties.metadata, - "credential_type": "default_azure_credential", - "credential_details": token_details - }, - response_time_ms=response_time, - last_checked=datetime.now(timezone.utc).isoformat(), - endpoint=self.storage_account_worker_queue_url - ) - - except ResourceNotFoundError: - response_time = int((datetime.now(timezone.utc) - start_time).total_seconds() * 1000) - return ServiceHealth( - name="storage_queue", - status="error", - message=f"Queue '{self.storage_worker_queue_name}' not found", - error=f"Queue '{self.storage_worker_queue_name}' not found", - details={"error_type": "ResourceNotFoundError"}, - response_time_ms=response_time, - last_checked=datetime.now(timezone.utc).isoformat(), - endpoint=self.storage_account_worker_queue_url - ) - except Exception as e: - response_time = int((datetime.now(timezone.utc) - start_time).total_seconds() * 1000) - return ServiceHealth( - name="storage_queue", - status="error", - message=f"Connection failed: {e.__class__.__name__}", - error=f'{str(e)}', - details={"error_type": type(e).__name__}, - response_time_ms=response_time, - last_checked=datetime.now(timezone.utc).isoformat(), - endpoint=self.storage_account_worker_queue_url - ) - async def _check_app_config_health(self) -> ServiceHealth: """Check Azure App Configuration connection health""" start_time = datetime.now(timezone.utc) @@ -424,74 +335,14 @@ async def _check_app_config_health(self) -> ServiceHealth: endpoint=endpoint or connection_string or "Not configured" ) - async def _check_worker_health(self) -> ServiceHealth: - """Check worker service health by querying the worker FastAPI health endpoint via aiohttp.""" - import aiohttp - - start_time = datetime.now(timezone.utc) - endpoint = self.worker_engine_api_endpoint - url = endpoint.rstrip("/") + "/status" - - logger.debug(f"Checking worker health with URL: {url}") - - try: - async with aiohttp.ClientSession() as session: - async with session.get(url, timeout=5) as resp: - response_time = int((datetime.now(timezone.utc) - start_time).total_seconds() * 1000) - if resp.status == 200: - data = await resp.json() - return ServiceHealth( - name="worker", - status="connected", - message="Worker engine is running", - details={ - "status": data.get("running"), - "worker_name": data.get("worker_name"), - "timestamp": data.get("timestamp"), - "processing_workers": data.get("processing_workers"), - "source_workers": data.get("source_workers") - }, - response_time_ms=response_time, - last_checked=datetime.now(timezone.utc).isoformat(), - endpoint=self.worker_engine_api_endpoint - ) - else: - text = await resp.text() - return ServiceHealth( - name="worker", - status="error", - message=f"Worker health endpoint returned status {resp.status}", - error=text, - details={"http_status": resp.status}, - response_time_ms=response_time, - last_checked=datetime.now(timezone.utc).isoformat(), - endpoint=url - ) - except Exception as e: - response_time = int((datetime.now(timezone.utc) - start_time).total_seconds() * 1000) - return ServiceHealth( - name="worker", - status="error", - message=f"Worker health check failed: {e.__class__.__name__}", - error=f'{str(e)}', - details={"error_type": type(e).__name__}, - response_time_ms=response_time, - last_checked=datetime.now(timezone.utc).isoformat(), - endpoint=url - ) - async def check_service_health(self, service_name: str) -> ServiceHealth: """Check the health of a specific service""" if service_name == "cosmos_db": return await self._check_cosmos_db_health() - elif service_name == "storage_queue": - return await self._check_storage_queue_health() elif service_name == "app_config": return await self._check_app_config_health() elif service_name == "blob_storage": return await self._check_blob_storage_health() - elif service_name == "worker": - return await self._check_worker_health() else: return ServiceHealth( name=service_name, diff --git a/contentflow-api/app/services/ingest_service.py b/contentflow-api/app/services/ingest_service.py new file mode 100644 index 0000000..f77d668 --- /dev/null +++ b/contentflow-api/app/services/ingest_service.py @@ -0,0 +1,109 @@ +""" +Ingest service for handling document uploads and ProvidedDetails.json generation. +""" +import json +import logging +import re +from typing import List + +from fastapi import UploadFile + +from app.models import IngestPayload +from app.utils.blob_storage import BlobStorageService + +logger = logging.getLogger("contentflow.api.services.ingest_service") + +# Characters allowed in case_id: alphanumeric, hyphens, underscores +CASE_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_\-]{1,128}$") + + +class IngestService: + """Service for document ingestion: upload, ProvidedDetails generation, validation.""" + + def __init__(self, blob_service: BlobStorageService): + self.blob_service = blob_service + + def validate_case_id(self, case_id: str) -> None: + """Validate case_id format. Raises ValueError on invalid format.""" + if not case_id or not case_id.strip(): + raise ValueError("case_id is required and cannot be empty.") + if not CASE_ID_PATTERN.match(case_id): + raise ValueError( + f"Invalid case_id format: '{case_id}'. " + "Only alphanumeric characters, hyphens, and underscores are allowed (max 128 chars)." + ) + + def sanitize_filename(self, filename: str) -> str: + """Sanitize a filename to prevent path traversal and invalid characters.""" + # Remove any path components + name = filename.replace("\\", "/").split("/")[-1] + # Remove leading dots + name = name.lstrip(".") + # Remove null bytes + name = name.replace("\x00", "") + # Replace other problematic characters + name = re.sub(r'[<>:"|?*]', "_", name) + if not name: + raise ValueError(f"Invalid filename after sanitization: '{filename}'") + return name + + def generate_provided_details(self, payload: IngestPayload) -> bytes: + """ + Generate ProvidedDetails.json content from the ingest payload. + + Uses the simple approach: all relevant fields are included in a single + details block. The validation rules engine selects which fields to check + per document type. + """ + # Field names must match rules.json / FetchedDetails conventions + # (PascalCase, matching Azure Content Understanding output fields) + full_name = f"{payload.firstName} {payload.lastName}".strip() + details = { + "FirstName": payload.firstName, + "LastName": payload.lastName, + "FullName": full_name, + "Address": payload.mailingAddress, + "BirthDate": payload.dateOfBirth, + } + + provided_details = { + "caseId": payload.caseId, + "details": details, + } + + return json.dumps(provided_details, indent=2, ensure_ascii=False).encode("utf-8") + + async def upload_files( + self, folder_prefix: str, files: List[UploadFile] + ) -> List[str]: + """ + Upload document files to blob storage under the given folder prefix. + + Returns list of uploaded blob paths. + """ + uploaded_paths = [] + for file in files: + sanitized_name = self.sanitize_filename(file.filename) + blob_name = f"{folder_prefix}{sanitized_name}" + content = await file.read() + await self.blob_service.upload_file( + file_content=content, + blob_name=blob_name, + content_type=file.content_type, + ) + uploaded_paths.append(blob_name) + logger.info(f"Uploaded file: {blob_name}") + return uploaded_paths + + async def upload_provided_details( + self, folder_prefix: str, content: bytes + ) -> str: + """Upload generated ProvidedDetails.json to the case folder.""" + blob_name = f"{folder_prefix}ProvidedDetails.json" + await self.blob_service.upload_file( + file_content=content, + blob_name=blob_name, + content_type="application/json", + ) + logger.info(f"Uploaded ProvidedDetails.json: {blob_name}") + return blob_name diff --git a/contentflow-api/app/services/pipeline_execution_service.py b/contentflow-api/app/services/pipeline_execution_service.py index f6b7f92..2754036 100644 --- a/contentflow-api/app/services/pipeline_execution_service.py +++ b/contentflow-api/app/services/pipeline_execution_service.py @@ -26,7 +26,7 @@ # Import contentflow library components from contentflow.pipeline import PipelineExecutor as ContentFlowPipelineExecutor -from contentflow.models import Content +from contentflow.models import Content, ContentIdentifier from contentflow.utils import make_safe_json logger = logging.getLogger("contentflow.api.services.pipeline_execution_service") @@ -154,8 +154,31 @@ async def execute_pipeline_async( has_pipeline_failed = False + # Build initial input from execution record inputs + # If case_prefix is present (ingest API flow), pass a Content + # object so the blob input discovery executor can use + # prefix_from_input_field to scope to the correct folder. + initial_input = [] + execution_record = await self.get_execution(execution_id) + if ( + execution_record + and execution_record.inputs + and execution_record.inputs.get("case_prefix") + ): + initial_input = Content( + id=ContentIdentifier( + canonical_id=f"ingest-{execution_id}", + unique_id=execution_id, + source_name="ingest_api", + ), + data={ + "case_prefix": execution_record.inputs["case_prefix"], + "execution_id": execution_id, + }, + ) + # Execute and collect events - async for event in executor.execute_stream([]): + async for event in executor.execute_stream(initial_input): # Convert to our event model exec_event = PipelineExecutionEvent( event_type=event.event_type, diff --git a/contentflow-api/app/settings.py b/contentflow-api/app/settings.py index 3f6e8f2..be34f13 100644 --- a/contentflow-api/app/settings.py +++ b/contentflow-api/app/settings.py @@ -57,17 +57,24 @@ class AppSettings(BaseModel): # Azure Storage Blob settings BLOB_STORAGE_ACCOUNT_NAME: str = "" BLOB_STORAGE_CONTAINER_NAME: str = "content" - - # Azure Storage Queue settings - STORAGE_ACCOUNT_WORKER_QUEUE_URL: str = "" - STORAGE_WORKER_QUEUE_NAME: str = "contentflow-execution-requests" - WORKER_ENGINE_API_ENDPOINT: str = "http://localhost:8099" + # Ingest API settings + INGEST_PIPELINE_NAME: str = "Details extraction and validation pipeline" + INGEST_MAX_FILE_COUNT: int = 20 + INGEST_MAX_FILE_SIZE_MB: int = 100 + INGEST_MAX_TOTAL_SIZE_MB: int = 500 + INGEST_ALLOWED_EXTENSIONS: list[str] = [ + ".pdf", ".docx", ".pptx", ".xlsx", + ".png", ".jpg", ".jpeg", ".tiff" + ] def __init__(self, **kwargs): """Initialize AppSettings and load configuration from Azure App Configuration""" super().__init__(**kwargs) - self._load_from_app_config() + try: + self._load_from_app_config() + except Exception as e: + print(f"\033[93m⚠️ Could not load from App Config: {e}. Using defaults/env vars.\033[0m") def _load_from_app_config(self): """Load configuration values from Azure App Configuration""" @@ -84,9 +91,6 @@ def _load_from_app_config(self): "COSMOS_DB_NAME", "BLOB_STORAGE_ACCOUNT_NAME", "BLOB_STORAGE_CONTAINER_NAME", - "STORAGE_ACCOUNT_WORKER_QUEUE_URL", - "STORAGE_WORKER_QUEUE_NAME", - "WORKER_ENGINE_API_ENDPOINT", "API_SERVER_HOST", "API_SERVER_PORT", "API_SERVER_WORKERS", diff --git a/contentflow-api/app/startup.py b/contentflow-api/app/startup.py index a00af5e..273c380 100644 --- a/contentflow-api/app/startup.py +++ b/contentflow-api/app/startup.py @@ -5,7 +5,7 @@ import logging from opentelemetry import trace -from app.dependencies import initialize_cosmos, initialize_blob_storage, initialize_executor_catalog +from app.dependencies import initialize_cosmos, initialize_blob_storage, initialize_executor_catalog, initialize_seed_pipelines logger = logging.getLogger("contentflow.api.startup") @@ -17,6 +17,7 @@ async def startup_tasks(): initialize_cosmos(), initialize_blob_storage(), initialize_executor_catalog(), + initialize_seed_pipelines(), # Add other startup tasks here ] diff --git a/contentflow-api/main.py b/contentflow-api/main.py index 2a36fe9..0da8867 100644 --- a/contentflow-api/main.py +++ b/contentflow-api/main.py @@ -12,7 +12,7 @@ from app.settings import get_settings from app.startup import startup, shutdown -from app.routers import health_router, pipelines_router, executors_router, vaults_router +from app.routers import health_router, pipelines_router, executors_router, vaults_router, ingest_router @asynccontextmanager async def lifespan(app: FastAPI): @@ -45,6 +45,7 @@ def initialize_api_application() -> FastAPI: app.include_router(pipelines_router, prefix="/api") app.include_router(executors_router, prefix="/api") app.include_router(vaults_router, prefix="/api") + app.include_router(ingest_router, prefix="/api") # # Global exception handler # @app.exception_handler(Exception) diff --git a/contentflow-api/seed/analyzers/details_extractor_documents_new1.json b/contentflow-api/seed/analyzers/details_extractor_documents_new1.json new file mode 100644 index 0000000..60d2804 --- /dev/null +++ b/contentflow-api/seed/analyzers/details_extractor_documents_new1.json @@ -0,0 +1,131 @@ +{ + "description": "to fetch details from uploaded documents", + "baseAnalyzerId": "prebuilt-document", + "models": { + "completion": "gpt-4.1", + "embedding": "text-embedding-3-large" + }, + "config": { + "returnDetails": false, + "enableOcr": true, + "enableLayout": false, + "enableFormula": true, + "enableFigureDescription": false, + "enableFigureAnalysis": false, + "chartFormat": "chartjs", + "tableFormat": "html", + "enableSegment": false, + "omitContent": false, + "segmentPerPage": false, + "annotationFormat": "markdown" + }, + "fieldSchema": { + "fields": { + "CountryRegion": { + "type": "string", + "method": "extract", + "description": "CountryRegion extracted from the document" + }, + "Region": { + "type": "string", + "method": "extract", + "description": "Region extracted from the document" + }, + "DocumentNumber": { + "type": "string", + "method": "extract", + "description": "DocumentNumber extracted from the document" + }, + "DocumentDiscriminator": { + "type": "string", + "method": "extract", + "description": "DocumentDiscriminator extracted from the document" + }, + "FirstName": { + "type": "string", + "method": "extract", + "description": "FirstName extracted from the document" + }, + "LastName": { + "type": "string", + "method": "extract", + "description": "LastName extracted from the document" + }, + "Address": { + "type": "string", + "method": "extract", + "description": "Address extracted from the document" + }, + "BirthDate": { + "type": "date", + "method": "extract", + "description": "BirthDate extracted from the document" + }, + "ExpirationDate": { + "type": "date", + "method": "extract", + "description": "ExpirationDate extracted from the document" + }, + "IssueDate": { + "type": "date", + "method": "extract", + "description": "IssueDate extracted from the document" + }, + "EyeColor": { + "type": "string", + "method": "extract", + "description": "EyeColor extracted from the document" + }, + "HairColor": { + "type": "string", + "method": "extract", + "description": "HairColor extracted from the document" + }, + "Height": { + "type": "string", + "method": "extract", + "description": "Height extracted from the document" + }, + "Weight": { + "type": "string", + "method": "extract", + "description": "Weight extracted from the document" + }, + "Sex": { + "type": "string", + "method": "extract", + "description": "Sex extracted from the document" + }, + "Endorsements": { + "type": "string", + "method": "extract", + "description": "Endorsements extracted from the document" + }, + "Restrictions": { + "type": "string", + "method": "extract", + "description": "Restrictions extracted from the document" + }, + "PersonalNumber": { + "type": "string", + "method": "extract", + "description": "PersonalNumber extracted from the document" + }, + "PlaceOfBirth": { + "type": "string", + "method": "extract", + "description": "PlaceOfBirth extracted from the document" + }, + "VehicleClass": { + "type": "string", + "method": "extract", + "description": "VehicleClass extracted from the document" + }, + "Category": { + "type": "string", + "method": "extract", + "description": "Category extracted from the document" + } + } + } +} diff --git a/contentflow-api/seed/pipelines/draft-fetched-details-pipeline.yaml b/contentflow-api/seed/pipelines/draft-fetched-details-pipeline.yaml new file mode 100644 index 0000000..200f476 --- /dev/null +++ b/contentflow-api/seed/pipelines/draft-fetched-details-pipeline.yaml @@ -0,0 +1,279 @@ +pipeline: + name: Details extraction and validation pipeline + description: |- + Processes documents from blob folder, classifies types, extracts details, outputs fetched_details.json + and validates the fetched details + executors: + - id: azure_blob_input_discovery-1777658905426 + name: Discover Input Files + type: azure_blob_input_discovery + position: + x: 409 + 'y': 61 + description: >- + Discovers and lists content files from Azure Blob Storage containers. Must be used with an Azure Blob Content + Retriever executor to fetch the actual content. + settings: + enabled: true + condition: '' + fail_pipeline_on_error: true + debug_mode: true + blob_storage_account: ${AZURE_STORAGE_ACCOUNT_NAME} + blob_storage_credential_type: default_azure_credential + blob_storage_account_key: '' + blob_container_name: content + discover_mode: files + prefix_from_input_field: case_prefix + prefix: input/ + file_extensions: .pdf,.docx,.pptx,.xlsx,.png,.jpg,.jpeg,.tiff + max_depth: 1 + max_results: 20 + batch_size: 10 + include_metadata: true + sort_by: name + sort_ascending: true + min_size_bytes: 0 + max_size_bytes: 0 + polling_interval_seconds: 300 + modified_after: '' + modified_before: '' + - id: azure_blob_content_retriever-1777658985345 + name: Retrieve File Content + type: azure_blob_content_retriever + position: + x: 405 + 'y': 216 + description: Retrieves document content from azure blob sources. Must be used after an Azure Blob Input Discovery executor. + settings: + enabled: true + condition: '' + fail_pipeline_on_error: true + debug_mode: true + include_content_bytes_as_field: false + use_temp_file_for_content: true + temp_folder: ./tmp/contentflow + max_concurrent: 3 + timeout_secs: 300 + continue_on_error: true + - id: azure_content_understanding_extractor-1777662467773 + name: Extract Document type + type: azure_content_understanding_extractor + position: + x: 402 + 'y': 372 + description: Extracts content from documents using Azure AI Content Understanding with 70+ prebuilt analyzers + settings: + enabled: true + condition: '' + fail_pipeline_on_error: true + debug_mode: true + content_field: '' + temp_file_path_field: temp_file_path + output_field: content_understanding_result + analyzer_id: prebuilt-read + retrieve_figures: false + figures_output_field: figures + content_understanding_endpoint: ${AZURE_CONTENT_UNDERSTANDING_ENDPOINT} + content_understanding_credential_type: default_azure_credential + content_understanding_subscription_key: '' + content_understanding_api_version: '2025-11-01' + content_understanding_timeout: 60 + content_understanding_polling_interval: 2 + content_understanding_max_retries: 3 + content_understanding_retry_backoff_factor: 2 + content_understanding_model_mappings: '{}' + max_concurrent: 3 + continue_on_error: false + - id: content_classifier-1777659569769 + name: Classify Document Type + type: content_classifier + position: + x: 405 + 'y': 543 + description: Classify text into predefined categories with confidence scores + settings: + enabled: true + condition: '' + fail_pipeline_on_error: true + debug_mode: true + categories: >- + { "Driving License": "Government-issued driver's license, driving permit, or DL used as an official proof of + identity", "Passport": "Government-issued passport document used for international travel and identity + verification", "Real ID": "REAL ID–compliant identification card or driver's license issued by a government + authority under the REAL ID Act, typically marked with a star", "Water Utility bill": "Water service utility + bill or statement issued by a municipal or private water service provider", "Power Utility bill": + "Electricity or power service utility bill or statement issued by an energy or power provider", "Telecom + utility bill_claro": "Telecommunications service bill issued by Claro, including mobile, internet, or related + telecom services", "Telecom utility bill_liberty": "Telecommunications service bill issued by Liberty, + including mobile, internet, or related telecom services", "Telecom utility bill_tmobile": + "Telecommunications service bill issued by T-Mobile, including mobile, internet, or related telecom + services", "Certification of death": "Official death certificate or certification of death issued by a + government authority, hospital, or civil registry", "Unrecognized": "Document does not match any of the + predefined document categories" } + multi_label: false + include_confidence: true + category_descriptions: '' + min_confidence: 0 + include_explanation: true + endpoint: ${AZURE_OPENAI_ENDPOINT} + deployment_name: gpt-4.1 + credential_type: default_azure_credential + api_key: '' + input_field: content_understanding_result + output_field: classification + include_full_response: false + temperature: 0 + max_tokens: 600 + max_retries: 3 + retry_backoff_seconds: 1 + max_concurrent: 3 + timeout_secs: 300 + continue_on_error: true + - id: azure_content_understanding_extractor-1777660571122 + name: Extract Document Details + type: azure_content_understanding_extractor + position: + x: 407 + 'y': 698 + description: Extracts content from documents using Azure AI Content Understanding with 70+ prebuilt analyzers + settings: + enabled: true + condition: classification.category != 'Unrecognized' + fail_pipeline_on_error: true + debug_mode: true + content_field: '' + temp_file_path_field: temp_file_path + output_field: extracted_details + analyzer_id: details_extractor_documents_new1 + retrieve_figures: false + figures_output_field: figures + content_understanding_endpoint: ${AZURE_CONTENT_UNDERSTANDING_ENDPOINT} + content_understanding_credential_type: default_azure_credential + content_understanding_subscription_key: '' + content_understanding_api_version: '2025-11-01' + content_understanding_timeout: 60 + content_understanding_polling_interval: 2 + content_understanding_max_retries: 3 + content_understanding_retry_backoff_factor: 2 + content_understanding_model_mappings: '{}' + max_concurrent: 3 + continue_on_error: false + - id: field_mapper-1777661225834 + name: Shape Document Record + type: field_mapper + position: + x: 399 + 'y': 856 + description: Rename, move, and remap fields within Content items for standardization and compatibility + settings: + enabled: true + condition: '' + fail_pipeline_on_error: true + debug_mode: true + mappings: '' + object_mappings: |- + { + "document_record": { + "filename": "_filename", + "document_type": "classification.category", + "confidence": "classification.confidence", + "details": "extracted_details" + } + } + source_id_mappings: '{"_filename": "id.filename"}' + copy_mode: copy + create_nested: true + overwrite_existing: true + template_fields: true + nested_delimiter: . + list_handling: merge + join_separator: |+ + + merge_filter_empty: true + merge_deduplicate: false + case_transform: '' + fail_on_missing_source: false + remove_empty_objects: false + - id: azure_blob_output-1777661894683 + name: Write Fetched Details + type: azure_blob_output + position: + x: 402 + 'y': 1000 + description: Write content entries as JSON files to Azure Blob Storage + settings: + enabled: true + condition: '' + fail_pipeline_on_error: true + debug_mode: true + storage_account_name: ${AZURE_STORAGE_ACCOUNT_NAME} + credential_type: default_azure_credential + credential_key: '' + container_name: content + path_template: '{case_prefix}' + filename_template: FetchedDetails_{id.unique_id}_{timestamp}.json + content_field: document_record + metadata_fields: '' + compression: '' + overwrite_existing: true + add_timestamp: true + pretty_print: true + max_concurrent: 3 + timeout_secs: 60 + continue_on_error: true + - id: document_validation-1777725315940 + name: Document Validation + type: document_validation + position: + x: 407 + 'y': 1161 + description: >- + Validates fetched document details against provided details using configurable rules. Supports exact match, date + match, expiry check, name match, and AI-powered address matching. + settings: + enabled: true + fail_pipeline_on_error: true + debug_mode: true + blob_storage_account: ${AZURE_STORAGE_ACCOUNT_NAME} + blob_container_name: content + blob_storage_credential_type: default_azure_credential + blob_storage_account_key: '' + provided_details_filename: ProvidedDetails.json + rules_filename: rules.json + fetched_details_prefix: FetchedDetails_ + output_filename: results.json + base_path: '' + fetched_details_path: '' + fetched_details_path_field: blob_output.blob_path + case_sensitive_comparison: false + input_prefix_field: blob_path + rules_base_path: '' + cleanup_input_after_results: true + cleanup_preserve_files: results.json + endpoint: ${AZURE_OPENAI_ENDPOINT} + deployment_name: '' + credential_type: default_azure_credential + api_key: https://aiaifdoa37xivbqipl5.openai.azure.com/ + temperature: 0.1 + edges: + - from: azure_blob_input_discovery-1777658905426 + to: azure_blob_content_retriever-1777658985345 + type: sequential + - from: azure_blob_content_retriever-1777658985345 + to: azure_content_understanding_extractor-1777662467773 + type: sequential + - from: azure_content_understanding_extractor-1777662467773 + to: content_classifier-1777659569769 + type: sequential + - from: content_classifier-1777659569769 + to: azure_content_understanding_extractor-1777660571122 + type: sequential + - from: azure_content_understanding_extractor-1777660571122 + to: field_mapper-1777661225834 + type: sequential + - from: field_mapper-1777661225834 + to: azure_blob_output-1777661894683 + type: sequential + - from: azure_blob_output-1777661894683 + to: document_validation-1777725315940 + type: sequential diff --git a/contentflow-lib/contentflow/executors/__init__.py b/contentflow-lib/contentflow/executors/__init__.py index 2131973..11dab83 100644 --- a/contentflow-lib/contentflow/executors/__init__.py +++ b/contentflow-lib/contentflow/executors/__init__.py @@ -39,6 +39,7 @@ from .web_scraping_executor import WebScrapingExecutor from .pass_through import PassThroughExecutor from .cosmos_db_lookup_executor import CosmosDBLookupExecutor +from .document_validation_executor import DocumentValidationExecutor # Document Set executors from .document_set_initializer import DocumentSetInitializerExecutor diff --git a/contentflow-lib/contentflow/executors/azure_blob_input_discovery.py b/contentflow-lib/contentflow/executors/azure_blob_input_discovery.py index 6690377..1a7c036 100644 --- a/contentflow-lib/contentflow/executors/azure_blob_input_discovery.py +++ b/contentflow-lib/contentflow/executors/azure_blob_input_discovery.py @@ -309,6 +309,14 @@ async def process_input( f"container '{self.blob_container_name}' in {elapsed:.2f}s" ) + # Propagate data fields from input content to each discovered item + # so downstream executors can reference parent context (e.g., execution_id) + if isinstance(input, Content) and input.data: + for item in content_items: + for key, value in input.data.items(): + if key not in item.data: + item.data[key] = value + # Virtual folder discovery mode if self.discover_mode == "virtual_folders": return self._extract_virtual_folders(content_items) diff --git a/contentflow-lib/contentflow/executors/azure_blob_output_executor.py b/contentflow-lib/contentflow/executors/azure_blob_output_executor.py index 116473d..b796053 100644 --- a/contentflow-lib/contentflow/executors/azure_blob_output_executor.py +++ b/contentflow-lib/contentflow/executors/azure_blob_output_executor.py @@ -246,8 +246,7 @@ def _format_template(self, template: str, content: Content) -> str: for field_path in matches: value = self._get_nested_value(source_data, field_path) if value is not None: - # Convert to string, handling special characters - str_value = str(value).replace('/', '_').replace('\\', '_') + str_value = str(value) result = result.replace(f'{{{field_path}}}', str_value) else: # Replace with 'unknown' if field not found diff --git a/contentflow-lib/contentflow/executors/azure_openai_agent_executor.py b/contentflow-lib/contentflow/executors/azure_openai_agent_executor.py index 0c842ef..38eaa07 100644 --- a/contentflow-lib/contentflow/executors/azure_openai_agent_executor.py +++ b/contentflow-lib/contentflow/executors/azure_openai_agent_executor.py @@ -206,10 +206,11 @@ async def process_content_item( # Update summary content.summary_data['agent_execution_status'] = "success" - content.summary_data['response_length'] = len(response_text) if response_text else 0 + content.summary_data['response_length'] = len(str(response_text)) if response_text else 0 if self.debug_mode: - logger.debug(f"Agent response for {content.id}: {response_text[:100]}...") + _debug_text = str(response_text)[:100] if response_text else "" + logger.debug(f"Agent response for {content.id}: {_debug_text}...") except Exception as e: logger.error( diff --git a/contentflow-lib/contentflow/executors/base.py b/contentflow-lib/contentflow/executors/base.py index e4488aa..d7bef5f 100644 --- a/contentflow-lib/contentflow/executors/base.py +++ b/contentflow-lib/contentflow/executors/base.py @@ -172,7 +172,22 @@ def get_setting( ) return default - return self._resolve_setting_value(value) + resolved = self._resolve_setting_value(value) + + # Warn if a sensitive setting appears to contain a literal value instead of ${ENV_VAR} + _sensitive_keywords = ('key', 'secret', 'password', 'credential', 'token', 'api_key') + if ( + isinstance(value, str) + and any(kw in setting_key.lower() for kw in _sensitive_keywords) + and not (value.startswith('${') and value.endswith('}')) + and value != default + ): + logger.warning( + f"Executor '{self.id}': Setting '{setting_key}' appears to contain a literal secret. " + f"Use ${{ENV_VAR}} syntax to reference environment variables for ZTA compliance." + ) + + return resolved def try_extract_nested_field_from_content( self, @@ -181,22 +196,50 @@ def try_extract_nested_field_from_content( ) -> Any: """ Extract a nested field value from a Content item's data. - + + Supports dot-separated paths and array indexing with bracket notation. + Examples: + "text" -> content.data["text"] + "result.contents[0].markdown" -> content.data["result"]["contents"][0]["markdown"] + Args: content: Content item to extract from - field_path: Dot-separated path to the field + field_path: Dot-separated path to the field, with optional [N] array indexing Returns: Extracted field value or None if not found """ fields = field_path.split('.') current_value = content.data - + for field in fields: - if isinstance(current_value, dict) and field in current_value: - current_value = current_value[field] + # Check for array index notation e.g. "contents[0]" + if '[' in field and field.endswith(']'): + bracket_pos = field.index('[') + key = field[:bracket_pos] + index_str = field[bracket_pos + 1:-1] + + # Access the dict key first + if key: + if isinstance(current_value, dict) and key in current_value: + current_value = current_value[key] + else: + return None + + # Then access the list index + try: + index = int(index_str) + if isinstance(current_value, list) and 0 <= index < len(current_value): + current_value = current_value[index] + else: + return None + except (ValueError, TypeError): + return None else: - return None - + if isinstance(current_value, dict) and field in current_value: + current_value = current_value[field] + else: + return None + return current_value diff --git a/contentflow-lib/contentflow/executors/content_classifier_executor.py b/contentflow-lib/contentflow/executors/content_classifier_executor.py index 73b88f8..a704163 100644 --- a/contentflow-lib/contentflow/executors/content_classifier_executor.py +++ b/contentflow-lib/contentflow/executors/content_classifier_executor.py @@ -72,8 +72,14 @@ def __init__( settings = settings or {} categories = settings.get("categories", None) + # Handle categories passed as JSON string from UI + if isinstance(categories, str): + categories = [c.strip() for c in categories.strip("[]").split(",") if c.strip()] + categories = [c.strip("'\"") for c in categories] + settings["categories"] = categories + if not categories or not isinstance(categories, list) or len(categories) == 0: - raise ValueError(f"{self.id}: ContentClassifierExecutor requires 'categories' setting with at least one category") + raise ValueError(f"{id}: ContentClassifierExecutor requires 'categories' setting with at least one category") multi_label = settings.get("multi_label", False) include_confidence = settings.get("include_confidence", True) diff --git a/contentflow-lib/contentflow/executors/document_validation_executor.py b/contentflow-lib/contentflow/executors/document_validation_executor.py new file mode 100644 index 0000000..a34d233 --- /dev/null +++ b/contentflow-lib/contentflow/executors/document_validation_executor.py @@ -0,0 +1,1088 @@ +""" +Document Validation Executor for comparing fetched document details +against provided details using configurable validation rules. + +This executor reads FetchedDetails_*.json, ProvidedDetails.json, and rules.json +from an Azure Blob Storage folder, performs field-by-field comparison per document type, +and writes a consolidated results.json back to the same blob location. +""" + +import asyncio +import json +import logging +from datetime import datetime, timezone +from typing import Dict, Any, Optional, List, Union + +try: + from agent_framework.openai import OpenAIChatClient + from agent_framework import Agent, AgentResponse +except ImportError: + raise ImportError( + "agent-framework import error. Either the library is not installed or there is " + "an issue with the version of the installed library." + ) + +from agent_framework import WorkflowContext + +from .base import BaseExecutor +from ..models import Content, ExecutorLogEntry +from ..connectors import AzureBlobConnector +from ..utils.credential_provider import get_azure_credential + +logger = logging.getLogger("contentflow.executors.document_validation_executor") + + +class DocumentValidationExecutor(BaseExecutor): + """ + Validate fetched document details against provided details using rules. + + This executor reads FetchedDetails_*.json, ProvidedDetails.json, and rules.json from + Azure Blob Storage, performs field-by-field comparison per document type using + configurable validation types (exact match, date match, expiry check, name match, + address match with AI-powered fuzzy comparison), and writes a consolidated + results.json back to the same blob location. + + Configuration (settings dict): + - blob_storage_account (str): Azure Storage account name + Required: True + - blob_container_name (str): Container where input files reside + Required: True + - blob_storage_credential_type (str): Credential type for blob storage + Default: "default_azure_credential" + Options: "default_azure_credential", "azure_key_credential" + - blob_storage_account_key (str): Storage account key (if using azure_key_credential) + Default: None + - provided_details_filename (str): Name of the provided details file + Default: "ProvidedDetails.json" + - rules_filename (str): Name of the rules file + Default: "rules.json" + - fetched_details_prefix (str): Prefix for fetched details files + Default: "FetchedDetails_" + - output_filename (str): Name of the output results file + Default: "results.json" + - case_sensitive_comparison (bool): Whether string comparisons are case-sensitive + Default: False + - input_prefix_field (str): Field in content.data containing the blob prefix/folder path + Default: "blob_path" + - endpoint (str): Azure OpenAI endpoint URL for AI-powered address matching + Default: None + - deployment_name (str): Azure OpenAI model deployment name + Default: None + - credential_type (str): Azure credential type for OpenAI + Default: "default_azure_credential" + Options: "default_azure_credential", "azure_key_credential" + - api_key (str): API key for Azure OpenAI (if using azure_key_credential) + Default: None + - temperature (float): Sampling temperature for AI address matching + Default: 0.1 + + Example: + ```yaml + - id: document_validator + type: document_validation + settings: + blob_storage_account: "${AZURE_STORAGE_ACCOUNT_NAME}" + blob_container_name: "inputs" + provided_details_filename: "ProvidedDetails.json" + rules_filename: "rules.json" + output_filename: "results.json" + endpoint: "${AZURE_OPENAI_ENDPOINT}" + deployment_name: "gpt-4.1" + ``` + + Input: + Content item representing a case folder (with blob_path in data from + upstream blob discovery using virtual_folders mode) + + Output: + Content with data["validation_results"] containing the consolidated results, + and results.json written to the same blob folder. + """ + + def __init__( + self, + id: str, + settings: Optional[Dict[str, Any]] = None, + **kwargs + ): + super().__init__(id=id, settings=settings, **kwargs) + + # Blob storage config + self.blob_storage_account = self.get_setting("blob_storage_account", required=True) + self.blob_container_name = self.get_setting("blob_container_name", required=True) + self.blob_storage_credential_type = self.get_setting( + "blob_storage_credential_type", default="default_azure_credential" + ) + self.blob_storage_account_key = self.get_setting("blob_storage_account_key", default=None) + + # File names + self.provided_details_filename = self.get_setting("provided_details_filename", default="ProvidedDetails.json") + self.rules_filename = self.get_setting("rules_filename", default="rules.json") + self.fetched_details_prefix = self.get_setting("fetched_details_prefix", default="FetchedDetails_") + self.output_filename = self.get_setting("output_filename", default="results.json") + + # Path settings + self.base_path = self.get_setting("base_path", default=None) + self.fetched_details_path = self.get_setting("fetched_details_path", default=None) + self.fetched_details_path_field = self.get_setting("fetched_details_path_field", default="blob_output.blob_path") + # Read rules_base_path directly from settings dict because get_setting + # converts empty string to None, but empty string is a valid value + # meaning "container root". + self.rules_base_path = self.settings.get("rules_base_path", None) + if isinstance(self.rules_base_path, str): + self.rules_base_path = self.rules_base_path.strip() + + # Cleanup settings + self.cleanup_input_after_results = self.get_setting("cleanup_input_after_results", default=False) + self.cleanup_preserve_files = self.get_setting("cleanup_preserve_files", default="results.json") + + # Comparison settings + self.case_sensitive = self.get_setting("case_sensitive_comparison", default=False) + self.input_prefix_field = self.get_setting("input_prefix_field", default="blob_path") + + # Azure OpenAI config for AI-powered address matching + self.openai_endpoint = self.get_setting("endpoint", default=None) + self.openai_deployment_name = self.get_setting("deployment_name", default=None) + self.openai_credential_type = self.get_setting("credential_type", default="default_azure_credential") + self.openai_api_key = self.get_setting("api_key", default=None) + self.temperature = self.get_setting("temperature", default=0.0) + + # Validate OpenAI credential config + if self.openai_credential_type not in ["default_azure_credential", "azure_key_credential"]: + raise ValueError(f"{self.id}: Invalid credential_type '{self.openai_credential_type}'") + if self.openai_credential_type == "azure_key_credential" and not self.openai_api_key: + raise ValueError(f"{self.id}: api_key must be provided for azure_key_credential") + + # Initialize blob connector + self.blob_connector = AzureBlobConnector( + name="validation_blob_connector", + settings={ + "account_name": self.blob_storage_account, + "credential_type": self.blob_storage_credential_type, + "credential_key": self.blob_storage_account_key, + } + ) + + # AI agent (lazily initialized) + self.agent: Optional[Agent] = None + + if self.debug_mode: + logger.debug(f"DocumentValidationExecutor {self.id} initialized") + + def _init_agent(self) -> None: + """Initialize the AI agent for address matching.""" + client_kwargs = { + "model": self.openai_deployment_name, + "azure_endpoint": self.openai_endpoint, + "credential": get_azure_credential() if self.openai_credential_type == "default_azure_credential" else None, + "api_key": self.openai_api_key if self.openai_credential_type == "azure_key_credential" else None, + } + + client = OpenAIChatClient(**client_kwargs) + + instructions = ( + "You are an address comparison expert. You will be given two addresses and must determine " + "if they refer to the same physical location. Consider abbreviations, formatting differences, " + "partial matches, and regional address conventions (especially Puerto Rico/US formats). " + "Respond ONLY with a JSON object: {\"match\": true/false, \"confidence\": 0.0-1.0, \"reason\": \"brief explanation\"}" + ) + + self.agent = client.as_agent( + id=f"{self.id}_address_agent", + name=f"{self.id}_address_agent", + instructions=instructions, + default_options={ + "temperature": self.temperature, + "max_tokens": 150 + }, + ) + + async def process_input( + self, + input: Union[Content, List[Content]], + ctx: WorkflowContext[Union[Content, List[Content]], Union[Content, List[Content]]] + ) -> Union[Content, List[Content]]: + """Main processing: load files, compare, produce results. + + Collects the exact blob paths written by upstream executors in the current run + to ensure only current-execution FetchedDetails files are validated. + """ + contents = input if isinstance(input, list) else [input] + + # Collect all specific FetchedDetails blob paths from current execution's content items. + # Each content item processed by the upstream blob output executor carries its own + # blob_output.blob_path in summary_data — these are the ONLY files from this run. + current_run_blob_paths = self._collect_current_run_fetched_paths(contents) + + if isinstance(input, list): + results = [] + for content in contents: + results.append(await self._process_single(content, current_run_blob_paths)) + # Run cleanup ONCE after all content items are processed + if self.cleanup_input_after_results and results: + base_prefix = self._get_base_prefix(contents[0]) + await self.blob_connector.initialize() + await self._cleanup_case_folder(base_prefix) + return results + + result = await self._process_single(contents[0], current_run_blob_paths) + # Run cleanup after the single item is processed + if self.cleanup_input_after_results: + base_prefix = self._get_base_prefix(contents[0]) + await self.blob_connector.initialize() + await self._cleanup_case_folder(base_prefix) + return result + + def _collect_current_run_fetched_paths(self, contents: List[Content]) -> List[str]: + """Collect all FetchedDetails blob paths written by upstream executors in the current run. + + Iterates all content items and extracts their blob_output.blob_path values, + filtering only those that match the fetched details prefix pattern. + This ensures we ONLY process files from the current pipeline execution. + """ + paths = [] + for content in contents: + blob_path = self._resolve_nested_field(content.summary_data, self.fetched_details_path_field) + if blob_path and isinstance(blob_path, str): + filename = blob_path.split("/")[-1] if "/" in blob_path else blob_path + if filename.startswith(self.fetched_details_prefix) and filename.endswith(".json"): + paths.append(blob_path) + + # Also check executor_logs from earlier executors in the same pipeline run + # that may have written multiple files (e.g., content items processed in parallel) + for content in contents: + for log in content.executor_logs: + if log.details and "blob_path" in log.details: + bp = log.details["blob_path"] + if isinstance(bp, str): + filename = bp.split("/")[-1] if "/" in bp else bp + if filename.startswith(self.fetched_details_prefix) and filename.endswith(".json"): + if bp not in paths: + paths.append(bp) + + return paths + + async def _process_single(self, content: Content, current_run_blob_paths: List[str]) -> Content: + """Process a single case (folder).""" + start_time = datetime.now(timezone.utc) + content_id = content.id.canonical_id if content.id else "unknown" + logger.info(f"{self.id}: Starting validation for content: {content_id}") + + try: + # Initialize blob connector + init_start = datetime.now(timezone.utc) + await self.blob_connector.initialize() + if self.debug_mode: + init_elapsed = (datetime.now(timezone.utc) - init_start).total_seconds() + logger.debug(f"{self.id}: Blob connector initialized in {init_elapsed:.2f}s") + + # Determine the blob prefix for ProvidedDetails, rules, and output + base_prefix = self._get_base_prefix(content) + # Determine where FetchedDetails files are stored (may differ from base) + fetched_prefix = self._get_fetched_details_prefix(content) + + logger.info(f"{self.id}: Base path: {base_prefix}, Fetched details path: {fetched_prefix}") + + # Step 1: Load ProvidedDetails.json + provided_details = await self._load_json_from_blob(base_prefix, self.provided_details_filename) + if self.debug_mode: + logger.debug(f"{self.id}: Loaded ProvidedDetails with {len(provided_details.get('documents', []))} documents") + + # Step 2: Load rules.json (from rules_base_path if set, else from base_prefix) + rules_prefix = self._get_rules_prefix() + if rules_prefix is None: + rules_prefix = base_prefix + rules = await self._load_json_from_blob(rules_prefix, self.rules_filename) + if self.debug_mode: + logger.debug(f"{self.id}: Loaded rules with {len(rules.get('rules', []))} document type rules") + + # Step 3: Load FetchedDetails - prefer exact paths from current run over folder listing + fetched_details_list = await self._load_fetched_details_for_current_run( + current_run_blob_paths, fetched_prefix + ) + if self.debug_mode: + logger.debug(f"{self.id}: Loaded {len(fetched_details_list)} fetched detail entries") + + # Step 4: Run validation + results = await self._run_validation(provided_details, fetched_details_list, rules) + + # Step 5: Write results.json to base path + await self._write_results_to_blob(base_prefix, results) + + # Step 6: Store results in content for downstream use + # Note: cleanup is handled in process_input() after ALL items are done + content.data["validation_results"] = results + content.summary_data["validation_status"] = results.get("summary", {}).get("overallStatus", "unknown") + content.summary_data["executor_status"] = "success" + + elapsed = (datetime.now(timezone.utc) - start_time).total_seconds() + logger.info( + f"{self.id}: Validation complete for {content_id} in {elapsed:.2f}s - " + f"Status: {results['summary']['overallStatus']}, " + f"Passed: {results['summary']['passed']}, Failed: {results['summary']['failed']}" + ) + + # Append executor log entry for pipeline status tracking + content.executor_logs.append(ExecutorLogEntry( + executor_id=self.id, + start_time=start_time, + end_time=datetime.now(timezone.utc), + status="completed", + details={ + "total_documents": results["summary"]["totalDocuments"], + "passed": results["summary"]["passed"], + "failed": results["summary"]["failed"], + }, + errors=[] + )) + + except Exception as e: + elapsed = (datetime.now(timezone.utc) - start_time).total_seconds() + logger.error( + f"{self.id}: Validation failed for {content_id} after {elapsed:.2f}s: {e}", + exc_info=True + ) + content.summary_data["executor_status"] = "failed" + content.executor_logs.append(ExecutorLogEntry( + executor_id=self.id, + start_time=start_time, + end_time=datetime.now(timezone.utc), + status="failed", + details={}, + errors=[str(e)] + )) + raise + + return content + + def _get_base_prefix(self, content: Content) -> str: + """Get the base path for ProvidedDetails, rules, and results output. + + Priority: static base_path setting > content field > content.id.path + """ + # If static base_path is set, use it directly + if self.base_path: + path = self.base_path.strip("/") + return f"{path}/" if path else "" + + # Fall back to dynamic resolution from content + prefix = self.try_extract_nested_field_from_content(content, self.input_prefix_field) + if prefix: + return prefix if prefix.endswith("/") else prefix + "/" + if content.id and content.id.path: + path = content.id.path + # content.id.path is typically a file path (e.g., input/case_001/doc.pdf) + # Extract the parent directory to get the case folder prefix + if "/" in path: + path = path.rsplit("/", 1)[0] + return f"{path}/" if path else "" + raise ValueError(f"{self.id}: Cannot determine base path. Set 'base_path' in settings.") + + def _get_fetched_details_prefix(self, content: Content) -> str: + """Get the path where FetchedDetails files are stored. + + Priority: + 1. Static fetched_details_path setting (manual override) + 2. Derived from upstream blob output (content.summary_data nested field) + 3. Fall back to base_path + """ + # 1. Static override + if self.fetched_details_path: + path = self.fetched_details_path.strip("/") + return f"{path}/" if path else "" + + # 2. Derive from upstream blob output path + # The blob output executor stores: content.summary_data['blob_output']['blob_path'] + # e.g., "azure_blob_output-xxx_2026_05_01/FetchedDetails_abc.json" + upstream_path = self._resolve_nested_field(content.summary_data, self.fetched_details_path_field) + + if upstream_path: + # Extract directory from full blob path + if "/" in upstream_path: + folder = upstream_path.rsplit("/", 1)[0] + return f"{folder}/" + # If no slash, it's a file at root level + return "" + + # 3. Fall back to base path + return self._get_base_prefix(content) + + def _get_rules_prefix(self) -> str: + """Get the path prefix for loading rules.json. + + If rules_base_path is set (even to empty string), use it. + Returns None if not configured, so caller can fall back to base_prefix. + """ + if self.rules_base_path is not None: + path = self.rules_base_path.strip("/") + return f"{path}/" if path else "" + return None + + async def _cleanup_case_folder(self, base_prefix: str) -> None: + """Delete all blobs under the case folder prefix except preserved files. + + Preserves files listed in cleanup_preserve_files (comma-separated). + """ + preserve_set = set() + if self.cleanup_preserve_files: + preserve_set = {f.strip() for f in self.cleanup_preserve_files.split(",") if f.strip()} + + deleted_count = 0 + async for blobs in self.blob_connector.list_blobs( + container_name=self.blob_container_name, + prefix=base_prefix, + max_results=1000, + batch_size=100 + ): + if not blobs: + continue + for blob in blobs: + blob_name = blob.get("name", "") + filename = blob_name.split("/")[-1] if "/" in blob_name else blob_name + if filename in preserve_set: + if self.debug_mode: + logger.debug(f"{self.id}: Preserving {blob_name}") + continue + try: + await self.blob_connector.delete_blob( + container_name=self.blob_container_name, + blob_path=blob_name + ) + deleted_count += 1 + except Exception as e: + logger.warning(f"{self.id}: Failed to delete {blob_name}: {e}") + + logger.info(f"{self.id}: Cleanup complete — deleted {deleted_count} blob(s) from {base_prefix}") + + def _resolve_nested_field(self, data: dict, field_path: str): + """Resolve a dot-notation field path from a dictionary. + + E.g., 'blob_output.blob_path' resolves data['blob_output']['blob_path'] + """ + if not data or not field_path: + return None + parts = field_path.split(".") + value = data + for part in parts: + if isinstance(value, dict) and part in value: + value = value[part] + else: + return None + return value if value is not data else None + + async def _load_json_from_blob(self, prefix: str, filename: str) -> dict: + """Download and parse a JSON file from blob storage.""" + blob_path = f"{prefix}{filename}" + try: + content_bytes = await self.blob_connector.download_blob( + container_name=self.blob_container_name, + blob_path=blob_path + ) + return json.loads(content_bytes.decode("utf-8")) + except Exception as e: + logger.error(f"{self.id}: Failed to load {blob_path}: {e}", exc_info=True) + raise ValueError(f"Failed to load required file '{filename}' from {blob_path}: {e}") + + async def _load_fetched_details_for_current_run( + self, current_run_blob_paths: List[str], fetched_prefix: str + ) -> List[dict]: + """Load FetchedDetails files scoped to the current pipeline execution only. + + Strategy: + 1. If exact blob paths from the current run are available (collected from + upstream blob output executor's summary_data), load ONLY those specific files. + This guarantees no cross-run contamination. + 2. Fallback: If no specific paths are available (e.g., manual/static config), + list the folder but log a warning about potential cross-run inclusion. + """ + if current_run_blob_paths: + # PREFERRED: Load only the exact files written in this execution + logger.info( + f"{self.id}: Loading {len(current_run_blob_paths)} FetchedDetails file(s) " + f"from current run (exact path resolution)" + ) + return await self._load_fetched_details_by_paths(current_run_blob_paths) + + # FALLBACK: No exact paths available — use folder listing with warning + logger.warning( + f"{self.id}: No exact FetchedDetails paths from current run available. " + f"Falling back to folder listing at '{fetched_prefix}'. " + f"This may include files from previous executions if the folder is shared." + ) + return await self._load_all_fetched_details_from_folder(fetched_prefix) + + async def _load_fetched_details_by_paths(self, blob_paths: List[str]) -> List[dict]: + """Load FetchedDetails from specific blob paths (current run only).""" + all_fetched = [] + for blob_path in blob_paths: + try: + content_bytes = await self.blob_connector.download_blob( + container_name=self.blob_container_name, + blob_path=blob_path + ) + fetched = json.loads(content_bytes.decode("utf-8")) + if isinstance(fetched, list): + all_fetched.extend(fetched) + else: + all_fetched.append(fetched) + if self.debug_mode: + logger.debug(f"{self.id}: Loaded FetchedDetails from exact path: {blob_path}") + except Exception as e: + logger.warning(f"{self.id}: Failed to load fetched details from {blob_path}: {e}") + return all_fetched + + async def _load_all_fetched_details_from_folder(self, fetched_prefix: str) -> List[dict]: + """Fallback: Find and load all FetchedDetails_*.json files in the fetched details path. + + WARNING: This may include files from previous pipeline executions if the folder + is shared across runs (e.g., date-based folder naming). + """ + all_fetched = [] + async for blobs in self.blob_connector.list_blobs( + container_name=self.blob_container_name, + prefix=fetched_prefix, + max_results=100, + batch_size=100 + ): + if not blobs: + continue + for blob in blobs: + blob_name = blob.get("name", "") + filename = blob_name.split("/")[-1] if "/" in blob_name else blob_name + if filename.startswith(self.fetched_details_prefix) and filename.endswith(".json"): + try: + content_bytes = await self.blob_connector.download_blob( + container_name=self.blob_container_name, + blob_path=blob_name + ) + fetched = json.loads(content_bytes.decode("utf-8")) + if isinstance(fetched, list): + all_fetched.extend(fetched) + else: + all_fetched.append(fetched) + except Exception as e: + logger.warning(f"{self.id}: Failed to load fetched details from {blob_name}: {e}") + return all_fetched + + async def _run_validation( + self, + provided_details: dict, + fetched_details_list: List[dict], + rules: dict + ) -> dict: + """ + Core validation logic: + - Match fetched documents to provided documents by documentType + - Apply rules for each document type + - Produce consolidated results + """ + results = { + "validationTimestamp": datetime.now(timezone.utc).isoformat(), + "summary": { + "totalDocuments": 0, + "passed": 0, + "failed": 0, + "invalid": 0, + "warnings": 0, + "notFound": 0, + "overallStatus": "passed" + }, + "documentResults": [] + } + + # Support two ProvidedDetails formats: + # 1. Per-document: {"documents": [{"documentType": "Passport", "details": {...}}, ...]} + # 2. Flat/shared: {"caseId": "...", "details": {"firstName": "...", ...}} + # In flat format, the same details apply to ALL document types. + if "documents" in provided_details and provided_details["documents"]: + provided_docs = { + doc["documentType"]: doc["details"] + for doc in provided_details["documents"] + } + shared_details = None + else: + provided_docs = {} + shared_details = provided_details.get("details", {}) + + rules_by_type = { + rule["documentType"]: rule["validations"] + for rule in rules.get("rules", []) + } + + for fetched in fetched_details_list: + fetched_doc_type = fetched.get("document_type", "") + doc_result = { + "documentType": fetched_doc_type, + "filename": fetched.get("filename", ""), + "status": "passed", + "errors": [], + "fieldResults": [] + } + results["summary"]["totalDocuments"] += 1 + + # Find matching provided document details + # If flat format (shared_details), use the same details for all doc types + if shared_details is not None: + provided = shared_details + else: + provided = self._find_matching_provided(fetched_doc_type, provided_docs) + if provided is None: + doc_result["status"] = "not_found" + doc_result["errors"].append({ + "errorCode": "DOC_NOT_IN_PROVIDED", + "message_en": f"Document type '{fetched_doc_type}' not found in ProvidedDetails.", + "message_es": f"Tipo de documento '{fetched_doc_type}' no encontrado en los detalles proporcionados." + }) + results["summary"]["notFound"] += 1 + results["documentResults"].append(doc_result) + continue + + # Find matching rules + validations = self._find_matching_rules(fetched_doc_type, rules_by_type) + if not validations: + # No rules — document type is not recognized or has no validation rules + doc_result["status"] = "invalid" + doc_result["errors"].append({ + "errorCode": "UNRECOGNIZED_DOCUMENT", + "message_en": f"Document type '{fetched_doc_type}' could not be validated — no matching rules.", + "message_es": f"El tipo de documento '{fetched_doc_type}' no pudo ser validado — sin reglas coincidentes." + }) + results["summary"]["invalid"] += 1 + results["documentResults"].append(doc_result) + continue + + # Extract fetched field values from Azure CU output + fetched_fields = self._extract_fetched_fields(fetched) + + # Apply each validation rule + for validation in validations: + field_result = await self._validate_field(validation, fetched_fields, provided, fetched_doc_type) + doc_result["fieldResults"].append(field_result) + if field_result["result"] == "fail": + doc_result["status"] = "failed" + + # Update summary + if doc_result["status"] == "failed": + results["summary"]["failed"] += 1 + else: + results["summary"]["passed"] += 1 + + if self.debug_mode: + passed_fields = sum(1 for fr in doc_result["fieldResults"] if fr["result"] == "pass") + failed_fields = sum(1 for fr in doc_result["fieldResults"] if fr["result"] == "fail") + logger.debug( + f"{self.id}: Document '{fetched_doc_type}' - " + f"{len(doc_result['fieldResults'])} fields validated: " + f"{passed_fields} passed, {failed_fields} failed" + ) + + results["documentResults"].append(doc_result) + + # Set overall status + if results["summary"]["failed"] > 0 or results["summary"]["notFound"] > 0 or results["summary"]["invalid"] > 0: + results["summary"]["overallStatus"] = "failed" + + return results + + def _find_matching_provided(self, fetched_type: str, provided_docs: dict) -> Optional[dict]: + """Match fetched document_type to ProvidedDetails documentType.""" + # Exact match first + if fetched_type in provided_docs: + return provided_docs[fetched_type] + # Case-insensitive match + for key, val in provided_docs.items(): + if key.lower() == fetched_type.lower(): + return val + # Partial/contains match (e.g., "Telecom Utility bill" matches "Telecom utility bill_claro") + for key, val in provided_docs.items(): + if fetched_type.lower() in key.lower() or key.lower() in fetched_type.lower(): + return val + return None + + def _find_matching_rules(self, fetched_type: str, rules_by_type: dict) -> Optional[List[dict]]: + """Match fetched document type to rules.""" + # Exact match + if fetched_type in rules_by_type: + return rules_by_type[fetched_type] + # Case-insensitive match + for key, val in rules_by_type.items(): + if key.lower() == fetched_type.lower(): + return val + # Partial match for variants (e.g., "Telecom Utility bill" partial matches "Telecom utility bill_claro") + for key, val in rules_by_type.items(): + if fetched_type.lower() in key.lower() or key.lower() in fetched_type.lower(): + return val + return None + + def _extract_fetched_fields(self, fetched: dict) -> Dict[str, Any]: + """ + Extract field values from the Azure Content Understanding response. + Returns dict like {"FirstName": "LIZANDRA", "LastName": "MARTES SIERRA", ...} + """ + fields = {} + try: + contents = fetched.get("details", {}).get("result", {}).get("contents", []) + if contents: + raw_fields = contents[0].get("fields", {}) + for field_name, field_data in raw_fields.items(): + if field_data.get("type") == "date": + value = field_data.get("valueDate") + else: + value = field_data.get("valueString") + if value: + fields[field_name] = value + except (KeyError, IndexError, TypeError) as e: + logger.warning(f"{self.id}: Error extracting fetched fields: {e}") + return fields + + async def _validate_field( + self, + validation: dict, + fetched_fields: Dict[str, Any], + provided: dict, + doc_type: str + ) -> dict: + """Apply a single validation rule and return the field result.""" + field = validation.get("field", "") + validation_type = validation.get("validationType", "exact_match") + error_code = validation.get("errorCode", "") + + result = { + "field": field, + "validationType": validation_type, + "errorCode": error_code, + "result": "pass", + "fetchedValue": None, + "providedValue": None, + "message_en": None, + "message_es": None + } + + # Dispatch based on validation type + if validation_type == "expiry_check": + await self._check_expiry(result, field, fetched_fields, provided, validation) + elif validation_type == "exact_match": + self._check_exact_match(result, field, fetched_fields, provided, validation) + elif validation_type == "date_match": + self._check_date_match(result, field, fetched_fields, provided, validation) + elif validation_type == "name_match": + self._check_name_match(result, field, fetched_fields, provided, validation) + elif validation_type == "address_match": + await self._check_address_match(result, field, fetched_fields, provided, validation) + else: + # Default to exact match + self._check_exact_match(result, field, fetched_fields, provided, validation) + + return result + + async def _check_expiry( + self, result: dict, field: str, + fetched_fields: dict, provided: dict, validation: dict + ) -> None: + """Check if a document has expired.""" + fetched_value = fetched_fields.get(field) + provided_value = provided.get(field) + result["fetchedValue"] = fetched_value + result["providedValue"] = provided_value + + # If provided says "Permanent", it never expires + if provided_value and str(provided_value).lower() == "permanent": + result["result"] = "pass" + return + + # Use fetched expiry date for the check + date_to_check = fetched_value or provided_value + if date_to_check: + try: + exp_date = datetime.strptime(str(date_to_check), "%Y-%m-%d") + if exp_date < datetime.now(): + result["result"] = "fail" + result["message_en"] = validation.get("message_en") + result["message_es"] = validation.get("message_es") + except ValueError: + result["result"] = "warning" + result["message_en"] = f"Could not parse expiry date: {date_to_check}" + else: + result["result"] = "warning" + result["message_en"] = "Expiry date not found in fetched or provided details" + + def _check_exact_match( + self, result: dict, field: str, + fetched_fields: dict, provided: dict, validation: dict + ) -> None: + """Exact string comparison (case-insensitive by default).""" + fetched_value = fetched_fields.get(field) + provided_value = provided.get(field) + result["fetchedValue"] = fetched_value + result["providedValue"] = provided_value + + if fetched_value is None: + result["result"] = "warning" + result["message_en"] = f"Field '{field}' not found in fetched document details" + return + if provided_value is None: + result["result"] = "warning" + result["message_en"] = f"Field '{field}' not found in provided details" + return + + if not self._values_match(str(fetched_value), str(provided_value)): + result["result"] = "fail" + result["message_en"] = validation.get("message_en") + result["message_es"] = validation.get("message_es") + + def _check_date_match( + self, result: dict, field: str, + fetched_fields: dict, provided: dict, validation: dict + ) -> None: + """Date comparison (normalizes to YYYY-MM-DD).""" + fetched_value = fetched_fields.get(field) + provided_value = provided.get(field) + result["fetchedValue"] = fetched_value + result["providedValue"] = provided_value + + if fetched_value is None: + result["result"] = "warning" + result["message_en"] = f"Field '{field}' not found in fetched document details" + return + if provided_value is None: + result["result"] = "warning" + result["message_en"] = f"Field '{field}' not found in provided details" + return + + # Normalize dates for comparison + fetched_normalized = self._normalize_date(str(fetched_value)) + provided_normalized = self._normalize_date(str(provided_value)) + + if fetched_normalized and provided_normalized: + if fetched_normalized != provided_normalized: + result["result"] = "fail" + result["message_en"] = validation.get("message_en") + result["message_es"] = validation.get("message_es") + else: + # Fallback to string comparison + if not self._values_match(str(fetched_value), str(provided_value)): + result["result"] = "fail" + result["message_en"] = validation.get("message_en") + result["message_es"] = validation.get("message_es") + + def _check_name_match( + self, result: dict, field: str, + fetched_fields: dict, provided: dict, validation: dict + ) -> None: + """ + Name comparison for full name fields. + For utility bills: concatenates FirstName + LastName from fetched, compares to FullName in provided. + """ + provided_value = provided.get(field) + + # Build fetched full name from FirstName + LastName if field is FullName + if field == "FullName": + first = fetched_fields.get("FirstName", "") + last = fetched_fields.get("LastName", "") + fetched_value = f"{first} {last}".strip() + else: + fetched_value = fetched_fields.get(field) + + result["fetchedValue"] = fetched_value + result["providedValue"] = provided_value + + if not fetched_value: + result["result"] = "warning" + result["message_en"] = f"Field '{field}' not found in fetched document details" + return + if not provided_value: + result["result"] = "warning" + result["message_en"] = f"Field '{field}' not found in provided details" + return + + if not self._names_match(str(fetched_value), str(provided_value)): + result["result"] = "fail" + result["message_en"] = validation.get("message_en") + result["message_es"] = validation.get("message_es") + + async def _check_address_match( + self, result: dict, field: str, + fetched_fields: dict, provided: dict, validation: dict + ) -> None: + """AI-powered address comparison using Azure OpenAI.""" + fetched_value = fetched_fields.get(field) + provided_value = provided.get(field) + result["fetchedValue"] = fetched_value + result["providedValue"] = provided_value + + if not fetched_value: + result["result"] = "warning" + result["message_en"] = f"Field '{field}' not found in fetched document details" + return + if not provided_value: + result["result"] = "warning" + result["message_en"] = f"Field '{field}' not found in provided details" + return + + # Also append Region if available for more complete address + region = fetched_fields.get("Region") + full_fetched_address = fetched_value + if region and region not in fetched_value: + full_fetched_address = f"{fetched_value}, {region}" + + # Normalize whitespace (newlines, tabs, multiple spaces) before comparison + normalized_fetched = self._normalize_address(str(full_fetched_address)) + normalized_provided = self._normalize_address(str(provided_value)) + + # Short-circuit: if normalized addresses are an exact match, skip OpenAI + nf = normalized_fetched.upper() if not self.case_sensitive else normalized_fetched + np_ = normalized_provided.upper() if not self.case_sensitive else normalized_provided + if nf == np_: + return + + # Use AI-powered comparison if endpoint is configured + if self.openai_endpoint and self.openai_deployment_name: + is_match = await self._ai_address_match(normalized_fetched, normalized_provided) + if not is_match: + result["result"] = "fail" + result["message_en"] = validation.get("message_en") + result["message_es"] = validation.get("message_es") + else: + # Fallback to basic string comparison + if not self._values_match(normalized_fetched, normalized_provided): + result["result"] = "fail" + result["message_en"] = validation.get("message_en") + result["message_es"] = validation.get("message_es") + + async def _ai_address_match(self, address1: str, address2: str) -> bool: + """Use Azure OpenAI to determine if two addresses match.""" + if not self.agent: + self._init_agent() + + query = ( + f"Address 1: {address1}\n" + f"Address 2: {address2}\n\n" + f"Do these two addresses refer to the same physical location?" + ) + + try: + response: AgentResponse = await self.agent.run(query) + response_text = response.content if hasattr(response, "content") else str(response) + + # Parse the JSON response + try: + parsed = json.loads(response_text) + match_result = parsed.get("match", False) + confidence = parsed.get("confidence", 0.0) + + if self.debug_mode: + logger.debug( + f"{self.id}: Address match result: {match_result}, " + f"confidence: {confidence}, reason: {parsed.get('reason', '')}" + ) + + # Use confidence threshold to recover borderline cases + if not match_result and confidence >= 0.75: + logger.info( + f"{self.id}: Overriding match=false with high confidence {confidence}" + ) + return True + + return bool(match_result) + except json.JSONDecodeError: + # If response isn't valid JSON, look for keywords + lower_response = response_text.lower() + return "true" in lower_response or "match" in lower_response + + except Exception as e: + logger.warning( + f"{self.id}: AI address match failed, falling back to string comparison: {e}", + exc_info=True + ) + return self._values_match(address1, address2) + + def _normalize_address(self, address: str) -> str: + """Normalize whitespace in address strings (replace newlines, tabs, multiple spaces with single space).""" + import re + return re.sub(r'\s+', ' ', address).strip() + + def _values_match(self, fetched: str, provided: str) -> bool: + """Compare two string values with normalization.""" + if not fetched or not provided: + return False + import re + f = re.sub(r'\s+', ' ', fetched).strip() + p = re.sub(r'\s+', ' ', provided).strip() + if not self.case_sensitive: + f = f.upper() + p = p.upper() + # Exact match + if f == p: + return True + # Contained match (one contains the other) + if f in p or p in f: + return True + return False + + def _names_match(self, fetched: str, provided: str) -> bool: + """ + Compare names with flexible matching. + Handles: "LASTNAME, FIRSTNAME" vs "FIRSTNAME LASTNAME" formats, + and partial name matching. + """ + if not fetched or not provided: + return False + + f = fetched.strip().upper() + p = provided.strip().upper() + + # Direct match + if f == p: + return True + + # Normalize comma-separated format ("OROZCO DONES, CARMEN" → "CARMEN OROZCO DONES") + f_normalized = self._normalize_name(f) + p_normalized = self._normalize_name(p) + + if f_normalized == p_normalized: + return True + + # Check if all parts of one name appear in the other + f_parts = set(f_normalized.split()) + p_parts = set(p_normalized.split()) + + # If all parts of the shorter name are in the longer name + if f_parts.issubset(p_parts) or p_parts.issubset(f_parts): + return True + + # Check significant overlap (at least first+last name match) + common = f_parts.intersection(p_parts) + if len(common) >= 2: + return True + + return False + + def _normalize_name(self, name: str) -> str: + """Normalize name format: 'LAST, FIRST' → 'FIRST LAST'.""" + if "," in name: + parts = name.split(",", 1) + return f"{parts[1].strip()} {parts[0].strip()}" + return name + + def _normalize_date(self, date_str: str) -> Optional[str]: + """Normalize date to YYYY-MM-DD format.""" + formats = ["%Y-%m-%d", "%m/%d/%Y", "%d/%m/%Y", "%Y-%m-%dT%H:%M:%S", "%d %b %Y", "%b %d, %Y"] + for fmt in formats: + try: + dt = datetime.strptime(date_str.strip(), fmt) + return dt.strftime("%Y-%m-%d") + except ValueError: + continue + return None + + async def _write_results_to_blob(self, prefix: str, results: dict) -> None: + """Write results.json back to the same blob folder.""" + blob_path = f"{prefix}{self.output_filename}" + content_bytes = json.dumps(results, indent=2, ensure_ascii=False).encode("utf-8") + await self.blob_connector.upload_blob( + container_name=self.blob_container_name, + blob_path=blob_path, + data=content_bytes, + overwrite=True + ) + logger.info(f"{self.id}: Wrote validation results to {blob_path}") diff --git a/contentflow-lib/executor_catalog.yaml b/contentflow-lib/executor_catalog.yaml index 1e99ad5..00633a2 100644 --- a/contentflow-lib/executor_catalog.yaml +++ b/contentflow-lib/executor_catalog.yaml @@ -6246,4 +6246,230 @@ executor_catalog: ui_metadata: icon: "database" description_short: "Look up records in Cosmos DB NoSQL" - description_long: "Queries Azure Cosmos DB NoSQL containers to validate or enrich content fields. Supports both direct point-reads (by id and partition key) and parameterised SQL queries. Lookup values are extracted from content.data via configurable field mappings and results are written back into each content item." \ No newline at end of file + description_long: "Queries Azure Cosmos DB NoSQL containers to validate or enrich content fields. Supports both direct point-reads (by id and partition key) and parameterised SQL queries. Lookup values are extracted from content.data via configurable field mappings and results are written back into each content item." + + ######################################################## + # Document Validation Executor + - id: document_validation + name: "Document Validation" + description: "Validates fetched document details against provided details using configurable rules. Supports exact match, date match, expiry check, name match, and AI-powered address matching." + module_path: contentflow.executors.document_validation_executor + class_name: DocumentValidationExecutor + tags: [validation, comparison, rules, documents, azure-openai] + category: "analyse" + version: "1.3" + + settings_schema: + # Common settings + enabled: + type: boolean + title: "Enabled" + description: "Enable or disable this executor" + required: false + default: true + ui_component: "checkbox" + + fail_pipeline_on_error: + type: boolean + title: "Fail Pipeline On Error" + description: "Fail the entire pipeline if validation execution fails" + required: false + default: true + ui_component: "checkbox" + + debug_mode: + type: boolean + title: "Debug Mode" + description: "Enable debug mode for detailed logging" + required: false + default: false + ui_component: "checkbox" + + # Blob storage settings + blob_storage_account: + type: string + title: "Blob Storage Account" + description: "Azure Blob Storage account name where input files reside" + placeholder: "" + required: true + default: null + ui_component: "input" + + blob_container_name: + type: string + title: "Container Name" + description: "Azure Blob Storage container name containing the case folders" + placeholder: "" + required: true + default: null + ui_component: "input" + + blob_storage_credential_type: + type: string + title: "Blob Storage Credential Type" + description: "Credential type for accessing Azure Blob Storage" + required: false + default: "default_azure_credential" + options: ["default_azure_credential", "azure_key_credential"] + ui_component: "select" + + blob_storage_account_key: + type: string + title: "Blob Storage Account Key" + description: "Azure Blob Storage account key (if using azure_key_credential)" + required: false + default: null + ui_component: "password" + + # File configuration + provided_details_filename: + type: string + title: "Provided Details Filename" + description: "Name of the file containing provided/expected details" + required: false + default: "ProvidedDetails.json" + ui_component: "input" + + rules_filename: + type: string + title: "Rules Filename" + description: "Name of the file containing validation rules" + required: false + default: "rules.json" + ui_component: "input" + + fetched_details_prefix: + type: string + title: "Fetched Details Prefix" + description: "Filename prefix for fetched details files (e.g., 'FetchedDetails_')" + required: false + default: "FetchedDetails_" + ui_component: "input" + + output_filename: + type: string + title: "Output Filename" + description: "Name of the output results file" + required: false + default: "results.json" + ui_component: "input" + + # Path settings + base_path: + type: string + title: "Base Path" + description: "Static blob folder path where ProvidedDetails.json, rules.json reside and results.json will be written (e.g., 'inputs'). If not set, derived from content." + placeholder: "inputs" + required: false + default: null + ui_component: "input" + + fetched_details_path: + type: string + title: "Fetched Details Path" + description: "Static blob folder path where FetchedDetails_*.json files are stored. If not set, automatically derived from upstream blob output executor's written path." + placeholder: "azure_blob_output-xxx_2026_05_01" + required: false + default: null + ui_component: "input" + + fetched_details_path_field: + type: string + title: "Fetched Details Path Field" + description: "Dot-notation path in content.summary_data to the blob path written by upstream blob output executor (e.g., 'blob_output.blob_path')" + required: false + default: "blob_output.blob_path" + ui_component: "input" + + # Comparison settings + case_sensitive_comparison: + type: boolean + title: "Case Sensitive Comparison" + description: "Whether string comparisons should be case-sensitive" + required: false + default: false + ui_component: "checkbox" + + input_prefix_field: + type: string + title: "Input Prefix Field" + description: "Field in content.data containing the blob folder path for this case" + required: false + default: "blob_path" + ui_component: "input" + + rules_base_path: + type: string + title: "Rules Base Path" + description: "Separate blob folder path for rules.json. Use empty string for container root. If not set, rules.json is loaded from the same location as base_path." + placeholder: "" + required: false + default: null + ui_component: "input" + + cleanup_input_after_results: + type: boolean + title: "Cleanup Input After Results" + description: "Delete all blobs in the case folder after results.json is written. Preserved files are configured via 'Cleanup Preserve Files'." + required: false + default: false + ui_component: "checkbox" + + cleanup_preserve_files: + type: string + title: "Cleanup Preserve Files" + description: "Comma-separated list of filenames to preserve during cleanup (e.g., 'results.json'). Only used when 'Cleanup Input After Results' is enabled." + required: false + default: "results.json" + ui_component: "input" + + # Azure OpenAI settings for AI-powered address matching + endpoint: + type: string + title: "Azure OpenAI Endpoint" + description: "Azure OpenAI endpoint URL for AI-powered address matching" + placeholder: "https://.openai.azure.com/" + required: false + default: null + ui_component: "input" + + deployment_name: + type: string + title: "Deployment Name" + description: "Azure OpenAI model deployment name (e.g., gpt-4.1)" + placeholder: "gpt-4.1" + required: false + default: null + ui_component: "input" + + credential_type: + type: string + title: "OpenAI Credential Type" + description: "Authentication method for Azure OpenAI" + required: false + default: "default_azure_credential" + options: ["default_azure_credential", "azure_key_credential"] + ui_component: "select" + + api_key: + type: string + title: "OpenAI API Key" + description: "API key for Azure OpenAI (if using azure_key_credential)" + required: false + default: null + ui_component: "password" + + temperature: + type: number + title: "Temperature" + description: "Sampling temperature for AI address matching (0.0-1.0)" + required: false + default: 0.1 + min: 0.0 + max: 1.0 + ui_component: "number" + + ui_metadata: + icon: "check-circle" + description_short: "Validate documents against provided details" + description_long: "Compares fetched document details (from Azure Content Understanding) against provided/expected details using configurable validation rules. Supports exact match, date match, expiry check, name matching, and AI-powered fuzzy address comparison. Reads ProvidedDetails.json, rules.json, and FetchedDetails_*.json from a blob folder and writes consolidated results.json back to the same location." \ No newline at end of file diff --git a/contentflow-web/src/components/Footer.tsx b/contentflow-web/src/components/Footer.tsx index d6e3887..02c1af0 100644 --- a/contentflow-web/src/components/Footer.tsx +++ b/contentflow-web/src/components/Footer.tsx @@ -154,14 +154,12 @@ interface SystemHealth { cosmosDB: "connected" | "offline"; blobStorage: "connected" | "offline"; storageQueue: "connected" | "offline"; - worker?: "connected" | "offline"; lastChecked: Date; serviceDetails: { appConfig?: ServiceStatus; cosmosDB?: ServiceStatus; blobStorage?: ServiceStatus; storageQueue?: ServiceStatus; - worker?: ServiceStatus; }; } @@ -173,7 +171,6 @@ export const Footer = () => { cosmosDB: "offline", blobStorage: "offline", storageQueue: "offline", - worker: "offline", lastChecked: new Date(), serviceDetails: {}, }); @@ -192,14 +189,12 @@ export const Footer = () => { cosmosDB: healthData.services.cosmos_db?.status === "connected" ? "connected" : "offline", blobStorage: healthData.services.blob_storage?.status === "connected" ? "connected" : "offline", storageQueue: healthData.services.storage_queue?.status === "connected" ? "connected" : "offline", - worker: healthData.services.worker?.status === "connected" ? "connected" : "offline", lastChecked: new Date(), serviceDetails: { appConfig: healthData.services.app_config, cosmosDB: healthData.services.cosmos_db, blobStorage: healthData.services.blob_storage, storageQueue: healthData.services.storage_queue, - worker: healthData.services.worker, }, }); } catch (error) { @@ -210,7 +205,6 @@ export const Footer = () => { cosmosDB: "offline", blobStorage: "offline", storageQueue: "offline", - worker: "offline", lastChecked: new Date(), serviceDetails: {}, }); @@ -343,13 +337,6 @@ export const Footer = () => { getStatusColor={getStatusColor} /> - -
Last checked: diff --git a/contentflow-web/src/lib/api/apiTypes.ts b/contentflow-web/src/lib/api/apiTypes.ts index 45b65c5..bf85f4f 100644 --- a/contentflow-web/src/lib/api/apiTypes.ts +++ b/contentflow-web/src/lib/api/apiTypes.ts @@ -33,7 +33,6 @@ export interface HealthCheck { cosmos_db?: ServiceStatus; blob_storage?: ServiceStatus; storage_queue?: ServiceStatus; - worker?: ServiceStatus; }; } diff --git a/contentflow-worker/.env.example b/contentflow-worker/.env.example deleted file mode 100644 index f4d6509..0000000 --- a/contentflow-worker/.env.example +++ /dev/null @@ -1,50 +0,0 @@ -# ContentFlow Worker Environment Configuration - -# Azure App Configuration -AZURE_APP_CONFIG_ENDPOINT=https://your-app-config-resource.azconfig.io -APP_CONFIG_KEY_FILTERS=contentflow.worker.* - -# Worker Settings -WORKER_NAME=contentflow-worker -NUM_PROCESSING_WORKERS=2 -NUM_SOURCE_WORKERS=2 - -# Azure Storage Queue -STORAGE_ACCOUNT_WORKER_QUEUE_URL=https://your-storage-account.queue.core.windows.net -STORAGE_WORKER_QUEUE_NAME=contentflow-execution-requests - -# Queue Polling Settings -QUEUE_POLL_INTERVAL_SECONDS=5 -QUEUE_VISIBILITY_TIMEOUT_SECONDS=300 -QUEUE_MAX_MESSAGES=32 - -# Processing Settings -MAX_TASK_RETRIES=3 -TASK_TIMEOUT_SECONDS=600 - -# Azure Cosmos DB -COSMOS_DB_ENDPOINT= -COSMOS_DB_NAME=contentflow -COSMOS_DB_CONTAINER_PIPELINES=pipelines -COSMOS_DB_CONTAINER_VAULT_EXECUTIONS=vault_executions -COSMOS_DB_CONTAINER_VAULTS=vaults -COSMOS_DB_CONTAINER_VAULT_EXECUTION_LOCKS=vault_exec_locks -COSMOS_DB_CONTAINER_CRAWL_CHECKPOINTS=vault_crawl_checkpoints - -# Azure Blob Storage -BLOB_STORAGE_ACCOUNT_NAME=your-storage-account -BLOB_STORAGE_CONTAINER_NAME=content - -# Source Worker Settings -SOURCE_WORKER_POLL_INTERVAL_SECONDS=60 # Deprecated: use per-executor polling_interval_seconds -DEFAULT_POLLING_INTERVAL_SECONDS=300 # Default 5 minutes if executor doesn't specify -SCHEDULER_SLEEP_INTERVAL_SECONDS=5 # How often scheduler checks for ready pipelines -LOCK_TTL_SECONDS=300 # Distributed lock TTL (5 minutes) - -API_ENABLED=true -API_HOST=0.0.0.0 -API_PORT=8099 - -# Logging -LOG_LEVEL=DEBUG -DEBUG=true \ No newline at end of file diff --git a/contentflow-worker/.gitignore b/contentflow-worker/.gitignore deleted file mode 100644 index c2e9c0d..0000000 --- a/contentflow-worker/.gitignore +++ /dev/null @@ -1,76 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -*.manifest -*.spec - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# IDEs -.vscode/ -.idea/ -*.swp -*.swo -*~ - -# Logs -*.log -worker.log - -# OS -.DS_Store -Thumbs.db - -# Temporary files -*.tmp -*.temp - -/tmp \ No newline at end of file diff --git a/contentflow-worker/Dockerfile b/contentflow-worker/Dockerfile deleted file mode 100644 index 68f4d61..0000000 --- a/contentflow-worker/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# ContentFlow Worker - Multi-processing Content Processing Engine -FROM python:3.13-slim - -# Set working directory -WORKDIR /app - -# Install system dependencies -RUN apt-get update && apt-get install -y \ - gcc \ - g++ \ - && rm -rf /var/lib/apt/lists/* - -# Copy contentflow-lib first (shared dependency) -COPY contentflow-lib /app/contentflow-lib - -# Install contentflow-lib -WORKDIR /app/contentflow-lib -RUN pip install --no-cache-dir -e . - -# Switch to Worker directory -WORKDIR /app/contentflow-worker - -# Copy Worker requirements -COPY contentflow-worker/requirements.txt . - -# Install Python dependencies -RUN pip install --no-cache-dir -r requirements.txt - -# Copy worker code -COPY contentflow-worker/ . - -# # Set Python path to include contentflow-lib -# ENV PYTHONPATH="/app:/app/contentflow-lib:${PYTHONPATH}" - -# Environment variables (can be overridden) -ENV WORKER_NAME=contentflow-worker -ENV NUM_PROCESSING_WORKERS=4 -ENV NUM_SOURCE_WORKERS=2 -ENV LOG_LEVEL=INFO - -# Run the worker -CMD ["python", "main.py"] diff --git a/contentflow-worker/README.md b/contentflow-worker/README.md deleted file mode 100644 index d7ed312..0000000 --- a/contentflow-worker/README.md +++ /dev/null @@ -1,411 +0,0 @@ -# ContentFlow Worker - -Multi-processing based worker engine for processing content through ContentFlow pipelines. - -## Table of Contents - -- [Overview](#overview) -- [Architecture](#architecture) -- [Components](#components) -- [Configuration](#configuration) -- [Installation](#installation) -- [Usage](#usage) -- [Task Types](#task-types) -- [Workflow](#workflow) -- [Sending Tasks to the Queue](#sending-tasks-to-the-queue) -- [Monitoring](#monitoring) -- [Features](#features) -- [Error Handling](#error-handling) -- [Performance Tuning](#performance-tuning) -- [Requirements](#requirements) - -## Overview - -The ContentFlow Worker is a distributed, multi-processing engine designed to: -- Process content items through ContentFlow pipelines -- Discover content from input sources (Azure Blob Storage, etc.) -- Scale processing across multiple worker processes -- Handle task queuing, retries, and fault tolerance - -## Architecture - -The worker engine consists of two types of worker processes: - -### 1. Content Processing Workers -- Listen to the Azure Storage Queue for processing tasks -- Execute ContentFlow pipelines on content items (excluding already-executed input executors) -- Report execution status to Cosmos DB -- Handle task retries and error recovery - -### 2. Input Source Workers -- Poll Cosmos DB for enabled pipelines associated with vaults -- Parse pipeline configuration to identify input executors -- Execute input executors to discover content from sources -- Create processing tasks for discovered content items -- Queue tasks for processing workers - -## Components - -``` -contentflow-worker/ -├── app/ -│ ├── __init__.py -│ ├── api.py # FastAPI health and status monitoring -│ ├── engine.py # Main worker engine (manages processes) -│ ├── models.py # Task models (Pydantic) -│ ├── queue_client.py # Azure Storage Queue wrapper -│ ├── settings.py # Configuration and settings -│ ├── startup.py # Application startup -│ ├── utils.py # Utility functions -│ └── worker/ -│ ├── __init__.py -│ ├── processing_worker.py # Content processing worker -│ └── source_worker.py # Input source loading worker -├── main.py # Entry point -├── Dockerfile # Docker container configuration -├── requirements.txt # Python dependencies -├── .env # Environment variables (local) -├── .env.example # Environment variables template -└── README.md # This file -``` - -## Configuration - -Configuration is loaded from: -1. Environment variables (`.env` file) -2. Azure App Configuration (if available) - -### Environment Variables - -```bash -# Azure App Configuration -AZURE_APP_CONFIG_ENDPOINT=https://your-app-config-resource.azconfig.io -APP_CONFIG_KEY_FILTERS=contentflow.worker.* - -# Worker Settings -WORKER_NAME=contentflow-worker -NUM_PROCESSING_WORKERS=2 -NUM_SOURCE_WORKERS=2 - -# Azure Storage Queue -STORAGE_ACCOUNT_WORKER_QUEUE_URL=https://your-storage-account.queue.core.windows.net -STORAGE_WORKER_QUEUE_NAME=contentflow-execution-requests - -# Queue Polling Settings -QUEUE_POLL_INTERVAL_SECONDS=5 -QUEUE_VISIBILITY_TIMEOUT_SECONDS=300 -QUEUE_MAX_MESSAGES=32 - -# Processing Settings -MAX_TASK_RETRIES=3 -TASK_TIMEOUT_SECONDS=600 - -# Cosmos DB -COSMOS_DB_ENDPOINT=https://your-cosmos.documents.azure.com:443/ -COSMOS_DB_NAME=contentflow -COSMOS_DB_CONTAINER_PIPELINES=pipelines -COSMOS_DB_CONTAINER_VAULT_EXECUTIONS=vault_executions -COSMOS_DB_CONTAINER_VAULTS=vaults -COSMOS_DB_CONTAINER_LOCKS=vault_exec_locks -COSMOS_DB_CONTAINER_CRAWL_CHECKPOINTS=vault_crawl_checkpoints - -# Azure Blob Storage -BLOB_STORAGE_ACCOUNT_NAME=your-storage-account -BLOB_STORAGE_CONTAINER_NAME=content - -# Source Worker Settings -DEFAULT_POLLING_INTERVAL_SECONDS=300 # Default 5 minutes if executor doesn't specify -SCHEDULER_SLEEP_INTERVAL_SECONDS=5 # How often scheduler checks for ready pipelines -LOCK_TTL_SECONDS=300 # Distributed lock TTL (5 minutes) - -# API Settings -API_ENABLED=true -API_HOST=0.0.0.0 -API_PORT=8099 - -# Logging -LOG_LEVEL=DEBUG -DEBUG=true -``` - -**Per-Executor Polling Intervals**: Configure polling intervals directly in executor settings: -```yaml -executors: - - id: blob_input - type: azure_blob_input - settings: - polling_interval_seconds: 300 # Check every 5 minutes - blob_storage_account: "mystorageaccount" -``` - -See `.env.example` for complete configuration options. - -## Installation - -1. Install dependencies: -```bash -pip install -r requirements.txt -``` - -2. Configure environment: -```bash -cp .env.example .env -# Edit .env with your configuration -``` - -3. Ensure contentflow-lib is available: -```bash -# Worker expects contentflow-lib in parent directory -# ../contentflow-lib -``` - -## Usage - -### Running the Worker - -```bash -python main.py -``` - -The worker will: -1. Start configured number of processing and source workers -2. Start the FastAPI health and status monitoring service -3. Connect to Azure Storage Queue -4. Begin processing tasks -5. Monitor worker health and restart failed workers -6. Handle graceful shutdown on SIGINT/SIGTERM - -### API Endpoints - -When `API_ENABLED=true`, the following health and status endpoints are available: - -- **Health Check**: `GET /health` - Returns worker health status -- **Worker Status**: `GET /status` - Returns detailed worker process status -- **Metrics**: `GET /metrics` - Returns worker performance metrics - -Example health check: -```bash -curl http://localhost:8099/health -``` - -Response: -```json -{ - "status": "healthy", - "timestamp": "2026-01-02T10:30:00Z", - "worker_name": "contentflow-worker" -} -``` - -### Graceful Shutdown - -Press `Ctrl+C` or send SIGTERM to gracefully stop all workers: -```bash -kill -TERM -``` - -## Task Types - -### ContentProcessingTask -Executes a pipeline on a content item. - -```python -{ - "task_id": "task_abc123", - "task_type": "content_processing", - "pipeline_id": "pipeline_xyz", - "pipeline_name": "Document Processing", - "execution_id": "exec_123", - "content_id": "content_456", - "content_data": {...}, - "executed_input_executor": "blob_input", # Already-executed input executor - "priority": "normal", - "max_retries": 3 -} -``` - -## Workflow - -### Source Worker Workflow (Continuous Scheduling) -1. **Continuous Scheduling Loop**: - - Runs continuously, checking pipeline schedule every 5 seconds (configurable via `SCHEDULER_SLEEP_INTERVAL_SECONDS`) - - Maintains next execution time for each pipeline based on per-executor polling intervals - -2. **Pipeline Discovery**: - - Queries Cosmos DB for enabled pipelines with associated vaults - - Extracts `polling_interval_seconds` from input executor settings (default: 300s) - - Schedules pipelines for execution at appropriate intervals - -3. **Distributed Locking**: - - Before executing a pipeline, attempts to acquire a distributed lock in Cosmos DB - - Prevents multiple workers from processing the same pipeline simultaneously - - Locks auto-expire after 5 minutes (configurable via `LOCK_TTL_SECONDS`) - -4. **Content Discovery**: - - Parse pipeline YAML to find input executor - - Execute input executor to discover content - - Create ContentProcessingTask for each discovered content item - - Mark the input executor as already executed (`executed_input_executor` field) - - Send tasks to queue - -5. **Schedule Update**: - - After successful execution, update next execution time - - Release distributed lock - - Pipeline will execute again after polling interval expires - -**Example: Pipeline with 1-hour polling** -```yaml -executors: - - id: blob_input - type: azure_blob_input - settings: - polling_interval_seconds: 3600 # Check every hour - blob_storage_account: "mystorageaccount" - blob_container_name: "documents" -``` - -### Processing Worker Workflow -1. Poll queue for ContentProcessingTask messages -2. For each task: - - Load pipeline configuration - - Exclude the already-executed input executor - - Execute remaining pipeline on content - - Update execution status - - Delete message on success - -## Sending Tasks to the Queue - -You can send tasks from the API or other services: - -```python -from contentflow_worker import TaskQueueClient, ContentProcessingTask - -# Initialize queue client -queue_client = TaskQueueClient( - queue_url="https://your-storage.queue.core.windows.net", - queue_name="contentflow-execution-requests" -) - -# Create task -task = ContentProcessingTask( - task_id="task_123", - pipeline_id="pipeline_xyz", - pipeline_name="My Pipeline", - execution_id="exec_456", - content_id="content_789", - content_data={"text": "Sample content"} -) - -# Send to queue -queue_client.send_content_processing_task(task) -``` - -## Monitoring - -### Worker Status via API - -When the API is enabled, you can monitor worker status via HTTP: - -```bash -# Check health -curl http://localhost:8099/health - -# Get detailed status -curl http://localhost:8099/status -``` - -### Worker Status Programmatically - -The engine provides status information: -```python -from contentflow_worker import WorkerEngine - -engine = WorkerEngine() -status = engine.get_status() -print(status) -``` - -Output: -```json -{ - "running": true, - "processing_workers": { - "configured": 2, - "active": 2, - "workers": [ - {"id": 0, "pid": 12345, "alive": true}, - {"id": 1, "pid": 12346, "alive": true} - ] - }, - "source_workers": { - "configured": 2, - "active": 2, - "workers": [ - {"id": 0, "pid": 12347, "alive": true}, - {"id": 1, "pid": 12348, "alive": true} - ] - } -} -``` - -### Logs - -Workers log to: -- Console (stdout) -- File (configured via LOG_LEVEL) - -## Features - -✅ Multi-processing parallelism -✅ Azure Storage Queue integration -✅ Automatic worker restart on failure -✅ Task retry handling -✅ Graceful shutdown -✅ Configurable worker counts -✅ Pipeline execution tracking -✅ Content source discovery -✅ Cosmos DB integration - -## Error Handling - -- **Task Failures**: Tasks are retried up to `max_retries` times -- **Worker Crashes**: Engine automatically restarts crashed workers -- **Queue Visibility**: Failed tasks become visible again after timeout -- **Execution Tracking**: All execution status updates are recorded in Cosmos DB - -## Performance Tuning - -Adjust these settings for your workload: - -```bash -# More workers = higher throughput -NUM_PROCESSING_WORKERS=4 -NUM_SOURCE_WORKERS=2 - -# Faster polling = lower latency -QUEUE_POLL_INTERVAL_SECONDS=2 - -# More messages per poll = higher throughput -QUEUE_MAX_MESSAGES=32 - -# Longer timeout = support longer-running pipelines -TASK_TIMEOUT_SECONDS=1200 - -# Queue visibility timeout for retries -QUEUE_VISIBILITY_TIMEOUT_SECONDS=300 -``` - -### Tuning Guidelines - -- **Throughput**: Increase `NUM_PROCESSING_WORKERS` and `QUEUE_MAX_MESSAGES` -- **Latency**: Decrease `QUEUE_POLL_INTERVAL_SECONDS` and `SCHEDULER_SLEEP_INTERVAL_SECONDS` -- **Memory**: Reduce worker counts to limit concurrent task processing -- **Reliability**: Increase `MAX_TASK_RETRIES` and `TASK_TIMEOUT_SECONDS` for complex pipelines - -## Requirements - -- Python 3.12+ -- Azure Storage Account (Queue) -- Azure Cosmos DB -- contentflow-lib package -- Azure credentials (DefaultAzureCredential) diff --git a/contentflow-worker/__init__.py b/contentflow-worker/__init__.py deleted file mode 100644 index 2a60444..0000000 --- a/contentflow-worker/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -ContentFlow Worker - Multi-processing content processing engine. -""" - -# from .engine import WorkerEngine -# from .models import ( -# ContentProcessingTask, -# InputSourceTask, -# TaskType, -# TaskPriority, -# TaskStatus, -# ) -# from .settings import WorkerSettings, get_settings -# from .queue_client import TaskQueueClient - -# __version__ = "0.1.0" - -# __all__ = [ -# "WorkerEngine", -# "ContentProcessingTask", -# "InputSourceTask", -# "TaskType", -# "TaskPriority", -# "TaskStatus", -# "WorkerSettings", -# "get_settings", -# "TaskQueueClient", -# ] diff --git a/contentflow-worker/app/__init__.py b/contentflow-worker/app/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/contentflow-worker/app/api.py b/contentflow-worker/app/api.py deleted file mode 100644 index 83ae650..0000000 --- a/contentflow-worker/app/api.py +++ /dev/null @@ -1,130 +0,0 @@ -""" -FastAPI application for worker health and status monitoring. - -This module provides HTTP endpoints for monitoring the health and status -of the ContentFlow worker engine. -""" -import logging -from datetime import datetime, timezone -from typing import Optional -from contextlib import asynccontextmanager - -from fastapi import FastAPI, status -from fastapi.responses import JSONResponse -from pydantic import BaseModel - -from app.settings import WorkerSettings -from app.engine import WorkerEngine - -logger = logging.getLogger("contentflow.worker.api") - -class HealthResponse(BaseModel): - """Health check response model""" - status: str - timestamp: str - worker_name: str - - -class WorkerStatusResponse(BaseModel): - """Worker status response model""" - worker_name: str - running: bool - timestamp: str - processing_workers: dict - source_workers: dict - -def create_app(settings: WorkerSettings, engine: Optional[WorkerEngine] = None) -> FastAPI: - """ - Create and configure the FastAPI application. - - Args: - settings: Worker configuration - engine: Reference to the WorkerEngine instance - - Returns: - Configured FastAPI application - """ - @asynccontextmanager - async def lifespan(app: FastAPI): - # Load the ML model into app.state during startup - app.state.engine = engine - app.state.settings = settings - app.state.start_time = datetime.now(timezone.utc) - yield - # Clean up (optional) during shutdown - app.state.engine = None - app.state.settings = None - app.state.start_time = None - - app = FastAPI( - title="ContentFlow Worker API", - description="Health and status monitoring API for ContentFlow worker service", - version="1.0.0", - lifespan=lifespan - ) - - @app.get("/", tags=["root"]) - async def root(): - """Root endpoint""" - return { - "service": "ContentFlow Worker API", - "version": "1.0.0", - "worker_name": app.state.settings.WORKER_NAME - } - - @app.get("/health", response_model=HealthResponse, tags=["monitoring"]) - async def health(): - """ - Health check endpoint. - - Returns basic health status indicating the API is responsive. - """ - return HealthResponse( - status="healthy", - timestamp=datetime.now(timezone.utc).isoformat(), - worker_name=app.state.settings.WORKER_NAME - ) - - @app.get("/status", response_model=WorkerStatusResponse, tags=["monitoring"]) - async def get_status(): - """ - Get detailed worker status. - - Returns comprehensive status information about the worker engine - and all worker processes. - """ - - engine = app.state.engine - engine_status = None - - if engine is None: - return JSONResponse( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - content={ - "error": "Worker engine not initialized", - "timestamp": datetime.now(timezone.utc).isoformat() - } - ) - - if isinstance(engine, WorkerEngine): - engine_status = engine.get_status() - - if not engine_status: - return JSONResponse( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - content={ - "error": "Unable to retrieve worker status", - "timestamp": datetime.now(timezone.utc).isoformat() - } - ) - - return WorkerStatusResponse( - worker_name=app.state.settings.WORKER_NAME, - running=engine_status["running"], - timestamp=datetime.now(timezone.utc).isoformat(), - processing_workers=engine_status["processing_workers"], - source_workers=engine_status["source_workers"] - ) - - return app - \ No newline at end of file diff --git a/contentflow-worker/app/engine.py b/contentflow-worker/app/engine.py deleted file mode 100644 index 5f4b13f..0000000 --- a/contentflow-worker/app/engine.py +++ /dev/null @@ -1,237 +0,0 @@ -""" -Worker engine for ContentFlow multi-processing content processing. - -This module implements the main WorkerEngine that manages: -- Content processing workers -- Input source loading workers -- Worker lifecycle and graceful shutdown -""" -import logging -import multiprocessing as mp -import signal -import sys -import time -from typing import List - -from app.worker.processing_worker import ContentProcessingWorker -from app.worker.source_worker import InputSourceWorker -from app.settings import WorkerSettings, get_settings - -logger = logging.getLogger("contentflow.worker.engine") - - -class WorkerEngine: - """ - Main engine for managing worker processes. - - The engine creates and manages two types of workers: - 1. Content Processing Workers: Execute pipelines on content - 2. Input Source Workers: Discover content and create processing tasks - - Features: - - Multi-processing based parallelism - - Graceful shutdown handling - - Worker health monitoring - - Automatic worker restart on failure - """ - - def __init__(self, settings: WorkerSettings = None): - """ - Initialize the worker engine. - - Args: - settings: Worker configuration (uses default if not provided) - """ - self.settings = settings or get_settings() - self.processing_workers: List[mp.Process] = [] - self.source_workers: List[mp.Process] = [] - self.stop_event = mp.Event() - self.running = False - - logger.info(f"Initialized WorkerEngine: {self.settings.WORKER_NAME}") - logger.info(f" Processing Workers: {self.settings.NUM_PROCESSING_WORKERS}") - logger.info(f" Source Workers: {self.settings.NUM_SOURCE_WORKERS}") - - def start(self): - """Start all worker processes""" - if self.running: - logger.warning("WorkerEngine is already running") - return - - logger.info("Starting WorkerEngine...") - - # Setup signal handlers - signal.signal(signal.SIGINT, self._signal_handler) - signal.signal(signal.SIGTERM, self._signal_handler) - - # Start processing workers - logger.info(f"Starting {self.settings.NUM_PROCESSING_WORKERS} processing workers...") - for i in range(self.settings.NUM_PROCESSING_WORKERS): - worker = self._create_processing_worker(i) - worker.start() - self.processing_workers.append(worker) - logger.info(f"Started processing worker {i} (PID: {worker.pid})") - - # Start source workers - logger.info(f"Starting {self.settings.NUM_SOURCE_WORKERS} source workers...") - for i in range(self.settings.NUM_SOURCE_WORKERS): - worker = self._create_source_worker(i) - worker.start() - self.source_workers.append(worker) - logger.info(f"Started source worker {i} (PID: {worker.pid})") - - self.running = True - logger.info("WorkerEngine started successfully") - - def _create_processing_worker(self, worker_id: int) -> mp.Process: - """Create a content processing worker process""" - worker = ContentProcessingWorker( - worker_id=worker_id, - settings=self.settings, - stop_event=self.stop_event - ) - - process = mp.Process( - target=worker.run, - name=f"ProcessingWorker-{worker_id}" - ) - process.daemon = False # Allow graceful shutdown - - return process - - def _create_source_worker(self, worker_id: int) -> mp.Process: - """Create an input source worker process""" - worker = InputSourceWorker( - worker_id=worker_id, - settings=self.settings, - stop_event=self.stop_event - ) - - process = mp.Process( - target=worker.run, - name=f"SourceWorker-{worker_id}" - ) - process.daemon = False # Allow graceful shutdown - - return process - - def stop(self): - """Stop all worker processes gracefully""" - if not self.running: - logger.warning("WorkerEngine is not running") - return - - logger.info("Stopping WorkerEngine...") - - # Signal all workers to stop - self.stop_event.set() - - # Wait for processing workers to finish - logger.info("Waiting for processing workers to stop...") - for worker in self.processing_workers: - worker.join(timeout=30) - if worker.is_alive(): - logger.warning(f"Worker {worker.name} did not stop gracefully, terminating...") - worker.terminate() - worker.join(timeout=5) - if worker.is_alive(): - logger.error(f"Worker {worker.name} could not be terminated") - - # Wait for source workers to finish - logger.info("Waiting for source workers to stop...") - for worker in self.source_workers: - worker.join(timeout=30) - if worker.is_alive(): - logger.warning(f"Worker {worker.name} did not stop gracefully, terminating...") - worker.terminate() - worker.join(timeout=5) - if worker.is_alive(): - logger.error(f"Worker {worker.name} could not be terminated") - - self.running = False - logger.info("WorkerEngine stopped") - - def run(self): - """ - Run the engine (blocking). - - This method starts all workers and monitors their health - until a shutdown signal is received. - """ - self.start() - - try: - # Main monitoring loop - logger.info("Monitoring workers... (Press Ctrl+C to stop)") - - while not self.stop_event.is_set(): - # Check worker health - self._check_worker_health() - - # Sleep between health checks - time.sleep(30) - - except KeyboardInterrupt: - logger.info("Received interrupt signal") - finally: - self.stop() - - def _check_worker_health(self): - """Check health of all workers and restart if needed""" - # Check processing workers - for i, worker in enumerate(self.processing_workers): - if not worker.is_alive() and not self.stop_event.is_set(): - logger.warning(f"Processing worker {i} died, restarting...") - new_worker = self._create_processing_worker(i) - new_worker.start() - self.processing_workers[i] = new_worker - logger.info(f"Restarted processing worker {i} (PID: {new_worker.pid})") - - # Check source workers - for i, worker in enumerate(self.source_workers): - if not worker.is_alive() and not self.stop_event.is_set(): - logger.warning(f"Source worker {i} died, restarting...") - new_worker = self._create_source_worker(i) - new_worker.start() - self.source_workers[i] = new_worker - logger.info(f"Restarted source worker {i} (PID: {new_worker.pid})") - - def _signal_handler(self, signum, frame): - """Handle shutdown signals""" - logger.info(f"WorkerEngine received signal {signum}") - self.stop_event.set() - - def get_status(self) -> dict: - """ - Get current engine status. - - Returns: - Dictionary with engine status information - """ - return { - "running": self.running, - "processing_workers": { - "configured": self.settings.NUM_PROCESSING_WORKERS, - "active": sum(1 for w in self.processing_workers if w.is_alive()), - "workers": [ - { - "id": i, - "pid": w.pid, - "alive": w.is_alive() - } - for i, w in enumerate(self.processing_workers) - ] - }, - "source_workers": { - "configured": self.settings.NUM_SOURCE_WORKERS, - "active": sum(1 for w in self.source_workers if w.is_alive()), - "workers": [ - { - "id": i, - "pid": w.pid, - "alive": w.is_alive() - } - for i, w in enumerate(self.source_workers) - ] - } - } diff --git a/contentflow-worker/app/models.py b/contentflow-worker/app/models.py deleted file mode 100644 index ff29896..0000000 --- a/contentflow-worker/app/models.py +++ /dev/null @@ -1,111 +0,0 @@ -""" -Task models for the ContentFlow worker engine. - -This module defines the data structures for processing tasks that are -queued and processed by worker processes. -""" -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional -from enum import Enum -from pydantic import BaseModel, Field - -from contentflow.models import Content - - -class TaskType(str, Enum): - """Type of task to be processed""" - CONTENT_PROCESSING = "content_processing" - INPUT_SOURCE_LOADING = "input_source_loading" - - -class TaskPriority(str, Enum): - """Priority levels for task processing""" - LOW = "low" - NORMAL = "normal" - HIGH = "high" - CRITICAL = "critical" - - -class TaskStatus(str, Enum): - """Status of task processing""" - QUEUED = "queued" - PROCESSING = "processing" - COMPLETED = "completed" - FAILED = "failed" - RETRYING = "retrying" - - -class ContentProcessingTask(BaseModel): - """ - Task for processing a content item through a pipeline. - - This task is created by input source workers and consumed by - content processing workers. - """ - task_id: str = Field(description="Unique identifier for this task") - task_type: TaskType = TaskType.CONTENT_PROCESSING - priority: TaskPriority = TaskPriority.NORMAL - - # Pipeline execution details - pipeline_id: str = Field(description="ID of the pipeline to execute") - pipeline_name: str = Field(description="Name of the pipeline") - execution_id: str = Field(description="ID of the pipeline execution") - vault_id: Optional[str] = Field(default=None, description="ID of the vault associated with this content") - - # Input content details - content: List[Content] = Field(description="List of content items to process") - - # Executor tracking - executed_input_executor: Optional[str] = Field(default=None, description="ID of input executor already executed") - - # Metadata - created_at: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) - retry_count: int = Field(default=0, description="Number of retry attempts") - max_retries: int = Field(default=3, description="Maximum number of retries") - - class Config: - use_enum_values = True - - -class InputSourceTask(BaseModel): - """ - Task for loading content from an input source. - - This task instructs an input source worker to scan a source - (e.g., Azure Blob Storage) and create content processing tasks. - """ - task_id: str = Field(description="Unique identifier for this task") - task_type: TaskType = TaskType.INPUT_SOURCE_LOADING - priority: TaskPriority = TaskPriority.NORMAL - - # Source configuration - source_type: str = Field(description="Type of input source (e.g., azure_blob, azure_files)") - source_name: str = Field(description="Name/identifier of the source") - source_config: Dict[str, Any] = Field(description="Configuration for the source") - - # Pipeline to execute on discovered content - pipeline_id: str = Field(description="ID of the pipeline to execute for each content item") - pipeline_name: str = Field(description="Name of the pipeline") - - # Filtering options - filters: Optional[Dict[str, Any]] = Field(default=None, description="Filters for content discovery") - - # Metadata - created_at: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) - - class Config: - use_enum_values = True - - -class TaskMessage(BaseModel): - """ - Generic task message envelope for queue messages. - - This wrapper allows us to handle different task types - with a single queue structure. - """ - task_type: TaskType - payload: Dict[str, Any] = Field(description="Task payload (serialized task)") - - class Config: - use_enum_values = True diff --git a/contentflow-worker/app/queue_client.py b/contentflow-worker/app/queue_client.py deleted file mode 100644 index 6b9b0f9..0000000 --- a/contentflow-worker/app/queue_client.py +++ /dev/null @@ -1,236 +0,0 @@ -""" -Queue client for managing task messages in Azure Storage Queue. - -This module provides a wrapper around Azure Storage Queue for sending -and receiving processing tasks. -""" -import json -import logging -from typing import Optional, List -from azure.storage.queue import QueueClient, QueueMessage -from azure.identity import ChainedTokenCredential -from azure.core.exceptions import ResourceNotFoundError - -from contentflow.utils import get_azure_credential, make_safe_json - -from app.models import TaskMessage, ContentProcessingTask, InputSourceTask, TaskType - -logger = logging.getLogger("contentflow.worker.queue_client") - - -class TaskQueueClient: - """ - Client for managing task messages in Azure Storage Queue. - - This client provides methods to: - - Send tasks to the queue - - Receive tasks from the queue - - Delete processed messages - - Update message visibility - """ - - def __init__( - self, - queue_url: str, - queue_name: str, - credential: Optional[ChainedTokenCredential] = None - ): - """ - Initialize the task queue client. - - Args: - queue_url: Azure Storage account queue URL - queue_name: Name of the queue - credential: Azure credential (uses DefaultAzureCredential if not provided) - """ - self.queue_url = queue_url - self.queue_name = queue_name - self.credential = credential or get_azure_credential() - - # Create queue client - full_queue_url = f"{queue_url}/{queue_name}" - self.client = QueueClient.from_queue_url( - queue_url=full_queue_url, - credential=self.credential - ) - - logger.info(f"Initialized TaskQueueClient for queue: {queue_name}") - - async def ensure_queue_exists(self): - """Ensure the queue exists, create if it doesn't""" - try: - # Check if queue exists - properties = self.client.get_queue_properties() - logger.info(f"Queue '{self.queue_name}' exists with {properties.approximate_message_count} messages") - except ResourceNotFoundError: - # Create queue if it doesn't exist - logger.info(f"Creating queue: {self.queue_name}") - self.client.create_queue() - - def send_content_processing_task( - self, - task: ContentProcessingTask, - visibility_timeout: Optional[int] = None - ) -> str: - """ - Send a content processing task to the queue. - - Args: - task: ContentProcessingTask to send - visibility_timeout: Optional visibility timeout in seconds - - Returns: - Message ID - """ - message = TaskMessage( - task_type=TaskType.CONTENT_PROCESSING, - payload=make_safe_json(task.model_dump()) - ) - return self._send_message(message, visibility_timeout) - - def send_input_source_task( - self, - task: InputSourceTask, - visibility_timeout: Optional[int] = None - ) -> str: - """ - Send an input source loading task to the queue. - - Args: - task: InputSourceTask to send - visibility_timeout: Optional visibility timeout in seconds - - Returns: - Message ID - """ - message = TaskMessage( - task_type=TaskType.INPUT_SOURCE_LOADING, - payload=task.model_dump() - ) - return self._send_message(message, visibility_timeout) - - def _send_message( - self, - message: TaskMessage, - visibility_timeout: Optional[int] = None - ) -> str: - """ - Send a message to the queue. - - Args: - message: TaskMessage to send - visibility_timeout: Optional visibility timeout in seconds - - Returns: - Message ID - """ - message_text = json.dumps(message.model_dump()) - - result = self.client.send_message( - content=message_text, - visibility_timeout=visibility_timeout - ) - - logger.debug(f"Sent message to queue: {result.id}") - return result.id - - def receive_messages( - self, - max_messages: int = 1, - visibility_timeout: int = 300 - ) -> List[QueueMessage]: - """ - Receive messages from the queue. - - Args: - max_messages: Maximum number of messages to receive - visibility_timeout: Visibility timeout in seconds - - Returns: - List of QueueMessage objects - """ - messages = self.client.receive_messages( - max_messages=max_messages, - visibility_timeout=visibility_timeout - ) - - return list(messages) - - def parse_message(self, message: QueueMessage) -> tuple[TaskType, dict]: - """ - Parse a queue message into a task. - - Args: - message: QueueMessage from the queue - - Returns: - Tuple of (TaskType, task_payload_dict) - """ - try: - # Parse message content - message_data = json.loads(message.content) - - # Extract task type and payload - task_type = TaskType(message_data["task_type"]) - payload = message_data["payload"] - - return task_type, payload - - except Exception as e: - logger.error(f"Failed to parse message {message.id}: {e}") - raise - - def delete_message(self, message: QueueMessage): - """ - Delete a message from the queue after successful processing. - - Args: - message: QueueMessage to delete - """ - try: - self.client.delete_message(message.id, message.pop_receipt) - logger.debug(f"Deleted message: {message.id}") - except Exception as e: - logger.error(f"Failed to delete message {message.id}: {e}") - raise - - def update_message_visibility( - self, - message: QueueMessage, - visibility_timeout: int - ) -> QueueMessage: - """ - Update message visibility timeout (e.g., to extend processing time). - - Args: - message: QueueMessage to update - visibility_timeout: New visibility timeout in seconds - - Returns: - Updated QueueMessage - """ - try: - updated = self.client.update_message( - message.id, - message.pop_receipt, - visibility_timeout=visibility_timeout - ) - logger.debug(f"Updated message visibility: {message.id}") - return updated - except Exception as e: - logger.error(f"Failed to update message visibility {message.id}: {e}") - raise - - def get_queue_length(self) -> int: - """ - Get approximate number of messages in the queue. - - Returns: - Approximate message count - """ - try: - properties = self.client.get_queue_properties() - return properties.approximate_message_count - except Exception as e: - logger.error(f"Failed to get queue properties: {e}") - return 0 diff --git a/contentflow-worker/app/settings.py b/contentflow-worker/app/settings.py deleted file mode 100644 index d9474b9..0000000 --- a/contentflow-worker/app/settings.py +++ /dev/null @@ -1,176 +0,0 @@ -""" -Worker settings and configuration for the ContentFlow worker engine. - -This module handles configuration loading from environment variables -and Azure App Configuration. -""" -import os -from typing import Optional -from pydantic import BaseModel, Field -from dotenv import load_dotenv, find_dotenv - -load_dotenv(find_dotenv('.env')) - - -class WorkerSettings(BaseModel): - """Configuration settings for the worker engine""" - - AZURE_APP_CONFIG_ENDPOINT: Optional[str] = Field(default=None, description="Azure App Configuration endpoint URL") - APP_CONFIG_KEY_FILTERS: list = Field(default=["contentflow.worker.*"], description="Key filters for App Configuration") - - # Worker settings - WORKER_NAME: str = Field(default="contentflow-worker", description="Worker instance name") - NUM_PROCESSING_WORKERS: int = Field(default=0, description="Number of content processing worker processes") - NUM_SOURCE_WORKERS: int = Field(default=1, description="Number of input source loading worker processes") - - # Queue settings - STORAGE_ACCOUNT_WORKER_QUEUE_URL: str = Field(description="Azure Storage Queue URL") - STORAGE_WORKER_QUEUE_NAME: str = Field(default="contentflow-execution-requests", description="Queue name for processing tasks") - - # Polling settings - QUEUE_POLL_INTERVAL_SECONDS: int = Field(default=5, description="Interval between queue polls") - QUEUE_VISIBILITY_TIMEOUT_SECONDS: int = Field(default=300, description="Message visibility timeout (5 minutes)") - QUEUE_MAX_MESSAGES: int = Field(default=32, description="Maximum messages to retrieve per poll per worker") - DEFAULT_POLLING_INTERVAL_SECONDS: int = Field(default=300, description="Default polling interval if not specified in input executor (5 minutes)") - SCHEDULER_SLEEP_INTERVAL_SECONDS: int = Field(default=5, description="How often scheduler checks for ready pipelines") - LOCK_TTL_SECONDS: int = Field(default=300, description="Lock time-to-live (5 minutes)") - - # Processing settings - MAX_TASK_RETRIES: int = Field(default=3, description="Maximum retries for failed tasks") - TASK_TIMEOUT_SECONDS: int = Field(default=600, description="Timeout for task processing (10 minutes)") - - # Cosmos DB settings (for pipeline and execution tracking) - COSMOS_DB_ENDPOINT: str = Field(description="Cosmos DB endpoint") - COSMOS_DB_NAME: str = Field(default="contentflow", description="Cosmos DB database name") - COSMOS_DB_CONTAINER_PIPELINES: str = Field(default="pipelines", description="Container for pipelines") - COSMOS_DB_CONTAINER_VAULT_EXECUTIONS: str = Field(default="vault_executions", description="Container for executions") - COSMOS_DB_CONTAINER_VAULTS: str = Field(default="vaults", description="Container for vaults") - COSMOS_DB_CONTAINER_VAULT_EXECUTION_LOCKS: str = Field(default="vault_exec_locks", description="Container for distributed locks") - COSMOS_DB_CONTAINER_CRAWL_CHECKPOINTS: str = Field(default="vault_crawl_checkpoints", description="Container for vault input executor crawling checkpoints") - - # Blob Storage settings (for content retrieval) - BLOB_STORAGE_ACCOUNT_NAME: str = Field(description="Azure Blob Storage account name") - BLOB_STORAGE_CONTAINER_NAME: str = Field(default="content", description="Container for content") - - # API settings - API_ENABLED: bool = Field(default=True, description="Enable health/status API") - API_HOST: str = Field(default="0.0.0.0", description="API host to bind to") - API_PORT: int = Field(default=8099, description="API port to bind to") - - # Logging - LOG_LEVEL: str = Field(default="DEBUG", description="Logging level") - DEBUG: bool = Field(default=False, description="Debug mode") - - def __init__(self, **kwargs): - """Initialize WorkerSettings and load from environment/App Config""" - # Load from environment variables first - env_config = self._load_from_env() - merged_config = {**env_config, **kwargs} - super().__init__(**merged_config) - - # Try to load from Azure App Configuration - self._load_from_app_config() - - def _load_from_env(self) -> dict: - """Load configuration from environment variables""" - return { - "AZURE_APP_CONFIG_ENDPOINT": os.getenv("AZURE_APP_CONFIG_ENDPOINT", ""), - "APP_CONFIG_KEY_FILTERS": os.getenv("APP_CONFIG_KEY_FILTERS", "contentflow.worker.*").split(","), - "WORKER_NAME": os.getenv("WORKER_NAME", "contentflow-worker"), - "NUM_PROCESSING_WORKERS": int(os.getenv("NUM_PROCESSING_WORKERS", "0")), - "NUM_SOURCE_WORKERS": int(os.getenv("NUM_SOURCE_WORKERS", "1")), - "STORAGE_ACCOUNT_WORKER_QUEUE_URL": os.getenv("STORAGE_ACCOUNT_WORKER_QUEUE_URL", ""), - "STORAGE_WORKER_QUEUE_NAME": os.getenv("STORAGE_WORKER_QUEUE_NAME", "contentflow-execution-requests"), - "QUEUE_POLL_INTERVAL_SECONDS": int(os.getenv("QUEUE_POLL_INTERVAL_SECONDS", "5")), - "QUEUE_VISIBILITY_TIMEOUT_SECONDS": int(os.getenv("QUEUE_VISIBILITY_TIMEOUT_SECONDS", "300")), - "QUEUE_MAX_MESSAGES": int(os.getenv("QUEUE_MAX_MESSAGES", "32")), - "MAX_TASK_RETRIES": int(os.getenv("MAX_TASK_RETRIES", "3")), - "TASK_TIMEOUT_SECONDS": int(os.getenv("TASK_TIMEOUT_SECONDS", "600")), - "COSMOS_DB_ENDPOINT": os.getenv("COSMOS_DB_ENDPOINT", ""), - "COSMOS_DB_NAME": os.getenv("COSMOS_DB_NAME", "contentflow"), - "COSMOS_DB_CONTAINER_PIPELINES": os.getenv("COSMOS_DB_CONTAINER_PIPELINES", "pipelines"), - "COSMOS_DB_CONTAINER_VAULTS": os.getenv("COSMOS_DB_CONTAINER_VAULTS", "vaults"), - "COSMOS_DB_CONTAINER_VAULT_EXECUTIONS": os.getenv("COSMOS_DB_CONTAINER_VAULT_EXECUTIONS", "vault_executions"), - "COSMOS_DB_CONTAINER_VAULT_EXECUTION_LOCKS": os.getenv("COSMOS_DB_CONTAINER_VAULT_EXECUTION_LOCKS", "vault_exec_locks"), - "COSMOS_DB_CONTAINER_CRAWL_CHECKPOINTS": os.getenv("COSMOS_DB_CONTAINER_CRAWL_CHECKPOINTS", "vault_crawl_checkpoints"), - "DEFAULT_POLLING_INTERVAL_SECONDS": int(os.getenv("DEFAULT_POLLING_INTERVAL_SECONDS", "300")), - "SCHEDULER_SLEEP_INTERVAL_SECONDS": int(os.getenv("SCHEDULER_SLEEP_INTERVAL_SECONDS", "5")), - "LOCK_TTL_SECONDS": int(os.getenv("LOCK_TTL_SECONDS", "300")), - "BLOB_STORAGE_ACCOUNT_NAME": os.getenv("BLOB_STORAGE_ACCOUNT_NAME", ""), - "BLOB_STORAGE_CONTAINER_NAME": os.getenv("BLOB_STORAGE_CONTAINER_NAME", "content"), - "API_ENABLED": os.getenv("API_ENABLED", "True").lower() in ("true", "1", "yes"), - "API_HOST": os.getenv("API_HOST", "0.0.0.0"), - "API_PORT": int(os.getenv("API_PORT", "8099")), - "LOG_LEVEL": os.getenv("LOG_LEVEL", "DBEUG"), - "DEBUG": os.getenv("DEBUG", "False").lower() in ("true", "1", "yes"), - } - - def _load_from_app_config(self): - """Load configuration from Azure App Configuration (if available)""" - try: - if self.AZURE_APP_CONFIG_ENDPOINT in [None, ""]: - print("Azure App Configuration endpoint not set in env variable AZURE_APP_CONFIG_ENDPOINT, skipping App Config load.") - return - - from contentflow.utils import ConfigurationProvider - config_provider = ConfigurationProvider(app_config_endpoint=self.AZURE_APP_CONFIG_ENDPOINT, config_key_filters=self.APP_CONFIG_KEY_FILTERS) - - # List of settings to load from App Config - config_keys = [ - "STORAGE_ACCOUNT_WORKER_QUEUE_URL", - "STORAGE_WORKER_QUEUE_NAME", - "COSMOS_DB_ENDPOINT", - "COSMOS_DB_NAME", - "BLOB_STORAGE_ACCOUNT_NAME", - "BLOB_STORAGE_CONTAINER_NAME", - "NUM_PROCESSING_WORKERS", - "NUM_SOURCE_WORKERS", - "LOG_LEVEL", - "DEBUG", - "API_ENABLED", - "API_HOST", - "API_PORT", - "QUEUE_POLL_INTERVAL_SECONDS", - "QUEUE_VISIBILITY_TIMEOUT_SECONDS", - "QUEUE_MAX_MESSAGES", - "MAX_TASK_RETRIES", - "TASK_TIMEOUT_SECONDS", - "DEFAULT_POLLING_INTERVAL_SECONDS", - "SCHEDULER_SLEEP_INTERVAL_SECONDS", - "LOCK_TTL_SECONDS" - ] - - for key in config_keys: - value = config_provider.get_config_value(key) - if value is not None: - # Convert to appropriate type - if key in ["NUM_PROCESSING_WORKERS", "NUM_SOURCE_WORKERS", "API_PORT", "QUEUE_POLL_INTERVAL_SECONDS", - "QUEUE_VISIBILITY_TIMEOUT_SECONDS", "QUEUE_MAX_MESSAGES", "MAX_TASK_RETRIES", "TASK_TIMEOUT_SECONDS", - "SOURCE_WORKER_POLL_INTERVAL_SECONDS", "DEFAULT_POLLING_INTERVAL_SECONDS", "SCHEDULER_SLEEP_INTERVAL_SECONDS", - "LOCK_TTL_SECONDS"]: - value = int(value) - elif key in ["API_ENABLED", "DEBUG"]: - value = value.lower() in ("true", "1", "yes") - - setattr(self, key, value) - - except ImportError: - # Config provider not available, use environment variables only - pass - except Exception as e: - print(f"Warning: Failed to load from App Configuration: {e}") - - class Config: - validate_assignment = True - - -def get_settings() -> WorkerSettings: - """Get worker settings (singleton pattern)""" - global _settings - if _settings is None: - _settings = WorkerSettings() - return _settings - - -# Global settings instance -_settings: Optional[WorkerSettings] = None diff --git a/contentflow-worker/app/startup.py b/contentflow-worker/app/startup.py deleted file mode 100644 index b0cd9e0..0000000 --- a/contentflow-worker/app/startup.py +++ /dev/null @@ -1,225 +0,0 @@ -""" -Startup validation and health checks for ContentFlow Worker. - -This module performs pre-flight checks before starting the worker engine: -- Validates required settings -- Verifies Cosmos DB connectivity and containers -- Verifies Azure Storage Queue connectivity -""" -import logging -from typing import List, Tuple - -from azure.cosmos import CosmosClient -from azure.core.exceptions import ResourceNotFoundError, AzureError -from azure.storage.queue import QueueClient - -from contentflow.utils import get_azure_credential -from app.settings import WorkerSettings - -logger = logging.getLogger("contentflow.worker.startup") - - -class StartupValidator: - """Validates system readiness before starting the worker engine""" - - def __init__(self, settings: WorkerSettings): - """ - Initialize the startup validator. - - Args: - settings: Worker configuration settings - """ - self.settings = settings - self.errors: List[str] = [] - self.warnings: List[str] = [] - - def validate_all(self) -> bool: - """ - Run all startup validations. - - Returns: - True if all checks pass, False if any critical errors occur - """ - logger.info("=" * 60) - logger.info("Running Startup Validation Checks") - logger.info("=" * 60) - - # Run all validation checks - checks = [ - ("Settings Validation", self._validate_settings), - ("Cosmos DB Connectivity", self._validate_cosmos_connectivity), - ("Cosmos DB Containers", self._validate_cosmos_containers), - ("Storage Queue Connectivity", self._validate_queue_connectivity), - ] - - for check_name, check_func in checks: - logger.info(f"Checking: {check_name}...") - try: - success, message = check_func() - if success: - logger.info(f"✅ {check_name}: {message}") - else: - logger.error(f"❌ {check_name}: {message}") - self.errors.append(f"{check_name}: {message}") - except Exception as e: - logger.error(f"❌ {check_name}: Unexpected error - {e}", exc_info=True) - self.errors.append(f"{check_name}: {str(e)}") - - # Display summary - logger.info("=" * 60) - if self.warnings: - logger.warning(f"⚠️ {len(self.warnings)} warning(s):") - for warning in self.warnings: - logger.warning(f" - {warning}") - - if self.errors: - logger.error(f"❌ {len(self.errors)} critical error(s) found:") - for error in self.errors: - logger.error(f" - {error}") - logger.error("=" * 60) - logger.error("Startup validation FAILED. Fix errors before starting.") - return False - else: - logger.info("✅ All startup validation checks passed!") - logger.info("=" * 60) - return True - - def _validate_settings(self) -> Tuple[bool, str]: - """Validate that all required settings are present""" - required_settings = { - "COSMOS_DB_ENDPOINT": self.settings.COSMOS_DB_ENDPOINT, - "COSMOS_DB_NAME": self.settings.COSMOS_DB_NAME, - "STORAGE_ACCOUNT_WORKER_QUEUE_URL": self.settings.STORAGE_ACCOUNT_WORKER_QUEUE_URL, - "STORAGE_WORKER_QUEUE_NAME": self.settings.STORAGE_WORKER_QUEUE_NAME, - "BLOB_STORAGE_ACCOUNT_NAME": self.settings.BLOB_STORAGE_ACCOUNT_NAME, - } - - missing = [] - for key, value in required_settings.items(): - if not value or value == "": - missing.append(key) - - if missing: - return False, f"Missing required settings: {', '.join(missing)}" - - # Validate worker counts - if self.settings.NUM_PROCESSING_WORKERS < 0: - return False, "NUM_PROCESSING_WORKERS must be >= 0" - - if self.settings.NUM_SOURCE_WORKERS < 0: - return False, "NUM_SOURCE_WORKERS must be >= 0" - - if self.settings.NUM_PROCESSING_WORKERS == 0 and self.settings.NUM_SOURCE_WORKERS == 0: - return False, "At least one of NUM_PROCESSING_WORKERS or NUM_SOURCE_WORKERS must be > 0" - - # Warnings for configuration - if self.settings.NUM_PROCESSING_WORKERS == 0: - self.warnings.append("NUM_PROCESSING_WORKERS is 0 - no content processing will occur") - - if self.settings.NUM_SOURCE_WORKERS == 0: - self.warnings.append("NUM_SOURCE_WORKERS is 0 - no source scanning will occur") - - return True, f"All required settings present (Processing Workers: {self.settings.NUM_PROCESSING_WORKERS}, Source Workers: {self.settings.NUM_SOURCE_WORKERS})" - - def _validate_cosmos_connectivity(self) -> Tuple[bool, str]: - """Validate connectivity to Cosmos DB""" - try: - credential = get_azure_credential() - cosmos_client = CosmosClient( - url=self.settings.COSMOS_DB_ENDPOINT, - credential=credential - ) - - # Try to list databases to verify connectivity - databases = list(cosmos_client.list_databases()) - - # Check if our database exists - db_exists = any(db["id"] == self.settings.COSMOS_DB_NAME for db in databases) - - if not db_exists: - return False, f"Database '{self.settings.COSMOS_DB_NAME}' not found in Cosmos DB" - - return True, f"Connected to Cosmos DB at {self.settings.COSMOS_DB_ENDPOINT}" - - except AzureError as e: - return False, f"Failed to connect to Cosmos DB: {str(e)}" - except Exception as e: - return False, f"Unexpected error connecting to Cosmos DB: {str(e)}" - - def _validate_cosmos_containers(self) -> Tuple[bool, str]: - """Validate that required Cosmos DB containers exist""" - required_containers = [ - self.settings.COSMOS_DB_CONTAINER_PIPELINES, - self.settings.COSMOS_DB_CONTAINER_VAULT_EXECUTIONS, - self.settings.COSMOS_DB_CONTAINER_VAULTS, - self.settings.COSMOS_DB_CONTAINER_VAULT_EXECUTION_LOCKS, - self.settings.COSMOS_DB_CONTAINER_CRAWL_CHECKPOINTS, - ] - - try: - credential = get_azure_credential() - cosmos_client = CosmosClient( - url=self.settings.COSMOS_DB_ENDPOINT, - credential=credential - ) - - database = cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - - # Check each container - existing_containers = list(database.list_containers()) - existing_names = [c["id"] for c in existing_containers] - - missing_containers = [] - for container_name in required_containers: - if container_name not in existing_names: - missing_containers.append(container_name) - - if missing_containers: - return False, f"Missing required containers: {', '.join(missing_containers)}" - - return True, f"All {len(required_containers)} required containers exist" - - except ResourceNotFoundError: - return False, f"Database '{self.settings.COSMOS_DB_NAME}' not found" - except AzureError as e: - return False, f"Failed to access Cosmos DB containers: {str(e)}" - except Exception as e: - return False, f"Unexpected error checking containers: {str(e)}" - - def _validate_queue_connectivity(self) -> Tuple[bool, str]: - """Validate connectivity to Azure Storage Queue""" - try: - credential = get_azure_credential() - full_queue_url = f"{self.settings.STORAGE_ACCOUNT_WORKER_QUEUE_URL}/{self.settings.STORAGE_WORKER_QUEUE_NAME}" - - queue_client = QueueClient.from_queue_url( - queue_url=full_queue_url, - credential=credential - ) - - # Try to get queue properties to verify connectivity and existence - properties = queue_client.get_queue_properties() - message_count = properties.approximate_message_count - - return True, f"Connected to queue '{self.settings.STORAGE_WORKER_QUEUE_NAME}' ({message_count} messages)" - - except ResourceNotFoundError: - return False, f"Queue '{self.settings.STORAGE_WORKER_QUEUE_NAME}' not found" - except AzureError as e: - return False, f"Failed to connect to Storage Queue: {str(e)}" - except Exception as e: - return False, f"Unexpected error connecting to queue: {str(e)}" - - -def run_startup_checks(settings: WorkerSettings) -> bool: - """ - Run startup validation checks. - - Args: - settings: Worker configuration settings - - Returns: - True if all checks pass, False otherwise - """ - validator = StartupValidator(settings) - return validator.validate_all() diff --git a/contentflow-worker/app/utils.py b/contentflow-worker/app/utils.py deleted file mode 100644 index 3a85232..0000000 --- a/contentflow-worker/app/utils.py +++ /dev/null @@ -1,182 +0,0 @@ -""" -Utility functions for the ContentFlow worker. -""" -import logging -from typing import Dict, Any, Optional -from datetime import datetime, timezone - -logger = logging.getLogger(__name__) - - -def create_execution_id() -> str: - """Generate a unique execution ID""" - import uuid - return f"exec_{uuid.uuid4().hex[:12]}" - - -def create_task_id() -> str: - """Generate a unique task ID""" - import uuid - return f"task_{uuid.uuid4().hex[:12]}" - - -def get_timestamp() -> str: - """Get current timestamp in ISO format""" - return datetime.now(timezone.utc).isoformat() - - -def format_duration(seconds: float) -> str: - """ - Format duration in seconds to human-readable string. - - Args: - seconds: Duration in seconds - - Returns: - Formatted string (e.g., "2m 30s", "1h 15m 30s") - """ - if seconds < 60: - return f"{seconds:.1f}s" - elif seconds < 3600: - minutes = int(seconds // 60) - secs = int(seconds % 60) - return f"{minutes}m {secs}s" - else: - hours = int(seconds // 3600) - minutes = int((seconds % 3600) // 60) - secs = int(seconds % 60) - return f"{hours}h {minutes}m {secs}s" - - -def sanitize_config(config: Dict[str, Any]) -> Dict[str, Any]: - """ - Sanitize configuration by removing sensitive values. - - Args: - config: Configuration dictionary - - Returns: - Sanitized configuration with sensitive values masked - """ - sensitive_keys = [ - 'password', 'secret', 'key', 'token', 'credential', - 'account_key', 'connection_string', 'api_key' - ] - - sanitized = {} - for k, v in config.items(): - key_lower = k.lower() - if any(sensitive in key_lower for sensitive in sensitive_keys): - sanitized[k] = "***REDACTED***" - elif isinstance(v, dict): - sanitized[k] = sanitize_config(v) - else: - sanitized[k] = v - - return sanitized - - -def validate_queue_config( - queue_url: str, - queue_name: str -) -> bool: - """ - Validate queue configuration. - - Args: - queue_url: Azure Storage Queue URL - queue_name: Queue name - - Returns: - True if valid, False otherwise - """ - if not queue_url: - logger.error("Queue URL is required") - return False - - if not queue_name: - logger.error("Queue name is required") - return False - - if not queue_url.startswith("https://"): - logger.error("Queue URL must start with https://") - return False - - if ".queue.core.windows.net" not in queue_url: - logger.warning("Queue URL does not appear to be an Azure Storage Queue URL") - - return True - - -def validate_cosmos_config( - endpoint: str, - database_name: str -) -> bool: - """ - Validate Cosmos DB configuration. - - Args: - endpoint: Cosmos DB endpoint URL - database_name: Database name - - Returns: - True if valid, False otherwise - """ - if not endpoint: - logger.error("Cosmos DB endpoint is required") - return False - - if not database_name: - logger.error("Database name is required") - return False - - if not endpoint.startswith("https://"): - logger.error("Cosmos DB endpoint must start with https://") - return False - - if ".documents.azure.com" not in endpoint: - logger.warning("Endpoint does not appear to be a Cosmos DB URL") - - return True - - -def retry_with_backoff( - func, - max_retries: int = 3, - base_delay: float = 1.0, - max_delay: float = 60.0 -): - """ - Retry a function with exponential backoff. - - Args: - func: Function to retry - max_retries: Maximum number of retries - base_delay: Base delay in seconds - max_delay: Maximum delay in seconds - - Returns: - Function result if successful - - Raises: - Exception: Last exception if all retries fail - """ - import time - - last_exception = None - delay = base_delay - - for attempt in range(max_retries + 1): - try: - return func() - except Exception as e: - last_exception = e - - if attempt < max_retries: - logger.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {delay}s...") - time.sleep(delay) - delay = min(delay * 2, max_delay) - else: - logger.error(f"All {max_retries} retries failed") - - raise last_exception diff --git a/contentflow-worker/app/worker/__init__.py b/contentflow-worker/app/worker/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/contentflow-worker/app/worker/processing_worker.py b/contentflow-worker/app/worker/processing_worker.py deleted file mode 100644 index c5a7b75..0000000 --- a/contentflow-worker/app/worker/processing_worker.py +++ /dev/null @@ -1,478 +0,0 @@ -""" -Content processing worker for executing pipeline tasks. - -This module implements the ContentProcessingWorker that: -- Listens to the task queue for content processing tasks -- Executes pipelines on content -- Reports execution status back to Cosmos DB -""" - -import asyncio -from datetime import datetime, timezone -import logging -import multiprocessing as mp -import signal -import sys -import time -import traceback -from pathlib import Path -from typing import List, Optional - -from azure.identity import ChainedTokenCredential -from azure.cosmos import CosmosClient -from azure.cosmos import exceptions as cosmos_exceptions - -from contentflow.pipeline import PipelineExecutor -from contentflow.models import Content, ContentIdentifier -from contentflow.pipeline import PipelineResult -from contentflow.utils import get_azure_credential, make_safe_json - -from app.models import ContentProcessingTask -from app.queue_client import TaskQueueClient -from app.settings import WorkerSettings - -logger = logging.getLogger("contentflow.worker.processing_worker") - - -class ContentProcessingWorker: - """ - Worker process for executing content processing tasks. - - This worker: - 1. Polls the task queue for ContentProcessingTask messages - 2. Loads pipeline configuration from Cosmos DB - 3. Executes the pipeline on the content - 4. Updates execution status in Cosmos DB - 5. Deletes the message from queue on success - """ - - def __init__( - self, - worker_id: int, - settings: WorkerSettings, - stop_event - ): - """ - Initialize the content processing worker. - - Args: - worker_id: Unique worker identifier - settings: Worker configuration settings - stop_event: Multiprocessing event for graceful shutdown - """ - self.worker_id = worker_id - self.settings = settings - self.stop_event = stop_event - self.name = f"ContentProcessingWorker-{worker_id}" - - # Azure clients (initialized in run) - self.credential: Optional[ChainedTokenCredential] = None - self.queue_client: Optional[TaskQueueClient] = None - self.cosmos_client: Optional[CosmosClient] = None - - logger.info(f"Initialized {self.name}") - - def run(self): - """Main worker loop""" - logger.info(f"{self.name} starting...") - - # Setup signal handlers for graceful shutdown - signal.signal(signal.SIGINT, self._signal_handler) - signal.signal(signal.SIGTERM, self._signal_handler) - - try: - # Initialize Azure clients - self._initialize_clients() - - # Main processing loop - logger.info(f"{self.name} entering main processing loop") - - while not self.stop_event.is_set(): - try: - self._process_batch() - - # Sleep between polls - if not self.stop_event.is_set(): - time.sleep(self.settings.QUEUE_POLL_INTERVAL_SECONDS) - - except Exception as e: - logger.error(f"{self.name} error in processing loop: {e}") - logger.error(traceback.format_exc()) - time.sleep(5) # Back off on error - - except KeyboardInterrupt: - logger.info(f"{self.name} received interrupt signal") - except Exception as e: - logger.error(f"{self.name} fatal error: {e}") - logger.error(traceback.format_exc()) - finally: - logger.info(f"{self.name} shutting down...") - - def _initialize_clients(self): - """Initialize Azure clients""" - logger.info(f"{self.name} initializing Azure clients...") - - # Initialize credential - self.credential = get_azure_credential() - - # Initialize queue client - self.queue_client = TaskQueueClient( - queue_url=self.settings.STORAGE_ACCOUNT_WORKER_QUEUE_URL, - queue_name=self.settings.STORAGE_WORKER_QUEUE_NAME, - credential=self.credential - ) - - # Initialize Cosmos DB client - self.cosmos_client = CosmosClient( - url=self.settings.COSMOS_DB_ENDPOINT, - credential=self.credential - ) - - logger.info(f"{self.name} clients initialized") - - def _process_batch(self): - """Process a batch of messages from the queue""" - try: - # Receive messages - logger.debug(f"{self.name} polling for messages... will retrieve up to {self.settings.QUEUE_MAX_MESSAGES} messages.") - - messages = self.queue_client.receive_messages( - max_messages=self.settings.QUEUE_MAX_MESSAGES, - visibility_timeout=self.settings.QUEUE_VISIBILITY_TIMEOUT_SECONDS - ) - - if not messages: - logger.debug(f"{self.name} no messages in queue") - return - - logger.debug(f"{self.name} received {len(messages)} messages") - - # Process each message - for message in messages: - if self.stop_event.is_set(): - break - - try: - self._process_message(message) - except Exception as e: - logger.error(f"{self.name} failed to process message {message.id}: {e}") - logger.error(traceback.format_exc()) - # Message will become visible again after timeout - - except Exception as e: - logger.error(f"{self.name} error receiving messages: {e}") - logger.error(traceback.format_exc()) - - def _process_message(self, message): - """Process a single message""" - logger.debug(f"{self.name} processing message {message.id}") - - try: - # Parse message - task_type, payload = self.queue_client.parse_message(message) - logger.debug(f"{self.name} parsed message {message.id} with task type {task_type}") - - # Only process content processing tasks - if task_type.value != "content_processing": - logger.warning(f"{self.name} received non-processing task: {task_type}") - # self.queue_client.delete_message(message) - return - - # Create task object - task = ContentProcessingTask(**payload) - - # Check retry count - if task.retry_count >= task.max_retries: - logger.error(f"{self.name} task {task.task_id} exceeded max retries") - self._update_execution_status(execution_id=task.execution_id, - status="failed", - error="Max retries exceeded") - self.queue_client.delete_message(message) - return - - # Execute task - success = self._execute_task(task) - - # Delete message on success - if success: - self.queue_client.delete_message(message) - logger.debug(f"{self.name} completed message {message.id}") - else: - logger.warning(f"{self.name} task failed, will retry") - # Message will become visible again after timeout - - except Exception as e: - logger.error(f"{self.name} error processing message {message.id}: {e}") - logger.exception(e) - raise - - def _execute_task(self, task: ContentProcessingTask) -> bool: - """ - Execute a content processing task. - - Args: - task: ContentProcessingTask to execute - - Returns: - True if successful (completed or failed), False if an exception occurred that was not handled - """ - logger.debug(f"{self.name} executing task '{task.task_id}', with execution id '{task.execution_id}' for pipeline '{task.pipeline_name}'") - - try: - # Update execution status to running - self._update_execution_status(execution_id=task.execution_id, status="running") - - # Get pipeline from Cosmos DB - pipeline_config = self._get_pipeline(task.pipeline_id) - if not pipeline_config: - logger.error(f"{self.name} pipeline not found: {task.pipeline_id}") - self._update_execution_status(execution_id=task.execution_id, status="failed", error="Pipeline not found") - return True - - # verify input content - content = task.content - if not content or len(content) == 0: - logger.error(f"{self.name} no input content for task {task.task_id}") - self._update_execution_status(execution_id=task.execution_id, status="failed", error="No input content") - return True - - # Execute pipeline synchronously (using asyncio.run) - result = self._run_pipeline(pipeline_config, content, task) - - if result and result.status == "completed": - self._update_execution_status(execution_id=task.execution_id, status="completed") - should_save_output = self._get_vault_should_save_output(task.vault_id) - try: - self._save_execution_output(task.execution_id, result, should_save_output) - except cosmos_exceptions.CosmosHttpResponseError as ce: - # check if it's due to request entity too large - if ce.status_code == 413: - - logger.warning(f"Result too large to store for execution {task.execution_id}, skipping event data storage.") - - # update the data field in each content item to indicate data too large - if isinstance(result.content, list): - for c in result.content: - c.data = {"data": "Output data too large to store in Cosmos DB. View output saved by output executor(s)."} - self._save_execution_output(task.execution_id, result, should_save_output) - else: - result.content.data = {"data": "Output data too large to store in Cosmos DB. View output saved by output executor(s)."} - self._save_execution_output(task.execution_id, result, should_save_output) - - else: - logger.error(f"Failed to add result to execution {task.execution_id}: {ce}", exc_info=True) - raise ce - - logger.debug(f"{self.name} task {task.task_id} completed successfully") - return True - elif result and result.status == "failed": - logger.error(f"{self.name} task {task.task_id} failed during execution: {result.error}") - self._update_execution_status(execution_id=task.execution_id, status="failed", error=result.error) - self._save_execution_output(task.execution_id, result, should_save_output=False) - return True - else: - self._update_execution_status(execution_id=task.execution_id, status=result.status, error="Pipeline execution has unknown status") - return True - - except Exception as e: - logger.error(f"{self.name} error executing task {task.task_id}: {e}") - logger.exception(e) - self._update_execution_status(task.execution_id, "failed", error=str(e)) - return False - - def _run_pipeline(self, pipeline_config: dict, content: List[Content], task: ContentProcessingTask) -> PipelineResult: - """Run pipeline execution""" - try: - # Load pipeline executor - import yaml - - # Parse the YAML - _parsed_pipeline_config = yaml.safe_load(pipeline_config["yaml"]) - - # If an input executor was already executed, remove it from the pipeline - if task.executed_input_executor: - logger.debug(f"{self.name} excluding already-executed input executor: {task.executed_input_executor}") - _parsed_pipeline_config = self._exclude_executor(_parsed_pipeline_config, task.executed_input_executor) - - - pipeline_definition = _parsed_pipeline_config.get('pipeline', _parsed_pipeline_config) - - # Create pipeline executor - async def execute(): - async with PipelineExecutor.from_pipeline_definition_dict( - pipeline_definition=pipeline_definition - ) as pipeline_executor: - - # Execute - return await pipeline_executor.execute(content) - - # Run async pipeline - result = asyncio.run(execute()) - - logger.debug(f"{self.name} pipeline execution result: {result.status}") - return result - - except Exception as e: - logger.error(f"{self.name} pipeline execution error: {e}") - logger.exception(e) - raise e - - def _exclude_executor(self, config: dict, executor_id: str) -> dict: - """ - Exclude a specific executor from the pipeline configuration. - - Args: - config: Pipeline configuration - executor_id: ID of executor to exclude - - Returns: - Modified configuration with executor removed - """ - try: - # Get pipeline definition - pipeline_def = config.get('pipeline', config) - - # Remove executor from executors list - if 'executors' in pipeline_def: - executors = pipeline_def['executors'] - pipeline_def['executors'] = [ - ex for ex in executors - if ex.get('id') != executor_id - ] - logger.info(f"{self.name} removed executor {executor_id} from pipeline") - - # Remove executor from execution_sequence if present - if 'execution_sequence' in pipeline_def: - sequence = pipeline_def['execution_sequence'] - if executor_id in sequence: - pipeline_def['execution_sequence'] = [ - ex_id for ex_id in sequence - if ex_id != executor_id - ] - logger.info(f"{self.name} removed {executor_id} from execution_sequence") - - # Remove executor from edges if present - if 'edges' in pipeline_def: - edges = pipeline_def['edges'] - pipeline_def['edges'] = [ - edge for edge in edges - if edge.get('from') != executor_id and edge.get('to') != executor_id - ] - logger.info(f"{self.name} removed edges involving {executor_id}") - - return config - - except Exception as e: - logger.error(f"{self.name} error excluding executor: {e}") - return config - - def _get_pipeline(self, pipeline_id: str) -> Optional[dict]: - """Get pipeline configuration from Cosmos DB""" - try: - database = self.cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - container = database.get_container_client(self.settings.COSMOS_DB_CONTAINER_PIPELINES) - - pipeline = container.read_item(item=pipeline_id, partition_key=pipeline_id) - return pipeline - - except Exception as e: - logger.error(f"{self.name} error reading pipeline {pipeline_id}: {e}") - return None - - def _update_execution_status(self, execution_id: str, status: str, error: Optional[str] = None): - """Update execution status in Cosmos DB""" - try: - database = self.cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - container = database.get_container_client(self.settings.COSMOS_DB_CONTAINER_VAULT_EXECUTIONS) - - # Read current execution - execution = container.read_item(item=execution_id, partition_key=execution_id) - - logger.debug(f"{self.name}: Retrieved execution {execution_id}. Current execution status: {execution.get('status')}") - - # Update status - execution["status"] = status - if status == "running": - execution["started_at"] = datetime.now(timezone.utc).isoformat() - execution["status_message"] = "Execution is running" - elif status in ["completed", "failed"]: - execution["completed_at"] = datetime.now(timezone.utc).isoformat() - execution["status_message"] = f"Execution {status}" - - if error: - execution["error"] = error - execution["status_message"] = f"Execution failed. View error details." - - execution["processing_worker_id"] = self.name - execution['updated_at'] = datetime.now(timezone.utc).isoformat() - - # Update in Cosmos DB - container.upsert_item(execution) - - logger.debug(f"{self.name} updated execution {execution_id} status to {status}") - - except Exception as e: - logger.error(f"{self.name} error updating execution {execution_id}. Error: {e}") - logger.exception(e) - - def _save_execution_output(self, execution_id: str, result: PipelineResult, should_save_output: bool = False): - """Save execution output to Cosmos DB""" - try: - database = self.cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - container = database.get_container_client(self.settings.COSMOS_DB_CONTAINER_VAULT_EXECUTIONS) - - # Read current execution - execution = container.read_item(item=execution_id, partition_key=execution_id) - - # Save output - if result.content is not None: - if should_save_output: - if isinstance(result.content, list): - execution["content"] = [make_safe_json(c) for c in result.content] - else: - execution["content"] = make_safe_json(result.content) - - else: - # Only save content identifiers and execution metadata - if isinstance(result.content, list): - execution["content"] = [c.id.model_dump() for c in result.content] - else: - execution["content"] = result.content.id.model_dump() - - execution["number_of_items"] = len(result.content) if isinstance(result.content, list) else 1 - - execution['updated_at'] = datetime.now(timezone.utc).isoformat() - - # Update in Cosmos DB - container.upsert_item(execution) - - logger.debug(f"{self.name} saved output for execution {execution_id}") - - except Exception as e: - logger.error(f"{self.name} error saving execution output: {e}") - raise e - - def _get_vault_should_save_output(self, vault_id: str) -> bool: - """Get whether the vault is configured to save execution output""" - - if not vault_id: - return False - - try: - database = self.cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - container = database.get_container_client(self.settings.COSMOS_DB_CONTAINER_VAULTS) - - # Read current execution - vault = container.read_item(item=vault_id, partition_key=vault_id) - - return vault.get("save_execution_output", False) - - except Exception as e: - logger.error(f"{self.name} error getting vault save output flag: {e}") - - return False - - def _signal_handler(self, signum, frame): - """Handle shutdown signals""" - logger.info(f"{self.name} received signal {signum}") - self.stop_event.set() diff --git a/contentflow-worker/app/worker/source_worker.py b/contentflow-worker/app/worker/source_worker.py deleted file mode 100644 index 3614c25..0000000 --- a/contentflow-worker/app/worker/source_worker.py +++ /dev/null @@ -1,853 +0,0 @@ -""" -Input source worker for loading and discovering content. - -This module implements the InputSourceWorker that: -- Continuously runs with per-executor polling intervals -- Polls Cosmos DB for pipelines associated with vaults -- Uses distributed locking to prevent concurrent execution by multiple workers -- Identifies and executes input executors from enabled pipelines -- Creates content processing tasks for discovered content -- Sends tasks to the processing queue -""" -import asyncio -import logging -import multiprocessing as mp -import signal -import sys -import time -import traceback -import uuid -import yaml -from datetime import datetime, timezone, timedelta -from pathlib import Path -from typing import AsyncGenerator, Optional, List, Dict, Any - -# # Add contentflow-lib to path -# lib_path = Path(__file__).parent.parent / "contentflow-lib" -# if lib_path.exists(): -# sys.path.insert(0, str(lib_path)) - -from azure.identity import ChainedTokenCredential -from azure.cosmos import CosmosClient -from azure.cosmos.exceptions import CosmosResourceNotFoundError, CosmosResourceExistsError - -from contentflow.executors import BaseExecutor, InputExecutor -from contentflow.executors.executor_config import ExecutorInstanceConfig -from contentflow.models import Content -from contentflow.executors import ExecutorRegistry -from contentflow.utils import get_azure_credential, make_safe_json - -from app.models import ContentProcessingTask, TaskPriority -from app.queue_client import TaskQueueClient -from app.settings import WorkerSettings - -logger = logging.getLogger("contentflow.worker.source_worker") - -class InputSourceWorker: - """ - Worker process for loading content from input sources. - - This worker: - 1. Continuously runs with a scheduler that tracks next execution time per pipeline - 2. Uses per-executor polling intervals from executor settings - 3. Implements distributed locking via Cosmos DB to prevent concurrent execution - 4. Polls Cosmos DB for pipelines associated with vaults (enabled pipelines) - 5. Parses pipeline configuration to find input executors - 6. Executes input executors to discover content - 7. Creates ContentProcessingTask for each discovered content item - 8. Sends processing tasks to the queue - """ - - def __init__( - self, - worker_id: int, - settings: WorkerSettings, - stop_event - ): - """ - Initialize the input source worker. - - Args: - worker_id: Unique worker identifier - settings: Worker configuration settings - stop_event: Multiprocessing event for graceful shutdown - """ - self.worker_id = worker_id - self.settings = settings - self.stop_event = stop_event - self.name = f"InputSourceWorker-{worker_id}" - - # Azure clients (initialized in run) - self.credential: Optional[ChainedTokenCredential] = None - self.queue_client: Optional[TaskQueueClient] = None - self.cosmos_client: Optional[CosmosClient] = None - self.executor_registry: Optional[ExecutorRegistry] = None - - # Scheduling: Track next execution time for each pipeline - # Key: pipeline_id, Value: next_execution_time (datetime) - self.pipeline_schedule: Dict[str, datetime] = {} - - # Map of pipeline to vault - # Key: pipeline_id, Value: vault_id - self.pipeline_vaults: Dict[str, str] = {} - - # Track polling intervals per pipeline - # Key: pipeline_id, Value: polling_interval_seconds - self.pipeline_intervals: Dict[str, int] = {} - - logger.info(f"Initialized {self.name}") - - def run(self): - """Main worker loop with continuous scheduling""" - logger.info(f"{self.name} starting...") - - # Setup signal handlers for graceful shutdown - signal.signal(signal.SIGINT, self._signal_handler) - signal.signal(signal.SIGTERM, self._signal_handler) - - try: - # Initialize Azure clients - self._initialize_clients() - - # Main continuous scheduling loop - while not self.stop_event.is_set(): - try: - # Refresh pipeline list and schedule - self._refresh_pipeline_schedule() - - # Process pipelines that are ready for execution - asyncio.run(self._process_ready_pipelines()) - - # Sleep briefly between schedule checks - if not self.stop_event.is_set(): - time.sleep(self.settings.SCHEDULER_SLEEP_INTERVAL_SECONDS) - - except Exception as e: - logger.error(f"{self.name} error in scheduling loop: {e}") - logger.error(traceback.format_exc()) - time.sleep(5) # Back off on error - - except KeyboardInterrupt: - logger.info(f"{self.name} received interrupt signal") - except Exception as e: - logger.error(f"{self.name} fatal error: {e}") - logger.error(traceback.format_exc()) - finally: - logger.info(f"{self.name} shutting down...") - - def _initialize_clients(self): - """Initialize Azure clients""" - logger.info(f"{self.name} initializing Azure clients...") - - # Initialize credential - self.credential = get_azure_credential() - - # Initialize queue client - self.queue_client = TaskQueueClient( - queue_url=self.settings.STORAGE_ACCOUNT_WORKER_QUEUE_URL, - queue_name=self.settings.STORAGE_WORKER_QUEUE_NAME, - credential=self.credential - ) - - # Initialize Cosmos DB client - self.cosmos_client = CosmosClient( - url=self.settings.COSMOS_DB_ENDPOINT, - credential=self.credential - ) - - self.executor_registry = ExecutorRegistry.load_default_catalog() # Load default executors - - logger.info(f"{self.name} clients initialized") - - def _refresh_pipeline_schedule(self): - """Refresh the list of pipelines and their schedules""" - try: - # Get pipelines with vaults - vault_pipelines = self._get_pipelines_with_vaults() - - current_time = datetime.now(timezone.utc) - - # Update schedule for new or modified pipelines - for vault_id, pipeline_id in vault_pipelines: - - # If pipeline is new to schedule, add it - if pipeline_id not in self.pipeline_schedule: - # Get polling interval from executor settings - interval = self._get_pipeline_polling_interval(pipeline_id) - self.pipeline_intervals[pipeline_id] = interval - - # Schedule for immediate execution (first run) - self.pipeline_schedule[pipeline_id] = current_time - self.pipeline_vaults[pipeline_id] = vault_id - - logger.info( - f"{self.name} added pipeline {pipeline_id} " - f"with {interval}s polling interval" - ) - - # Remove pipelines that are no longer active - active_pipeline_ids = set([pipeline_id for _, pipeline_id in vault_pipelines]) - inactive_pipeline_ids = set(self.pipeline_schedule.keys()) - active_pipeline_ids - - for inactive_id in inactive_pipeline_ids: - del self.pipeline_schedule[inactive_id] - del self.pipeline_intervals[inactive_id] - del self.pipeline_vaults[inactive_id] - logger.info(f"{self.name} removed inactive pipeline {inactive_id}") - - except Exception as e: - logger.error(f"{self.name} error refreshing pipeline schedule: {e}") - logger.error(traceback.format_exc()) - - def _get_pipeline_polling_interval(self, pipeline_id: str) -> int: - """Extract polling interval from pipeline's input executor settings""" - try: - pipeline = self._get_pipeline_by_id(pipeline_id) - - # Find input executor - input_executor_config = self._find_input_executor(pipeline) - - if not input_executor_config: - return self.settings.DEFAULT_POLLING_INTERVAL_SECONDS - - # Check for polling_interval_seconds in executor settings - settings = input_executor_config.get('settings', {}) - interval = settings.get('polling_interval_seconds') - - if interval and isinstance(interval, (int, float)) and interval > 0: - return int(interval) - - # Default fallback - return self.settings.DEFAULT_POLLING_INTERVAL_SECONDS - - except Exception as e: - logger.error(f"{self.name} error getting polling interval: {e}") - return self.settings.DEFAULT_POLLING_INTERVAL_SECONDS - - async def _process_ready_pipelines(self): - """Process pipelines that are ready for execution""" - current_time = datetime.now(timezone.utc) - - for pipeline_id, next_execution_time in list(self.pipeline_schedule.items()): - if self.stop_event.is_set(): - break - - # Check if pipeline is ready to execute - if current_time >= next_execution_time: - try: - # Try to acquire lock for this pipeline - if self._acquire_lock(pipeline_id): - try: - # Get pipeline details - pipeline = self._get_pipeline_by_id(pipeline_id) - - if pipeline: - # Process the pipeline - await self._process_pipeline(pipeline) - - # Update next execution time - interval = self.pipeline_intervals.get( - pipeline_id, - self.settings.DEFAULT_POLLING_INTERVAL_SECONDS - ) - next_time = current_time + timedelta(seconds=interval) - self.pipeline_schedule[pipeline_id] = next_time - - logger.debug( - f"{self.name} scheduled next execution for " - f"{pipeline.get('name')} at {next_time.isoformat()}" - ) - finally: - # Always release lock - self._release_lock(pipeline_id) - else: - logger.debug( - f"{self.name} could not acquire lock for pipeline {pipeline_id}, " - "another worker is processing it" - ) - - except Exception as e: - logger.error(f"{self.name} error processing pipeline {pipeline_id}: {e}") - logger.error(traceback.format_exc()) - - def _acquire_lock(self, pipeline_id: str) -> bool: - """ - Attempt to acquire a distributed lock for a pipeline. - - Uses Cosmos DB for distributed locking with TTL for auto-cleanup. - - Args: - pipeline_id: Pipeline identifier - - Returns: - True if lock acquired, False otherwise - """ - try: - database = self.cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - container = database.get_container_client(self.settings.COSMOS_DB_CONTAINER_VAULT_EXECUTION_LOCKS) - - lock_id = f"pipeline_{pipeline_id}_lock" - current_time = datetime.now(timezone.utc) - value_id = self.pipeline_vaults.get(pipeline_id, None) - - lock_doc = { - "id": lock_id, - "pipeline_id": pipeline_id, - "vault_id": value_id, - "worker_id": self.name, - "locked_at": current_time.isoformat(), - "ttl": self.settings.LOCK_TTL_SECONDS # Auto-expire after TTL - } - - try: - # Try to create lock document (will fail if already exists) - container.create_item(lock_doc) - logger.debug(f"{self.name} acquired lock for pipeline {pipeline_id}") - return True - - except CosmosResourceExistsError: - # Lock already exists, check if it's stale - try: - existing_lock = container.read_item( - item=lock_id, - partition_key=lock_id - ) - - locked_at = datetime.fromisoformat(existing_lock['locked_at']) - age_seconds = (current_time - locked_at).total_seconds() - - # If lock is older than TTL, it should have been cleaned up - # but in case TTL cleanup is delayed, we can force delete - if age_seconds > self.settings.LOCK_TTL_SECONDS: - logger.warning( - f"{self.name} found stale lock for pipeline {pipeline_id}, " - f"age: {age_seconds}s" - ) - container.delete_item(item=lock_id, partition_key=lock_id) - # Try to acquire again - return self._acquire_lock(pipeline_id) - - logger.debug( - f"{self.name} pipeline {pipeline_id} locked by " - f"{existing_lock.get('worker_id')}" - ) - return False - - except CosmosResourceNotFoundError: - # Lock was deleted between our attempts, try again - return self._acquire_lock(pipeline_id) - - except Exception as e: - logger.error(f"{self.name} error acquiring lock: {e}") - logger.error(traceback.format_exc()) - return False - - def _release_lock(self, pipeline_id: str): - """ - Release a distributed lock for a pipeline. - - Args: - pipeline_id: Pipeline identifier - """ - try: - database = self.cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - container = database.get_container_client(self.settings.COSMOS_DB_CONTAINER_VAULT_EXECUTION_LOCKS) - - lock_id = f"pipeline_{pipeline_id}_lock" - - try: - # Verify we own the lock before deleting - existing_lock = container.read_item( - item=lock_id, - partition_key=lock_id - ) - - if existing_lock.get('worker_id') == self.name: - container.delete_item(item=lock_id, partition_key=lock_id) - logger.debug(f"{self.name} released lock for pipeline {pipeline_id}") - else: - logger.warning( - f"{self.name} attempted to release lock owned by " - f"{existing_lock.get('worker_id')}" - ) - - except CosmosResourceNotFoundError: - # Lock already released or expired - logger.debug(f"{self.name} lock for pipeline {pipeline_id} already released") - - except Exception as e: - logger.error(f"{self.name} error releasing lock: {e}") - logger.error(traceback.format_exc()) - - def _get_pipeline_by_id(self, pipeline_id: str) -> Optional[Dict[str, Any]]: - """Get pipeline by ID from Cosmos DB""" - try: - database = self.cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - container = database.get_container_client(self.settings.COSMOS_DB_CONTAINER_PIPELINES) - - pipeline = container.read_item( - item=pipeline_id, - partition_key=pipeline_id - ) - - return pipeline - - except CosmosResourceNotFoundError: - logger.warning(f"{self.name} pipeline {pipeline_id} not found") - return None - except Exception as e: - logger.error(f"{self.name} error getting pipeline: {e}") - return None - - def _get_pipelines_with_vaults(self) -> List[tuple[str,str]]: - """Get enabled pipelines that have associated vaults""" - try: - database = self.cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - - # Query vaults container for enabled vaults - vaults_container = database.get_container_client(self.settings.COSMOS_DB_CONTAINER_VAULTS) - vault_pipelines = list(vaults_container.query_items( - query="SELECT DISTINCT c.id, c.pipeline_id FROM c WHERE c.enabled = true", - enable_cross_partition_query=True - )) - return [(vp["id"], vp["pipeline_id"]) for vp in vault_pipelines] - - except Exception as e: - logger.error(f"{self.name} error querying pipelines: {e}") - logger.error(traceback.format_exc()) - return [] - - async def _process_pipeline(self, pipeline: Dict[str, Any]): - """Process a single pipeline""" - pipeline_id = pipeline.get('id') - pipeline_name = pipeline.get('name') - - logger.info(f"{self.name}: processing pipeline '{pipeline_name}'") - - try: - # Parse pipeline YAML to find input executor - input_executor_config = self._find_input_executor(pipeline) - - if not input_executor_config: - logger.warning(f"{self.name}: no input executor found in pipeline {pipeline_name}") - return - - executor_id = input_executor_config.get('id') - logger.debug(f"{self.name}: found input executor: {executor_id}") - - # Retrieve checkpoint before execution - checkpoint_timestamp = self._get_checkpoint(pipeline_id, executor_id) - if checkpoint_timestamp: - logger.debug(f"{self.name}: resuming from checkpoint {checkpoint_timestamp.isoformat()}") - else: - logger.debug(f"{self.name}: no checkpoint found, starting fresh") - - # Track the latest timestamp for checkpoint - latest_timestamp = checkpoint_timestamp - execution_start_time = datetime.now(timezone.utc) - tasks_created = 0 - - # Execute input executor to discover content - async for content_items in self._execute_input_executor(pipeline_id=pipeline_id, - pipeline_name=pipeline_name, - executor_config=input_executor_config, - checkpoint_timestamp=checkpoint_timestamp): - if content_items is None: - logger.debug(f"{self.name}: no content discovered from pipeline {pipeline_name}") - self._create_no_content_discovered_execution_record(pipeline_id, pipeline_name) - break - - logger.debug(f"{self.name}: discovered {len(content_items)} content items") - - # Skip if no content, this might be the case if content is retrieved from the source but filtered out by the input executor - if len(content_items) == 0: - continue - - # Create processing tasks for each content item - try: - self._create_processing_task(pipeline, content_items, executor_id) - logger.debug(f"{self.name}: queued processing task for content with {len(content_items)} items") - - tasks_created += 1 - except Exception as e: - logger.error(f"{self.name}: failed to create processing task: {e}") - logger.exception(e) - - if self.stop_event.is_set(): - break - - logger.info(f"{self.name}: created {tasks_created} processing tasks for pipeline {pipeline_name}") - - # Save checkpoint after successful execution - # Use start time as the checkpoint for next execution - latest_timestamp = execution_start_time - self._save_checkpoint(pipeline_id, executor_id, latest_timestamp) - - except Exception as e: - logger.error(f"{self.name}: error processing pipeline {pipeline_name}: {e}") - logger.exception(e) - - def _find_input_executor(self, pipeline: Dict[str, Any]) -> Optional[Dict[str, Any]]: - """Find the input executor in the pipeline configuration""" - try: - yaml_content = pipeline.get('yaml') - if not yaml_content: - return None - - # Parse YAML - config = yaml.safe_load(yaml_content) - if not config: - return None - - # Get pipeline definition - pipeline_def = config.get('pipeline', config) - executors = pipeline_def.get('executors', []) - - # find the executor with input type by cross checking the executor-registry - for executor in executors: - executor_type = executor.get('type', '').lower() - executor_info = self.executor_registry.get_executor_info(executor_type) - if not executor_info: - continue - - if executor_info.category == 'input': - # Will always return the first input executor found - logger.debug(f"{self.name} found input executor: {executor.get('id')}") - return executor - - return None - - except Exception as e: - logger.error(f"{self.name} error parsing pipeline YAML: {e}") - return None - - async def _execute_input_executor(self, - pipeline_id: str, - pipeline_name:str, - executor_config: Dict[str, Any], - checkpoint_timestamp: Optional[datetime] = None - ) -> AsyncGenerator[List[Content], None]: - - """Execute an input executor to discover content""" - executor_type = executor_config.get('type') - executor_id = executor_config.get('id') - settings = executor_config.get('settings', {}) - - logger.debug(f"{self.name}: executing input executor {executor_id} (type: {executor_type}) with checkpoint {checkpoint_timestamp}") - - try: - # Create executor instance - executor = self._create_executor(executor_type, executor_id, settings) - - if not executor or not isinstance(executor, InputExecutor): - logger.error(f"{self.name}: failed to create executor of type {executor_type}. Returned {executor} which is not an InputExecutor.") - # create an execution record with failed status - self._create_failed_execution_record(pipeline_id=pipeline_id, - pipeline_name=pipeline_name, - error_message=f"{self.name}: failed to create executor of type {executor_type}, got {executor} which is not an InputExecutor.") - return - - # Execute to discover content - async for batch, has_more in executor.crawl(checkpoint_timestamp=checkpoint_timestamp): - # Ensure we have a list - if batch is not None: - yield batch - - if batch is None and has_more is False: - # No more items - yield None - - except Exception as e: - logger.error(f"{self.name}: error executing input executor: {e}") - logger.exception(e) - # Don't save checkpoint on error - will retry from last successful checkpoint - # create an execution record with failed status - self._create_failed_execution_record(pipeline_id=pipeline_id, - pipeline_name=pipeline_name, - error_message=f"{self.name}: {str(e)}. {traceback.format_exc()}") - raise - - def _create_executor(self, executor_type: str, executor_id: str, settings: Dict[str, Any]) -> BaseExecutor: - """Create an executor instance""" - try: - # Create instance config - instance_config = ExecutorInstanceConfig( - id=executor_id, - type=executor_type, - settings=settings - ) - - # Create executor using registry (dynamic loading) - executor = self.executor_registry.create_executor_instance( - executor_id=executor_type, - instance_config=instance_config - ) - - return executor - - except Exception as e: - logger.error(f"{self.name} error creating executor: {e}") - logger.exception(e) - raise - - def _create_execution_id(self) -> str: - """Generate a unique execution ID""" - return f"exec_{uuid.uuid4().hex[:12]}" - - def _create_processing_task( - self, - pipeline: Dict[str, Any], - content: List[Content], - executed_input_executor: str - ): - """Create and queue a content processing task""" - # Create execution record in Cosmos DB - execution_id = self._create_execution_id() - vault_id = self.pipeline_vaults.get(pipeline['id'], None) - - # Create processing task - processing_task = ContentProcessingTask( - task_id=f"task_{uuid.uuid4().hex[:12]}", - priority=TaskPriority.NORMAL, - pipeline_id=pipeline['id'], - pipeline_name=pipeline['name'], - vault_id=vault_id, - execution_id=execution_id, - content=content, - executed_input_executor=executed_input_executor, # Mark which executor was already run - max_retries=self.settings.MAX_TASK_RETRIES - ) - - try: - # Create execution record - self._create_execution_record(execution_id, processing_task) - - try: - # Send to queue - self.queue_client.send_content_processing_task(processing_task) - except Exception as e: - logger.error(f"{self.name} error sending processing task to queue: {e}") - # Mark execution as failed in Cosmos DB - self._mark_execution_failed(execution_id, str(e)) - raise - - logger.debug(f"{self.name} created processing task for {len(content)} content items") - except Exception as e: - logger.error(f"{self.name} error execution record or queue creating processing task: {e}") - raise - - def _create_execution_record( - self, - execution_id: str, - task: ContentProcessingTask - ): - """Create execution record in Cosmos DB""" - try: - database = self.cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - container = database.get_container_client(self.settings.COSMOS_DB_CONTAINER_VAULT_EXECUTIONS) - - vault_id = self.pipeline_vaults.get(task.pipeline_id, None) - - execution = { - "id": execution_id, - "pipeline_id": task.pipeline_id, - "pipeline_name": task.pipeline_name, - "vault_id": vault_id, - "status": "pending", - "status_message": "Task queued for processing with input content of size " + str(len(task.content)), - "task_id": task.task_id, - "source_worker_id": self.name, - "content": [make_safe_json(c) for c in task.content], - "number_of_items": len(task.content) if task.content else 0, - "created_at": datetime.now(timezone.utc).isoformat(), - "created_by": self.name, - "updated_at": datetime.now(timezone.utc).isoformat(), - "started_at": None, - "completed_at": None, - "error": None, - "executor_outputs": {}, - "events": [] - } - - container.create_item(execution) - logger.debug(f"{self.name} created execution record {execution_id}") - except Exception as e: - logger.error(f"{self.name} error creating execution record: {e}") - raise - - def _create_no_content_discovered_execution_record(self, pipeline_id: str, pipeline_name: str): - """Create an execution record for no content discovered""" - try: - execution_id = self._create_execution_id() - - database = self.cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - container = database.get_container_client(self.settings.COSMOS_DB_CONTAINER_VAULT_EXECUTIONS) - - vault_id = self.pipeline_vaults.get(pipeline_id, None) - - execution = { - "id": execution_id, - "pipeline_id": pipeline_id, - "pipeline_name": pipeline_name, - "vault_id": vault_id, - "status": "completed", - "status_message": "No content discovered during execution", - "task_id": None, - "source_worker_id": self.name, - "error": None, - "content": None, - "number_of_items": None, - "created_at": datetime.now(timezone.utc).isoformat(), - "created_by": self.name, - "updated_at": datetime.now(timezone.utc).isoformat(), - "started_at": None, - "completed_at": datetime.now(timezone.utc).isoformat(), - "executor_outputs": {}, - "events": [] - } - - container.create_item(execution) - logger.debug(f"{self.name} created no-content-discovered execution record {execution_id}") - except Exception as e: - logger.error(f"{self.name} error creating no-content-discovered execution record: {e}") - raise - - def _create_failed_execution_record(self, pipeline_id: str, pipeline_name: str, error_message: str): - """Create a failed execution record in Cosmos DB""" - try: - execution_id = self._create_execution_id() - - database = self.cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - container = database.get_container_client(self.settings.COSMOS_DB_CONTAINER_VAULT_EXECUTIONS) - - vault_id = self.pipeline_vaults.get(pipeline_id, None) - - execution = { - "id": execution_id, - "pipeline_id": pipeline_id, - "pipeline_name": pipeline_name, - "vault_id": vault_id, - "status": "failed", - "status_message": "Execution failed. View error for details.", - "task_id": None, - "source_worker_id": self.name, - "content": None, - "number_of_items": None, - "error": error_message, - "created_at": datetime.now(timezone.utc).isoformat(), - "created_by": self.name, - "updated_at": datetime.now(timezone.utc).isoformat(), - "started_at": None, - "completed_at": None, - "executor_outputs": {}, - "events": [] - } - - container.create_item(execution) - logger.debug(f"{self.name} created failed execution record {execution_id}") - except Exception as e: - logger.error(f"{self.name} error creating failed execution record: {e}") - raise - - def _mark_execution_failed(self, execution_id: str, error_message: str): - """Mark an execution as failed in Cosmos DB""" - try: - database = self.cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - container = database.get_container_client(self.settings.COSMOS_DB_CONTAINER_VAULT_EXECUTIONS) - - # Read existing execution - execution = container.read_item( - item=execution_id, - partition_key=execution_id - ) - - # Update status and error message - execution['status'] = 'failed' - execution['status_message'] = "Execution failed. View error for details." - execution['error'] = error_message - execution['updated_at'] = datetime.now(timezone.utc).isoformat() - - container.upsert_item(body=execution) - logger.debug(f"{self.name} marked execution {execution_id} as failed") - - except Exception as e: - logger.error(f"{self.name} error marking execution as failed: {e}") - - def _get_checkpoint(self, pipeline_id: str, executor_id: str) -> Optional[datetime]: - """ - Retrieve the last checkpoint timestamp for a pipeline's input executor. - - Args: - pipeline_id: Pipeline identifier - executor_id: Executor identifier - - Returns: - Last checkpoint timestamp, or None if no checkpoint exists - """ - try: - database = self.cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - container = database.get_container_client(self.settings.COSMOS_DB_CONTAINER_CRAWL_CHECKPOINTS) - - checkpoint_id = f"{pipeline_id}_{executor_id}" - - try: - checkpoint_doc = container.read_item( - item=checkpoint_id, - partition_key=checkpoint_id - ) - - timestamp_str = checkpoint_doc.get('checkpoint_timestamp') - if timestamp_str: - return datetime.fromisoformat(timestamp_str) - - return None - - except CosmosResourceNotFoundError: - # No checkpoint exists yet - logger.debug(f"{self.name} no checkpoint found for {pipeline_id}/{executor_id}") - return None - - except Exception as e: - logger.error(f"{self.name} error retrieving checkpoint: {e}") - logger.error(traceback.format_exc()) - # Return None to start fresh on error - return None - - def _save_checkpoint(self, pipeline_id: str, executor_id: str, checkpoint_timestamp: datetime): - """ - Save a checkpoint timestamp for a pipeline's input executor. - - Args: - pipeline_id: Pipeline identifier - executor_id: Executor identifier - checkpoint_timestamp: Timestamp to save as checkpoint - """ - try: - database = self.cosmos_client.get_database_client(self.settings.COSMOS_DB_NAME) - container = database.get_container_client(self.settings.COSMOS_DB_CONTAINER_CRAWL_CHECKPOINTS) - - vault_id = self.pipeline_vaults.get(pipeline_id, None) - checkpoint_id = f"{pipeline_id}_{executor_id}" - - checkpoint_doc = { - "id": checkpoint_id, - "pipeline_id": pipeline_id, - "vault_id": vault_id, - "executor_id": executor_id, - "checkpoint_timestamp": checkpoint_timestamp.isoformat(), - "worker_id": self.name - } - - # Upsert the checkpoint (create or update) - container.upsert_item(checkpoint_doc) - logger.debug(f"{self.name} saved checkpoint for {pipeline_id}/{executor_id}") - - except Exception as e: - logger.error(f"{self.name} error saving checkpoint: {e}") - logger.error(traceback.format_exc()) - # Log error but don't fail the execution - - def _signal_handler(self, signum, frame): - """Handle shutdown signals""" - logger.info(f"{self.name} received signal {signum}") - self.stop_event.set() diff --git a/contentflow-worker/main.py b/contentflow-worker/main.py deleted file mode 100644 index 12caf23..0000000 --- a/contentflow-worker/main.py +++ /dev/null @@ -1,110 +0,0 @@ -""" -ContentFlow Worker - Multi-processing content processing engine. - -This is the main entry point for the ContentFlow worker service. -It starts the worker engine which manages processing and source workers. -It also starts a FastAPI health/status monitoring API. -""" -import logging -import sys -import threading -from pathlib import Path -import uvicorn - -from contentflow.utils import setup_logging - -from app.engine import WorkerEngine -from app.settings import get_settings, WorkerSettings -from app.startup import run_startup_checks -from app.api import create_app - -def print_worker_banner(): - """Print worker banner""" - banner = """ - ╔════════════════════════════════════════════════════════╗ - ║ ║ - ║ ◆ ║ - ║ / ╲ ║ - ║ / ╲ ContentFlow ║ - ║ ◆ ◆ ║ - ║ ╲ / Worker Service ║ - ║ ╲ / ║ - ║ ◆ ║ - ║ ║ - ╚════════════════════════════════════════════════════════╝ - """ - print(banner) - - -def main(settings: WorkerSettings): - """Main entry point""" - - print_worker_banner() - print("=" * 60) - print("ContentFlow Worker Service") - print("=" * 60) - print(f"Worker Name: {settings.WORKER_NAME}") - print(f"Processing Workers: {settings.NUM_PROCESSING_WORKERS}") - print(f"Source Workers: {settings.NUM_SOURCE_WORKERS}") - print(f"Queue: {settings.STORAGE_WORKER_QUEUE_NAME}") - if settings.API_ENABLED: - print(f"API Enabled: http://{settings.API_HOST}:{settings.API_PORT}") - print("=" * 60) - - # Run startup checks - print("Running startup validation checks...") - if not run_startup_checks(settings): - print("\033[91m❌ Startup validation failed. Exiting.\033[0m") - sys.exit(1) - else: - print("✅ Startup validation passed.") - - # Create worker engine - print("Creating worker engine...") - engine = WorkerEngine(settings) - - # Start API server in a separate thread if enabled - api_thread = None - if settings.API_ENABLED: - print(f"Starting API server on {settings.API_HOST}:{settings.API_PORT}...") - app = create_app(settings, engine) - - # Run uvicorn in a separate thread - api_thread = threading.Thread( - target=uvicorn.run, - kwargs={ - "app": app, - "reload": False, - "host": settings.API_HOST, - "port": settings.API_PORT, - "log_level": settings.LOG_LEVEL.lower(), - "log_config": None # Disable uvicorn's default logging config - }, - daemon=True, - name="APIServerThread" - ) - api_thread.start() - print("✅ API server started.") - - print("Starting worker engine...") - - try: - # Run the worker engine (blocking) - engine.run() - - except Exception as e: - print(f"\033[91m🚨 Fatal error: {e}\033[0m") - sys.exit(1) - finally: - # Cleanup - if api_thread and api_thread.is_alive(): - print("API server will stop with the main process...") - - print("Worker service stopped") - -settings = get_settings() -setup_logging(settings.LOG_LEVEL) - - -if __name__ == "__main__": - main(settings) \ No newline at end of file diff --git a/contentflow-worker/requirements.txt b/contentflow-worker/requirements.txt deleted file mode 100644 index 3ecab87..0000000 --- a/contentflow-worker/requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -# ContentFlow Worker Dependencies - -# Core dependencies -pydantic>=2.12.5 -pydantic-settings==2.12.0 -python-dotenv>=1.2.1 -pyyaml>=6.0.3 - -# Azure SDK -azure-identity==1.25.1 -azure-storage-queue==12.14.1 -azure-cosmos==4.14.3 -azure-storage-blob==12.27.1 - -# FastAPI framework -fastapi==0.128.0 -uvicorn[standard]==0.40.0 \ No newline at end of file diff --git a/infra/bicep/main.bicep b/infra/bicep/main.bicep index 47af02c..7d2ebcf 100644 --- a/infra/bicep/main.bicep +++ b/infra/bicep/main.bicep @@ -21,6 +21,12 @@ param location string = resourceGroup().location @description('Location for AI Foundry resources') param foundryLocation string +@description('Location for Cosmos DB resources (override if primary region has capacity issues)') +param cosmosLocation string = location + +@description('Resource ID of an existing Azure OpenAI resource for RBAC assignment (if not using AI Foundry integrated endpoint)') +param existingOpenAIResourceId string = '' + @description('ID of the user running the deployment') param principalId string = '' @@ -42,6 +48,9 @@ param existingCognitiveServicesPrivateDnsZoneId string = '' @description('Resource ID of existing Blob Storage Private DNS Zone (required for ailz-integrated mode)') param existingBlobPrivateDnsZoneId string = '' +@description('Resource ID of existing Queue Storage Private DNS Zone (optional for ailz-integrated mode)') +param existingQueuePrivateDnsZoneId string = '' + @description('Resource ID of existing Cosmos DB Private DNS Zone (required for ailz-integrated mode)') param existingCosmosPrivateDnsZoneId string = '' @@ -84,15 +93,6 @@ param docsContainerName string = 'content' @description('API Container App target port') param apiContainerAppTargetPort int = 8090 -@description('Worker Container App target port') -param workerContainerAppTargetPort int = 8099 - -@description('Enable API service in worker container app - used for health checks') -param workerAPIEnabled bool = true - -@description('Queue name for worker tasks') -param workerQueueName string = 'contentflow-execution-requests' - // ========== DEPLOYMENT MODE VALIDATION ========== var isBasic = deploymentMode == 'basic' var isAILZIntegrated = deploymentMode == 'ailz-integrated' @@ -108,7 +108,8 @@ var ailzValidation = isAILZIntegrated ? { cosmosPrivateDnsZoneRequired: !empty(existingCosmosPrivateDnsZoneId) ?? fail('existingCosmosPrivateDnsZoneId is required for ailz-integrated mode') appConfigPrivateDnsZoneRequired: !empty(existingAppConfigPrivateDnsZoneId) ?? fail('existingAppConfigPrivateDnsZoneId is required for ailz-integrated mode') acrPrivateDnsZoneRequired: !empty(existingAcrPrivateDnsZoneId) ?? fail('existingAcrPrivateDnsZoneId is required for ailz-integrated mode') - containerAppsEnvPrivateDnsZoneRequired: !empty(existingContainerAppsEnvPrivateDnsZoneId) ?? fail('existingContainerAppsEnvPrivateDnsZoneId is required for ailz-integrated mode') + // CAE DNS zone is NOT validated here — it is created dynamically by cae-dns-zone.bicep + // using the CAE's defaultDomain and staticIp after provisioning. No pre-existing zone needed. } : {} // ========== VARIABLES ========== @@ -130,6 +131,7 @@ var networkConfig = isAILZIntegrated ? { privateDnsZoneIds: { cognitiveServices: existingCognitiveServicesPrivateDnsZoneId blob: existingBlobPrivateDnsZoneId + queue: existingQueuePrivateDnsZoneId cosmos: existingCosmosPrivateDnsZoneId appConfig: existingAppConfigPrivateDnsZoneId acr: existingAcrPrivateDnsZoneId @@ -151,7 +153,6 @@ var appConfigStoreName = 'appcs-${resourceToken}' var containerRegistryName = 'cr${resourceToken}' var containerAppsEnvironmentName = 'cae-${resourceToken}' var apiContainerAppName = 'api-${resourceToken}' -var workerContainerAppName = 'worker-${resourceToken}' var webContainerAppName = 'web-${resourceToken}' // ========== MODULES DEPLOYMENT ========= @@ -204,6 +205,23 @@ var appInsightsConnectionString = !empty(existingAppInsightsId) : (shouldCreateAppInsights ? appInsights!.outputs.connectionString : '') +// ========== QUEUE PRIVATE DNS ZONE (AILZ mode, auto-create if not provided) ========== +// When no existing queue DNS zone is provided, create one so the storage queue +// private endpoint can resolve via private IP within the VNet +var shouldCreateQueueDnsZone = isAILZIntegrated && empty(existingQueuePrivateDnsZoneId) + +module queueDnsZone 'modules/queue-dns-zone.bicep' = if (shouldCreateQueueDnsZone) { + name: 'queue-dns-${resourceToken}' + params: { + vnetResourceId: existingVnetResourceId + tags: tags + } +} + +var resolvedQueueDnsZoneId = isAILZIntegrated + ? (!empty(existingQueuePrivateDnsZoneId) ? existingQueuePrivateDnsZoneId : (shouldCreateQueueDnsZone ? queueDnsZone.outputs.dnsZoneId : '')) + : '' + // ========== STORAGE ACCOUNT ========== // Create new Storage Account, with private endpoint support (if using AILZ integrated mode), // or use public endpoints (basic mode) @@ -218,6 +236,7 @@ module storage 'modules/storage.bicep' = { enablePrivateEndpoint: isAILZIntegrated privateEndpointSubnetId: isAILZIntegrated ? networkConfig.privateEndpointSubnetId : '' blobPrivateDnsZoneId: isAILZIntegrated ? networkConfig.privateDnsZoneIds.blob : '' + queuePrivateDnsZoneId: resolvedQueueDnsZoneId publicNetworkAccess: isAILZIntegrated ? 'Disabled' : 'Enabled' logAnalyticsWorkspaceId: logAnalyticsWorkspaceId tags: tags @@ -232,7 +251,7 @@ module storage 'modules/storage.bicep' = { module cosmos 'modules/cosmos.bicep' = { name: 'cosmos-${resourceToken}' params: { - location: location + location: cosmosLocation cosmosAccountName: cosmosAccountName cosmosDbName: cosmosDbName cosmosDBContainerNames: cosmosDBContainerNames @@ -259,7 +278,7 @@ module appConfigStore 'modules/app-config-store.bicep' = { enablePrivateEndpoint: isAILZIntegrated privateEndpointSubnetId: isAILZIntegrated ? networkConfig.privateEndpointSubnetId : '' appConfigPrivateDnsZoneId: isAILZIntegrated ? networkConfig.privateDnsZoneIds.appConfig : '' - publicNetworkAccess: isAILZIntegrated ? 'Disabled' : 'Enabled' + publicNetworkAccess: 'Enabled' // Must be Enabled during provisioning for ARM to write key-values via data-plane proxy tags: tags } } @@ -290,16 +309,6 @@ module appConfigStoreKeys 'modules/app-config-store-keys.bicep' = { name: 'contentflow.common.BLOB_STORAGE_CONTAINER_NAME' value: docsContainerName } - { - contentType: 'text/plain' - name: 'contentflow.common.STORAGE_ACCOUNT_WORKER_QUEUE_URL' - value: storage!.outputs.primaryQueueEndpoint - } - { - contentType: 'text/plain' - name: 'contentflow.common.STORAGE_WORKER_QUEUE_NAME' - value: workerQueueName - } // API settings { contentType: 'text/plain' @@ -341,87 +350,6 @@ module appConfigStoreKeys 'modules/app-config-store-keys.bicep' = { name: 'contentflow.api.CORS_ALLOW_ORIGINS' value: '*' } - { - contentType: 'text/plain' - name: 'contentflow.api.WORKER_ENGINE_API_ENDPOINT' - value: 'https://${workerContainerApp.outputs.fqdn}' - } - // Worker settings - { - contentType: 'text/plain' - name: 'contentflow.worker.NUM_PROCESSING_WORKERS' - value: '4' - } - { - contentType: 'text/plain' - name: 'contentflow.worker.NUM_SOURCE_WORKERS' - value: '2' - } - { - contentType: 'text/plain' - name: 'contentflow.worker.LOG_LEVEL' - value: 'INFO' - } - { - contentType: 'text/plain' - name: 'contentflow.worker.DEBUG' - value: 'false' - } - { - contentType: 'text/plain' - name: 'contentflow.worker.API_ENABLED' - value: '${workerAPIEnabled}' - } - { - contentType: 'text/plain' - name: 'contentflow.worker.API_HOST' - value: '0.0.0.0' - } - { - contentType: 'text/plain' - name: 'contentflow.worker.API_PORT' - value: '${workerContainerAppTargetPort}' - } - { - contentType: 'text/plain' - name: 'contentflow.worker.QUEUE_POLL_INTERVAL_SECONDS' - value: '5' - } - { - contentType: 'text/plain' - name: 'contentflow.worker.QUEUE_VISIBILITY_TIMEOUT_SECONDS' - value: '300' - } - { - contentType: 'text/plain' - name: 'contentflow.worker.QUEUE_MAX_MESSAGES' - value: '32' - } - { - contentType: 'text/plain' - name: 'contentflow.worker.MAX_TASK_RETRIES' - value: '3' - } - { - contentType: 'text/plain' - name: 'contentflow.worker.TASK_TIMEOUT_SECONDS' - value: '600' - } - { - contentType: 'text/plain' - name: 'contentflow.worker.DEFAULT_POLLING_INTERVAL_SECONDS' - value: '60' - } - { - contentType: 'text/plain' - name: 'contentflow.worker.SCHEDULER_SLEEP_INTERVAL_SECONDS' - value: '5' - } - { - contentType: 'text/plain' - name: 'contentflow.worker.LOCK_TTL_SECONDS' - value: '300' - } { contentType: 'text/plain' name: 'sentinel' @@ -430,7 +358,7 @@ module appConfigStoreKeys 'modules/app-config-store-keys.bicep' = { ] } dependsOn: [ - // appConfigStore + appConfigStore ] } @@ -445,7 +373,7 @@ module containerRegistry 'modules/container-registry.bicep' = { location: location roleAssignedManagedIdentityPrincipalIds: [userAssignedIdentity.outputs.principalId] enablePrivateEndpoint: isAILZIntegrated - privateEndpointSubnetId: isAILZIntegrated ? networkConfig.privateEndpointsSubnetId : '' + privateEndpointSubnetId: isAILZIntegrated ? networkConfig.privateEndpointSubnetId : '' acrPrivateDnsZoneId: isAILZIntegrated ? networkConfig.privateDnsZoneIds.acr : '' publicNetworkAccess: isAILZIntegrated ? 'Disabled' : 'Enabled' tags: tags @@ -456,17 +384,17 @@ module containerRegistry 'modules/container-registry.bicep' = { // Create new Container Apps Environment, with private endpoint support (if using AILZ integrated mode), // or use public endpoints (basic mode) +// Pass resolved Log Analytics workspace resource ID to the module (it derives customerId/sharedKey internally) + module containerAppsEnvironment 'modules/container-apps-environment.bicep' = { name: 'cae-la-${resourceToken}' params: { containerAppsEnvironmentName: containerAppsEnvironmentName - logAnalyticsWorkspaceId: !empty(existingLogAnalyticsWorkspaceId) ? existingLogAnalyticsWorkspaceId : logAnalytics!.outputs.logAnalyticsWorkspaceId - logAnalyticsPrimarySharedKey: !empty(existingLogAnalyticsWorkspaceId) ? listKeys(existingLogAnalyticsWorkspaceId, '2021-12-01-preview').primarySharedKey : logAnalytics.outputs.primarySharedKey + logAnalyticsWorkspaceResourceId: logAnalyticsWorkspaceId userAssignedResourceIds: [userAssignedIdentity.outputs.resourceId] location: location enablePrivateEndpoint: isAILZIntegrated privateEndpointSubnetId: isAILZIntegrated ? networkConfig.containerAppsSubnetId : null - // caePrivateDnsZoneId: isAILZIntegrated ? networkConfig.privateDnsZoneIds.existingContainerAppsEnvPrivateDnsZoneId : '' publicNetworkAccess: isAILZIntegrated ? 'Disabled' : 'Enabled' tags: tags } @@ -483,22 +411,46 @@ module aiFoundry 'modules/ai-foundry.bicep' = { } } +// ========== OPENAI RBAC ROLE ASSIGNMENT (for standalone Azure OpenAI resources) ========== +module openaiRoleAssignment 'modules/openai-role-assignment.bicep' = if (!empty(existingOpenAIResourceId)) { + name: 'openai-role-${resourceToken}' + params: { + openAIResourceId: existingOpenAIResourceId + principalIds: !empty(principalId) + ? [userAssignedIdentity.outputs.principalId, principalId] + : [userAssignedIdentity.outputs.principalId] + } +} + +// ========== CAE PRIVATE DNS ZONE (AILZ mode only) ========== +// Internal CAE requires a private DNS zone matching its default domain +// with a wildcard A record pointing to its static IP for DNS resolution +module caeDnsZone 'modules/cae-dns-zone.bicep' = if (isAILZIntegrated) { + name: 'cae-dns-${resourceToken}' + params: { + caeDefaultDomain: containerAppsEnvironment.outputs.defaultDomain + caeStaticIp: containerAppsEnvironment.outputs.staticIp + vnetResourceId: existingVnetResourceId + tags: tags + } +} + // ========== API CONTAINER APP ========== module apiContainerApp 'modules/container-app.bicep' = { name: 'ca-api-${resourceToken}' params: { name: apiContainerAppName - location: containerAppsEnvironment!.outputs.location - containerAppsEnvId: containerAppsEnvironment!.outputs.resourceId + location: containerAppsEnvironment.outputs.location + containerAppsEnvId: containerAppsEnvironment.outputs.resourceId containerRegistryServer: containerRegistry!.outputs.loginServer managedIdentityId: userAssignedIdentity.outputs.resourceId targetPort: 8090 - externalIngress: !isAILZIntegrated + externalIngress: true corsEnabled: true - livenessProbePath: '/' + livenessProbePath: '' // Disabled for initial provisioning with placeholder image cpuCores: 2 memoryInGB: '4Gi' - minReplicas: 1 + minReplicas: 0 // Set to 0 for initial provisioning - container will scale up when real image is deployed maxReplicas: 2 environmentVariables: [ { @@ -513,62 +465,39 @@ module apiContainerApp 'modules/container-app.bicep' = { name: 'AZURE_CLIENT_ID' value: userAssignedIdentity.outputs.clientId } - ] - tags: union(tags, { 'azd-service-name': 'api' }) - } -} - -// ========== WORKER CONTAINER APP ========== -module workerContainerApp 'modules/container-app.bicep' = { - name: 'ca-worker-${resourceToken}' - params: { - name: workerContainerAppName - location: containerAppsEnvironment!.outputs.location - containerAppsEnvId: containerAppsEnvironment!.outputs.resourceId - containerRegistryServer: containerRegistry!.outputs.loginServer - managedIdentityId: userAssignedIdentity.outputs.resourceId - targetPort: workerContainerAppTargetPort - externalIngress: !isAILZIntegrated - corsEnabled: true - livenessProbePath: '/' - cpuCores: 2 - memoryInGB: '4Gi' - minReplicas: 1 - maxReplicas: 3 - environmentVariables: [ { - name: 'AZURE_APP_CONFIG_ENDPOINT' - value: appConfigStore!.outputs.endpoint + name: 'AZURE_STORAGE_ACCOUNT_NAME' + value: storageAccountName } { - name: 'APPLICATIONINSIGHTS_CONNECTION_STRING' - value: appInsightsConnectionString + name: 'AZURE_CONTENT_UNDERSTANDING_ENDPOINT' + value: 'https://${aiFoundry.outputs.aiServicesName}.services.ai.azure.com' } { - name: 'AZURE_CLIENT_ID' - value: userAssignedIdentity.outputs.clientId + name: 'AZURE_OPENAI_ENDPOINT' + value: 'https://${aiFoundry.outputs.aiServicesName}.services.ai.azure.com' } ] - tags: union(tags, { 'azd-service-name': 'worker' }) + tags: union(tags, { 'azd-service-name': 'api' }) } } -// ========== API CONTAINER APP ========== +// ========== WEB CONTAINER APP ========== module webContainerApp 'modules/container-app.bicep' = { name: 'ca-web-${resourceToken}' params: { name: webContainerAppName - location: containerAppsEnvironment!.outputs.location - containerAppsEnvId: containerAppsEnvironment!.outputs.resourceId + location: containerAppsEnvironment.outputs.location + containerAppsEnvId: containerAppsEnvironment.outputs.resourceId containerRegistryServer: containerRegistry!.outputs.loginServer managedIdentityId: userAssignedIdentity.outputs.resourceId targetPort: 8080 - externalIngress: !isAILZIntegrated + externalIngress: true corsEnabled: true - livenessProbePath: '/' + livenessProbePath: '' // Disabled for initial provisioning with placeholder image cpuCores: 1 memoryInGB: '2Gi' - minReplicas: 1 + minReplicas: 0 // Set to 0 for initial provisioning - container will scale up when real image is deployed maxReplicas: 1 environmentVariables: [ { @@ -592,7 +521,6 @@ output AZURE_CONTAINER_REGISTRY_NAME string = containerRegistry.outputs.name // Service endpoints output API_ENDPOINT string = 'https://${apiContainerApp.outputs.fqdn}' -output WORKER_ENDPOINT string = 'https://${workerContainerApp.outputs.fqdn}' output WEB_ENDPOINT string = 'https://${webContainerApp.outputs.fqdn}' // Resource outputs @@ -600,7 +528,6 @@ output COSMOS_DB_ENDPOINT string = cosmos.outputs.cosmosEndpoint output COSMOS_DB_NAME string = cosmosDbName output STORAGE_ACCOUNT_NAME string = storage.outputs.name output STORAGE_QUEUE_URL string = storage.outputs.primaryQueueEndpoint -output STORAGE_QUEUE_NAME string = workerQueueName output APP_CONFIG_ENDPOINT string = appConfigStore.outputs.endpoint output APPLICATIONINSIGHTS_CONNECTION_STRING string = appInsightsConnectionString diff --git a/infra/bicep/main.parameters.json b/infra/bicep/main.parameters.json index 57ef45e..d9db415 100644 --- a/infra/bicep/main.parameters.json +++ b/infra/bicep/main.parameters.json @@ -14,6 +14,12 @@ "foundryLocation": { "value": "${AZURE_AI_FOUNDRY_LOCATION}" }, + "cosmosLocation": { + "value": "${AZURE_COSMOS_LOCATION=eastus}" + }, + "existingOpenAIResourceId": { + "value": "${EXISTING_OPENAI_RESOURCE_ID=}" + }, "principalId": { "value": "${AZURE_PRINCIPAL_ID}" }, @@ -32,6 +38,9 @@ "existingBlobPrivateDnsZoneId": { "value": "${EXISTING_BLOB_PRIVATE_DNS_ZONE_ID=}" }, + "existingQueuePrivateDnsZoneId": { + "value": "${EXISTING_QUEUE_PRIVATE_DNS_ZONE_ID=}" + }, "existingCosmosPrivateDnsZoneId": { "value": "${EXISTING_COSMOS_PRIVATE_DNS_ZONE_ID=}" }, diff --git a/infra/bicep/modules/ai-foundry.bicep b/infra/bicep/modules/ai-foundry.bicep index aaae5ea..049e1c8 100644 --- a/infra/bicep/modules/ai-foundry.bicep +++ b/infra/bicep/modules/ai-foundry.bicep @@ -10,8 +10,16 @@ param roleAssignedManagedIdentityPrincipalIds string[] = [] @description('Tags for resources') param tags object = {} -// Ensure unique role assignments -var uniqueRoleAssignmentManagedIdentities = union(roleAssignedManagedIdentityPrincipalIds, roleAssignedManagedIdentityPrincipalIds) +// NOTE: User/human deployer role assignments (Azure AI Developer) are intentionally NOT done here. +// ARM role assignments have deterministic GUIDs computed from scope+principal+role. If the same role +// was previously assigned via CLI (random GUID), Bicep will conflict with a RoleAssignmentExists error +// and fail the entire deployment. post-provision.sh owns all user role assignments exclusively, +// using `az role assignment create` which is idempotent and handles duplicates gracefully. +var managedIdentityRoleAssignments = [for principalId in roleAssignedManagedIdentityPrincipalIds: { + principalId: principalId + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: '53ca6127-db72-4b80-b1b0-d745d6d5456d' // 'Azure AI User' +}] module aiFoundry 'br/public:avm/ptn/ai-ml/ai-foundry:0.5.0' = { params: { @@ -31,13 +39,7 @@ module aiFoundry 'br/public:avm/ptn/ai-ml/ai-foundry:0.5.0' = { displayName: 'ContentFlow' name: 'contentflow-project' } - roleAssignments: [ - for principalId in uniqueRoleAssignmentManagedIdentities: { - principalId: principalId - principalType: 'ServicePrincipal' - roleDefinitionIdOrName: '53ca6127-db72-4b80-b1b0-d745d6d5456d' // 'Azure AI User' - } - ] + roleAssignments: managedIdentityRoleAssignments sku: 'S0' } aiModelDeployments: [ @@ -65,6 +67,18 @@ module aiFoundry 'br/public:avm/ptn/ai-ml/ai-foundry:0.5.0' = { name: 'GlobalStandard' } } + { + model: { + format: 'OpenAI' + name: 'text-embedding-3-large' + version: '1' + } + name: 'text-embedding-3-large' + sku: { + capacity: 100 + name: 'Standard' + } + } ] // aiSearchConfiguration: { // name: '' diff --git a/infra/bicep/modules/app-config-store.bicep b/infra/bicep/modules/app-config-store.bicep index 565e447..3e70aed 100644 --- a/infra/bicep/modules/app-config-store.bicep +++ b/infra/bicep/modules/app-config-store.bicep @@ -64,17 +64,15 @@ module appConfigStore 'br/public:avm/res/app-configuration/configuration-store:0 resourceGroupResourceId: resourceGroup().id subnetResourceId: privateEndpointSubnetId privateLinkServiceConnectionName: '${appConfigStoreName}-app-config-plsc' - privateDnsZoneGroups: [ - { - name: 'app-config-dns-zone-group' - privateDnsZoneGroupConfigs: !empty(appConfigPrivateDnsZoneId) ? [ - { - name: 'app-config' - privateDnsZoneResourceId: appConfigPrivateDnsZoneId - } - ] : [] - } - ] + privateDnsZoneGroup: !empty(appConfigPrivateDnsZoneId) ? { + name: 'app-config-dns-zone-group' + privateDnsZoneGroupConfigs: [ + { + name: 'app-config' + privateDnsZoneResourceId: appConfigPrivateDnsZoneId + } + ] + } : null } ] : [] } diff --git a/infra/bicep/modules/cae-dns-zone.bicep b/infra/bicep/modules/cae-dns-zone.bicep new file mode 100644 index 0000000..35a6ead --- /dev/null +++ b/infra/bicep/modules/cae-dns-zone.bicep @@ -0,0 +1,64 @@ +@description('The default domain of the Container Apps Environment (e.g. ambitiousdesert-f1f7e820.eastus.azurecontainerapps.io)') +param caeDefaultDomain string + +@description('The static IP of the Container Apps Environment') +param caeStaticIp string + +@description('Resource ID of the VNet to link the DNS zone to') +param vnetResourceId string + +@description('Tags for resources') +param tags object = {} + +// Create a Private DNS Zone matching the CAE default domain +// Internal CAE requires DNS resolution of *.defaultDomain → staticIp +resource privateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = { + name: caeDefaultDomain + location: 'global' + tags: tags +} + +// Wildcard A record so all container app FQDNs resolve to the CAE static IP +resource wildcardRecord 'Microsoft.Network/privateDnsZones/A@2024-06-01' = { + parent: privateDnsZone + name: '*' + properties: { + ttl: 300 + aRecords: [ + { + ipv4Address: caeStaticIp + } + ] + } +} + +// Also add an @ record for the apex domain +resource apexRecord 'Microsoft.Network/privateDnsZones/A@2024-06-01' = { + parent: privateDnsZone + name: '@' + properties: { + ttl: 300 + aRecords: [ + { + ipv4Address: caeStaticIp + } + ] + } +} + +// Link the DNS zone to the VNet so jumpbox and other VNet resources can resolve +resource vnetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = { + parent: privateDnsZone + name: '${last(split(vnetResourceId, '/'))}-cae-link' + location: 'global' + tags: tags + properties: { + virtualNetwork: { + id: vnetResourceId + } + registrationEnabled: false + } +} + +output dnsZoneId string = privateDnsZone.id +output dnsZoneName string = privateDnsZone.name diff --git a/infra/bicep/modules/container-app.bicep b/infra/bicep/modules/container-app.bicep index 2156c6d..5f4af16 100644 --- a/infra/bicep/modules/container-app.bicep +++ b/infra/bicep/modules/container-app.bicep @@ -46,6 +46,9 @@ param environmentVariables array = [] @description('Tags to apply to resources') param tags object = {} +@description('Container image to use. Defaults to MCR quickstart (no ACR dependency during provisioning).') +param containerImage string = '' + // Use Azure Verified Module for Container App module containerApp 'br:mcr.microsoft.com/bicep/avm/res/app/container-app:0.19.0' = { @@ -66,8 +69,8 @@ module containerApp 'br:mcr.microsoft.com/bicep/avm/res/app/container-app:0.19.0 containers: [ { name: name - // Use base image as required by azd - will be replaced during deployment - image: 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' + // Use MCR quickstart image for initial provisioning (reachable even with private ACR); azd deploy replaces with real image + image: !empty(containerImage) ? containerImage : 'mcr.microsoft.com/k8se/quickstart:latest' resources: { cpu: cpuCores memory: memoryInGB @@ -100,10 +103,10 @@ module containerApp 'br:mcr.microsoft.com/bicep/avm/res/app/container-app:0.19.0 identity: managedIdentityId } ] - scaleSettings: maxReplicas > 1 ? { + scaleSettings: { minReplicas: minReplicas maxReplicas: maxReplicas - rules: [ + rules: maxReplicas > 1 ? [ { name: 'http-scaler' http: { @@ -112,8 +115,8 @@ module containerApp 'br:mcr.microsoft.com/bicep/avm/res/app/container-app:0.19.0 } } } - ] - } : null + ] : [] + } } } diff --git a/infra/bicep/modules/container-apps-environment.bicep b/infra/bicep/modules/container-apps-environment.bicep index c651728..8975159 100644 --- a/infra/bicep/modules/container-apps-environment.bicep +++ b/infra/bicep/modules/container-apps-environment.bicep @@ -4,12 +4,8 @@ param containerAppsEnvironmentName string @description('Location for all resources') param location string = resourceGroup().location -@description('Log Analytics workspace id output from log-analytics-ws.bicep module') -param logAnalyticsWorkspaceId string - -@description('Log Analytics workspace primary shared key output from log-analytics-ws.bicep module') -@secure() -param logAnalyticsPrimarySharedKey string +@description('Log Analytics workspace resource ID used to derive customer ID and shared key') +param logAnalyticsWorkspaceResourceId string @description('User Assigned Identity resource IDs that will be assigned to the Container Apps Environment') param userAssignedResourceIds string[] @@ -31,6 +27,10 @@ param publicNetworkAccess string = 'Enabled' @description('Tags for resources') param tags object = {} +// Derive Log Analytics credentials from workspace resource ID +var logAnalyticsCustomerId = reference(logAnalyticsWorkspaceResourceId, '2021-12-01-preview').customerId +var logAnalyticsSharedKey = listKeys(logAnalyticsWorkspaceResourceId, '2021-12-01-preview').primarySharedKey + // Use Azure Verified Module for Container Apps Environment module containerAppsEnvironment 'br:mcr.microsoft.com/bicep/avm/res/app/managed-environment:0.11.3' = { name: '${deployment().name}.containerAppsEnvironment' @@ -42,8 +42,8 @@ module containerAppsEnvironment 'br:mcr.microsoft.com/bicep/avm/res/app/managed- appLogsConfiguration: { destination: 'log-analytics' logAnalyticsConfiguration: { - customerId: logAnalyticsWorkspaceId - sharedKey: logAnalyticsPrimarySharedKey + customerId: logAnalyticsCustomerId + sharedKey: logAnalyticsSharedKey } } workloadProfiles: [ diff --git a/infra/bicep/modules/container-registry.bicep b/infra/bicep/modules/container-registry.bicep index eda929b..0cce25e 100644 --- a/infra/bicep/modules/container-registry.bicep +++ b/infra/bicep/modules/container-registry.bicep @@ -28,6 +28,8 @@ param acrPrivateDnsZoneId string = '' @allowed(['Enabled', 'Disabled']) param publicNetworkAccess string = 'Enabled' + + @description('Zone redundancy setting for the Azure Container Registry') @allowed(['Enabled', 'Disabled']) param zoneRedundancy string = 'Disabled' @@ -80,17 +82,15 @@ module containerRegistry 'br:mcr.microsoft.com/bicep/avm/res/container-registry/ subnetResourceId: privateEndpointSubnetId service: 'registry' privateLinkServiceConnectionName: '${containerRegistryName}-acr-plsc' - privateDnsZoneGroups: [ - { - name: 'acr-dns-zone-group' - privateDnsZoneGroupConfigs: !empty(acrPrivateDnsZoneId) ? [ - { - name: 'acr-config' - privateDnsZoneResourceId: acrPrivateDnsZoneId - } - ] : [] - } - ] + privateDnsZoneGroup: !empty(acrPrivateDnsZoneId) ? { + name: 'acr-dns-zone-group' + privateDnsZoneGroupConfigs: [ + { + name: 'acr-config' + privateDnsZoneResourceId: acrPrivateDnsZoneId + } + ] + } : null } ] : [] } diff --git a/infra/bicep/modules/cosmos.bicep b/infra/bicep/modules/cosmos.bicep index 051b952..a14b453 100644 --- a/infra/bicep/modules/cosmos.bicep +++ b/infra/bicep/modules/cosmos.bicep @@ -63,17 +63,15 @@ module cosmosAccount 'br:mcr.microsoft.com/bicep/avm/res/document-db/database-ac subnetResourceId: privateEndpointSubnetId service: 'Sql' privateLinkServiceConnectionName: '${cosmosAccountName}-cosmos-plsc' - privateDnsZoneGroups: [ - { - name: 'cosmos-dns-zone-group' - privateDnsZoneGroupConfigs: !empty(cosmosPrivateDnsZoneId) ? [ - { - name: 'cosmos-config' - privateDnsZoneResourceId: cosmosPrivateDnsZoneId - } - ] : [] - } - ] + privateDnsZoneGroup: !empty(cosmosPrivateDnsZoneId) ? { + name: 'cosmos-dns-zone-group' + privateDnsZoneGroupConfigs: [ + { + name: 'cosmos-config' + privateDnsZoneResourceId: cosmosPrivateDnsZoneId + } + ] + } : null } ] : [] zoneRedundant: zoneRedundant diff --git a/infra/bicep/modules/openai-role-assignment.bicep b/infra/bicep/modules/openai-role-assignment.bicep new file mode 100644 index 0000000..f223233 --- /dev/null +++ b/infra/bicep/modules/openai-role-assignment.bicep @@ -0,0 +1,31 @@ +// Assigns 'Cognitive Services OpenAI User' role to a managed identity on an existing Azure OpenAI resource. +// This enables the managed identity to perform inference operations (chat completions, embeddings, etc.) +// without requiring API keys — supporting Zero Trust Architecture (ZTA) compliance. + +@description('Resource ID of the existing Azure OpenAI / Cognitive Services resource') +param openAIResourceId string + +@description('Principal IDs of the managed identities to assign the role to') +param principalIds string[] + +// Cognitive Services OpenAI User - allows inference (completions, embeddings) but not management +var roleDefinitionId = '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd' + +// Reference the existing resource to scope the role assignment +resource openAIResource 'Microsoft.CognitiveServices/accounts@2024-10-01' existing = { + name: last(split(openAIResourceId, '/')) + scope: resourceGroup() +} + +// Create role assignments for each principal +resource roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = [ + for principalId in principalIds: { + name: guid(openAIResourceId, roleDefinitionId, principalId) + scope: openAIResource + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleDefinitionId) + principalId: principalId + principalType: 'ServicePrincipal' + } + } +] diff --git a/infra/bicep/modules/queue-dns-zone.bicep b/infra/bicep/modules/queue-dns-zone.bicep new file mode 100644 index 0000000..eea8847 --- /dev/null +++ b/infra/bicep/modules/queue-dns-zone.bicep @@ -0,0 +1,32 @@ +// Queue Private DNS Zone Module +// Creates privatelink.queue.core.windows.net DNS zone and links it to the VNet +// Used when no existing queue DNS zone is provided in AILZ mode + +@description('Resource ID of the VNet to link the DNS zone to') +param vnetResourceId string + +@description('Tags for resources') +param tags object = {} + +// Create the private DNS zone for Azure Storage Queue +resource privateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = { + name: 'privatelink.queue.core.windows.net' + location: 'global' + tags: tags +} + +// Link the DNS zone to the VNet so containers can resolve queue endpoints via private IP +resource vnetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = { + parent: privateDnsZone + name: '${last(split(vnetResourceId, '/'))}-queue-link' + location: 'global' + tags: tags + properties: { + virtualNetwork: { + id: vnetResourceId + } + registrationEnabled: false + } +} + +output dnsZoneId string = privateDnsZone.id diff --git a/infra/bicep/modules/storage.bicep b/infra/bicep/modules/storage.bicep index 2553e2e..55ddbb2 100644 --- a/infra/bicep/modules/storage.bicep +++ b/infra/bicep/modules/storage.bicep @@ -117,17 +117,15 @@ module storageAccount 'br/public:avm/res/storage/storage-account:0.27.1' = { subnetResourceId: privateEndpointSubnetId service: 'blob' privateLinkServiceConnectionName: '${storageAccountName}-blob-plsc' - privateDnsZoneGroups: [ - { - name: 'blob-dns-zone-group' - privateDnsZoneGroupConfigs: [ - { - name: 'blob-config' - privateDnsZoneResourceId: blobPrivateDnsZoneId - } - ] - } - ] + privateDnsZoneGroup: !empty(blobPrivateDnsZoneId) ? { + name: 'blob-dns-zone-group' + privateDnsZoneGroupConfigs: [ + { + name: 'blob-config' + privateDnsZoneResourceId: blobPrivateDnsZoneId + } + ] + } : null } { name: '${storageAccountName}-queue-pe' @@ -135,17 +133,15 @@ module storageAccount 'br/public:avm/res/storage/storage-account:0.27.1' = { subnetResourceId: privateEndpointSubnetId service: 'queue' privateLinkServiceConnectionName: '${storageAccountName}-queue-plsc' - privateDnsZoneGroups: [ - { - name: 'queue-dns-zone-group' - privateDnsZoneGroupConfigs: [ - { - name: 'queue-config' - privateDnsZoneResourceId: queuePrivateDnsZoneId - } - ] - } - ] + privateDnsZoneGroup: !empty(queuePrivateDnsZoneId) ? { + name: 'queue-dns-zone-group' + privateDnsZoneGroupConfigs: [ + { + name: 'queue-config' + privateDnsZoneResourceId: queuePrivateDnsZoneId + } + ] + } : null } ] : [] blobServices: { diff --git a/infra/scripts/ailz-resources.env b/infra/scripts/ailz-resources.env new file mode 100644 index 0000000..38a76f5 --- /dev/null +++ b/infra/scripts/ailz-resources.env @@ -0,0 +1,18 @@ +# AILZ Resource IDs for ContentFlow Deployment +# Generated on Wed, May 27, 2026 2:05:04 PM +# AILZ Resource Group: rg-ai-amer-v-rramini-ailz-contentflow-poc1 + +EXISTING_VNET_RESOURCE_ID="/subscriptions/9ad6f7f4-b0d6-4d88-a6d1-3fc2257d5583/resourceGroups/rg-ai-amer-v-rramini-ailz-contentflow-poc1/providers/Microsoft.Network/virtualNetworks/vnet-7rtqmkghg6mq6" +PRIVATE_ENDPOINT_SUBNET_NAME="pe-subnet" +CONTAINER_APPS_SUBNET_NAME="aca-environment-subnet" +EXISTING_COGNITIVE_SERVICES_PRIVATE_DNS_ZONE_ID="/subscriptions/9ad6f7f4-b0d6-4d88-a6d1-3fc2257d5583/resourceGroups/rg-ai-amer-v-rramini-ailz-contentflow-poc1/providers/Microsoft.Network/privateDnsZones/privatelink.cognitiveservices.azure.com" +EXISTING_BLOB_PRIVATE_DNS_ZONE_ID="/subscriptions/9ad6f7f4-b0d6-4d88-a6d1-3fc2257d5583/resourceGroups/rg-ai-amer-v-rramini-ailz-contentflow-poc1/providers/Microsoft.Network/privateDnsZones/privatelink.blob.core.windows.net" +EXISTING_COSMOS_PRIVATE_DNS_ZONE_ID="/subscriptions/9ad6f7f4-b0d6-4d88-a6d1-3fc2257d5583/resourceGroups/rg-ai-amer-v-rramini-ailz-contentflow-poc1/providers/Microsoft.Network/privateDnsZones/privatelink.documents.azure.com" +EXISTING_APP_CONFIG_PRIVATE_DNS_ZONE_ID="/subscriptions/9ad6f7f4-b0d6-4d88-a6d1-3fc2257d5583/resourceGroups/rg-ai-amer-v-rramini-ailz-contentflow-poc1/providers/Microsoft.Network/privateDnsZones/privatelink.azconfig.io" +EXISTING_ACR_PRIVATE_DNS_ZONE_ID="/subscriptions/9ad6f7f4-b0d6-4d88-a6d1-3fc2257d5583/resourceGroups/rg-ai-amer-v-rramini-ailz-contentflow-poc1/providers/Microsoft.Network/privateDnsZones/privatelink.azurecr.io" +# EXISTING_CONTAINER_APPS_ENV_PRIVATE_DNS_ZONE_ID not found +# JUMPBOX_VM_RESOURCE_ID not found - no JumpBox VM deployed +# JUMPBOX_VM_NAME not found - no JumpBox VM deployed +EXISTING_KEY_VAULT_PRIVATE_DNS_ZONE_ID="/subscriptions/9ad6f7f4-b0d6-4d88-a6d1-3fc2257d5583/resourceGroups/rg-ai-amer-v-rramini-ailz-contentflow-poc1/providers/Microsoft.Network/privateDnsZones/privatelink.vaultcore.azure.net" +EXISTING_LOG_ANALYTICS_WORKSPACE_ID="/subscriptions/9ad6f7f4-b0d6-4d88-a6d1-3fc2257d5583/resourceGroups/rg-ai-amer-v-rramini-ailz-contentflow-poc1/providers/Microsoft.OperationalInsights/workspaces/log-7rtqmkghg6mq6" +# EXISTING_APP_INSIGHTS_ID not found diff --git a/infra/scripts/get-ailz-resources.sh b/infra/scripts/get-ailz-resources.sh index 2ce21eb..89d1f56 100755 --- a/infra/scripts/get-ailz-resources.sh +++ b/infra/scripts/get-ailz-resources.sh @@ -151,7 +151,7 @@ if [ ! -z "$VNET_NAME" ]; then fi echo -n "Searching for Container Apps Environment Subnet..." - ACA_SUBNET=$(az network vnet subnet list --resource-group $AILZ_RG --vnet-name "$VNET_NAME" --query "[?name=='aca-env-subnet'].name" -o tsv) + ACA_SUBNET=$(az network vnet subnet list --resource-group $AILZ_RG --vnet-name "$VNET_NAME" --query "[?name=='aca-env-subnet' || name=='aca-environment-subnet' || contains(name, 'aca')].name" -o tsv | head -n1) if [ ! -z "$ACA_SUBNET" ]; then echo " FOUND ✓" echo "CONTAINER_APPS_SUBNET_NAME=\"$ACA_SUBNET\"" >> $OUTPUT_FILE diff --git a/infra/scripts/post-deploy-worker.sh b/infra/scripts/post-deploy-worker.sh deleted file mode 100755 index a1408d3..0000000 --- a/infra/scripts/post-deploy-worker.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Post-deploy hook for Worker service -set -e - -echo "==================================================" -echo "ContentFlow Worker - Post-Deploy Hook" -echo "==================================================" - -echo "✓ Worker service deployed successfully" - -# Optional: Run any Worker-specific post-deployment tasks here - -echo "==================================================" diff --git a/infra/scripts/post-deploy.sh b/infra/scripts/post-deploy.sh index 0cf3e77..76701d1 100755 --- a/infra/scripts/post-deploy.sh +++ b/infra/scripts/post-deploy.sh @@ -1,5 +1,6 @@ #!/bin/bash # Post-deploy hook - runs after all services are deployed +# Displays endpoints and locks down resources opened during provisioning set -e echo "==================================================" @@ -9,7 +10,6 @@ echo "==================================================" # Get deployment outputs API_ENDPOINT=$(azd env get-value API_ENDPOINT 2>/dev/null || echo "Not available") WEB_ENDPOINT=$(azd env get-value WEB_ENDPOINT 2>/dev/null || echo "Not available") -WORKER_ENDPOINT=$(azd env get-value WORKER_ENDPOINT 2>/dev/null || echo "Not available") echo "" echo "╔════════════════════════════════════════════════╗" @@ -19,7 +19,70 @@ echo "" echo "Service Endpoints:" echo " API: $API_ENDPOINT" echo " Web: $WEB_ENDPOINT" -echo " Worker: $WORKER_ENDPOINT" +echo "" + +# ========== SECURITY HARDENING (AILZ mode only) ========== +DEPLOYMENT_MODE=$(azd env get-value DEPLOYMENT_MODE 2>/dev/null || echo "basic") +RESOURCE_GROUP=$(azd env get-value AZURE_RESOURCE_GROUP 2>/dev/null || echo "") +ACR_NAME=$(azd env get-value AZURE_CONTAINER_REGISTRY_NAME 2>/dev/null || echo "") + +if [ "$DEPLOYMENT_MODE" = "ailz-integrated" ]; then + echo "==================================================" + echo "Security Hardening (AILZ mode)" + echo "==================================================" + + if [ -z "$RESOURCE_GROUP" ]; then + RESOURCE_GROUP=$(az group list --query "[?tags.\"azd-env-name\"=='$(azd env get-value AZURE_ENV_NAME 2>/dev/null)'].name | [0]" -o tsv 2>/dev/null || echo "") + fi + + # --- P-1: Disable App Config public access --- + echo "" + echo "[P-1] Disabling App Config public network access..." + APPCONFIG_NAME=$(az resource list --resource-group "$RESOURCE_GROUP" \ + --resource-type "Microsoft.AppConfiguration/configurationStores" \ + --query "[?tags.application=='contentflow'].name | [0]" -o tsv 2>/dev/null || echo "") + if [ -n "$APPCONFIG_NAME" ]; then + az appconfig update --name "$APPCONFIG_NAME" --resource-group "$RESOURCE_GROUP" \ + --enable-public-network false --only-show-errors 2>/dev/null \ + && echo " ✅ App Config '$APPCONFIG_NAME' — public access disabled" \ + || echo " ⚠️ Could not disable App Config public access" + else + echo " ⚠️ App Config not found — skipping" + fi + + # --- P-2: ACR stays private (no action needed) --- + echo "[P-2] ACR public access — already Disabled at provisioning (ZTA compliant, no action needed)" + + # --- P-3: Verify private DNS A records --- + echo "[P-3] Verifying private DNS zone A records..." + for ZONE in "privatelink.azurecr.io" "privatelink.blob.core.windows.net" "privatelink.documents.azure.com" "privatelink.azconfig.io"; do + COUNT=$(az network private-dns record-set a list --zone-name "$ZONE" \ + --resource-group "$RESOURCE_GROUP" --query "length([])" -o tsv 2>/dev/null || echo "0") + if [ "$COUNT" -gt 0 ] 2>/dev/null; then + echo " ✅ $ZONE — $COUNT A record(s)" + else + echo " ⚠️ $ZONE — no A records (check PE DNS zone groups)" + fi + done + + # --- P-4: Queue DNS zone --- + echo "[P-4] Checking Queue DNS zone..." + az network private-dns zone show --name "privatelink.queue.core.windows.net" \ + --resource-group "$RESOURCE_GROUP" -o none 2>/dev/null \ + && echo " ✅ Queue DNS zone exists" \ + || echo " ℹ️ No queue DNS zone — queue PE uses public resolution" + + echo "" + echo "==================================================" + echo "Hardening complete. Remaining manual actions:" + echo " P-5: Restrict CORS origins to: $WEB_ENDPOINT" + echo " P-6: Add API authentication (OAuth/JWT middleware)" + echo " P-7: Disable App Config local auth" + echo " P-8: Remove storage account Contributor role" + echo " P-9: Add diagnostic settings to all resources" + echo "==================================================" +fi + echo "" echo "Next Steps:" echo " 1. Access the web UI at: $WEB_ENDPOINT" diff --git a/infra/scripts/post-provision.sh b/infra/scripts/post-provision.sh index 96281c0..b70fccc 100755 --- a/infra/scripts/post-provision.sh +++ b/infra/scripts/post-provision.sh @@ -17,14 +17,234 @@ echo "Resource Group: $RESOURCE_GROUP" echo "Storage Account: $STORAGE_ACCOUNT" echo "Cosmos DB Endpoint: $COSMOS_ENDPOINT" -echo "✓ Creating storage queue (if not exists)..." -QUEUE_NAME="contentflow-execution-requests" -az storage queue create \ - --name "$QUEUE_NAME" \ - --account-name "$STORAGE_ACCOUNT" \ - --auth-mode login \ - --only-show-errors || echo "Queue already exists or error creating queue" +# ========== DEPLOYER ROLE ASSIGNMENT ========== +echo "" +echo "==================================================" +echo "Deployer Role - Cognitive Services Contributor & Azure AI Developer" +echo "==================================================" + +AI_SERVICES_NAME=$(azd env get-value AI_SERVICES_NAME 2>/dev/null || echo "") +DEPLOYER_OID=$(az ad signed-in-user show --query id -o tsv 2>/dev/null || echo "") + +if [ -n "$DEPLOYER_OID" ] && [ -n "$AI_SERVICES_NAME" ]; then + AI_SERVICES_SCOPE="/subscriptions/$(az account show --query id -o tsv)/resourceGroups/${RESOURCE_GROUP}/providers/Microsoft.CognitiveServices/accounts/${AI_SERVICES_NAME}" + + echo "Ensuring deployer ($DEPLOYER_OID) has Cognitive Services Contributor on $AI_SERVICES_NAME..." + ROLE_OUT=$(az role assignment create \ + --assignee-object-id "$DEPLOYER_OID" \ + --assignee-principal-type User \ + --role "Cognitive Services Contributor" \ + --scope "$AI_SERVICES_SCOPE" \ + --only-show-errors 2>&1) && \ + echo " ✅ Deployer has Cognitive Services Contributor" || \ + { echo "$ROLE_OUT" | grep -qi "RoleAssignmentExists" && \ + echo " ✅ Deployer already has Cognitive Services Contributor" || \ + echo " ⚠️ Could not assign Cognitive Services Contributor (may need Owner/UAA permissions on the scope)"; } + + # Azure AI Developer is the DATA-PLANE write role required for: + # - Setting CU defaults (PATCH /contentunderstanding/defaults) + # - Creating/updating analyzers (PUT /contentunderstanding/analyzers/{id}) + # - Accessing CU Studio (superset of Azure AI User — covers read access too) + # Azure AI User (read-only) is NOT sufficient; it causes 401 on any write operation. + echo "Ensuring deployer ($DEPLOYER_OID) has Azure AI Developer on $AI_SERVICES_NAME..." + ROLE_OUT=$(az role assignment create \ + --assignee-object-id "$DEPLOYER_OID" \ + --assignee-principal-type User \ + --role "Azure AI Developer" \ + --scope "$AI_SERVICES_SCOPE" \ + --only-show-errors 2>&1) && \ + echo " ✅ Deployer has Azure AI Developer (data-plane read+write — required for CU setup & Studio)" || \ + { echo "$ROLE_OUT" | grep -qi "RoleAssignmentExists" && \ + echo " ✅ Deployer already has Azure AI Developer" || \ + echo " ⚠️ Could not assign Azure AI Developer (may need Owner/UAA permissions on the scope)"; } +else + echo "⚠️ Could not determine deployer identity or AI Services name. Skipping role assignment." +fi + +# ========== CONTENT UNDERSTANDING SETUP ========== +echo "" +echo "==================================================" +echo "Content Understanding - Model Defaults & Analyzers" +echo "==================================================" + +# AI_SERVICES_NAME already resolved above + +if [ -z "$AI_SERVICES_NAME" ]; then + echo "⚠️ AI_SERVICES_NAME not found in azd env. Skipping CU setup." + echo " Set AI_SERVICES_NAME manually and re-run if needed." +else + # Construct CU endpoint from AI Services account name + CU_ENDPOINT="https://${AI_SERVICES_NAME}.services.ai.azure.com" + CU_API_VERSION="2025-11-01" + echo "CU Endpoint: $CU_ENDPOINT" + + # --- Helper: acquire a bearer token for Cognitive Services scope --- + acquire_token() { + az account get-access-token \ + --resource "https://cognitiveservices.azure.com" \ + --query accessToken -o tsv 2>/dev/null || echo "" + } + + # --- Combined RBAC + CU Defaults Probe --- + # The RBAC probe must test a WRITE operation, not just a read. + # On a brand-new AI Services resource, data-plane WRITE permission + # propagation takes 15-30 minutes — significantly longer than reads + # (~6 min). A read-only probe gives a false-positive, causing writes + # to fail with HTTP 401 while propagation is still in progress. + # + # Solution: use the actual PATCH defaults call as the probe. + # This retries every 30s for up to 30 minutes (60 attempts), + # ensuring we wait for write-level RBAC propagation. + echo "" + echo "[CU-1] Setting default model deployments (with RBAC write-readiness probe, up to 30 min)..." + DEFAULTS_MAX_ATTEMPTS=60 + DEFAULTS_WAIT_SECONDS=30 + DEFAULTS_OK=false + ACCESS_TOKEN="" + + for DEFAULTS_ATTEMPT in $(seq 1 $DEFAULTS_MAX_ATTEMPTS); do + ACCESS_TOKEN=$(acquire_token) + if [ -z "$ACCESS_TOKEN" ]; then + echo " [attempt $DEFAULTS_ATTEMPT/$DEFAULTS_MAX_ATTEMPTS] Failed to acquire token — retrying in ${DEFAULTS_WAIT_SECONDS}s..." + sleep "$DEFAULTS_WAIT_SECONDS" + continue + fi + + CU_DEFAULTS_RESPONSE=$(curl -s -w "\n%{http_code}" -X PATCH \ + "${CU_ENDPOINT}/contentunderstanding/defaults?api-version=${CU_API_VERSION}" \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: application/merge-patch+json" \ + -d '{ + "modelDeployments": { + "gpt-4.1": "gpt-4.1", + "gpt-4.1-mini": "gpt-4.1-mini", + "text-embedding-3-large": "text-embedding-3-large" + } + }' 2>/dev/null) + + CU_DEFAULTS_HTTP_CODE=$(echo "$CU_DEFAULTS_RESPONSE" | tail -1) + CU_DEFAULTS_BODY=$(echo "$CU_DEFAULTS_RESPONSE" | sed '$d') + + if [ "$CU_DEFAULTS_HTTP_CODE" -ge 200 ] 2>/dev/null && [ "$CU_DEFAULTS_HTTP_CODE" -lt 300 ] 2>/dev/null; then + echo " ✅ Default model deployments configured (HTTP $CU_DEFAULTS_HTTP_CODE, attempt $DEFAULTS_ATTEMPT)" + DEFAULTS_OK=true + break + fi + + # Retry on auth/propagation and transient errors + case "$CU_DEFAULTS_HTTP_CODE" in + 401|403|429|500|502|503|504) + echo " [attempt $DEFAULTS_ATTEMPT/$DEFAULTS_MAX_ATTEMPTS] CU defaults PATCH failed (HTTP $CU_DEFAULTS_HTTP_CODE) — retrying in ${DEFAULTS_WAIT_SECONDS}s..." + sleep "$DEFAULTS_WAIT_SECONDS" + ;; + *) + echo " ❌ CU defaults failed with non-retryable error (HTTP $CU_DEFAULTS_HTTP_CODE)" + echo " Response: $CU_DEFAULTS_BODY" + break + ;; + esac + done + + if [ "$DEFAULTS_OK" != "true" ]; then + echo "❌ Failed to set CU defaults after $DEFAULTS_MAX_ATTEMPTS attempts ($((DEFAULTS_MAX_ATTEMPTS * DEFAULTS_WAIT_SECONDS))s)." + echo " RBAC write-level propagation may not have completed." + echo " Re-run 'azd hooks run postprovision' after a few minutes." + exit 1 + else + + # --- Step 2: Create/update custom analyzers from seed folder --- + echo "" + echo "[CU-2] Creating custom analyzers from seed definitions..." + + SEED_DIR="$(cd "$(dirname "$0")/../.." && pwd)/contentflow-api/seed/analyzers" + + if [ ! -d "$SEED_DIR" ]; then + echo " ℹ️ No seed/analyzers folder found at: $SEED_DIR" + echo " Skipping custom analyzer creation." + else + ANALYZER_COUNT=0 + ANALYZER_SUCCESS=0 + ANALYZER_FAILED=0 + + for ANALYZER_FILE in "$SEED_DIR"/*.json; do + # Skip if no JSON files exist (glob returns literal pattern) + [ -e "$ANALYZER_FILE" ] || continue + + ANALYZER_COUNT=$((ANALYZER_COUNT + 1)) + # Derive analyzer ID from filename (e.g., details_extractor_documents_new1.json -> details_extractor_documents_new1) + ANALYZER_ID=$(basename "$ANALYZER_FILE" .json) + + echo " Creating analyzer: $ANALYZER_ID ..." + + # PUT is create-or-replace (idempotent) — retry with exponential backoff + PUT_MAX_RETRIES=5 + PUT_BACKOFF=15 + PUT_OK=false + + for PUT_ATTEMPT in $(seq 1 $PUT_MAX_RETRIES); do + # Refresh token before each retry (handles expiry edge case) + if [ "$PUT_ATTEMPT" -gt 1 ]; then + ACCESS_TOKEN=$(acquire_token) + fi + + ANALYZER_RESPONSE=$(curl -s -w "\n%{http_code}" -X PUT \ + "${CU_ENDPOINT}/contentunderstanding/analyzers/${ANALYZER_ID}?api-version=${CU_API_VERSION}" \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: application/json" \ + -d @"$ANALYZER_FILE" 2>/dev/null) + + ANALYZER_HTTP_CODE=$(echo "$ANALYZER_RESPONSE" | tail -1) + ANALYZER_BODY=$(echo "$ANALYZER_RESPONSE" | sed '$d') + + if [ "$ANALYZER_HTTP_CODE" -ge 200 ] 2>/dev/null && [ "$ANALYZER_HTTP_CODE" -lt 300 ] 2>/dev/null; then + echo " ✅ $ANALYZER_ID — created/updated (HTTP $ANALYZER_HTTP_CODE, attempt $PUT_ATTEMPT)" + PUT_OK=true + break + fi + + # Retry on transient errors (401, 403, 429, 5xx) + case "$ANALYZER_HTTP_CODE" in + 401|403|429|500|502|503|504) + echo " [attempt $PUT_ATTEMPT/$PUT_MAX_RETRIES] $ANALYZER_ID failed (HTTP $ANALYZER_HTTP_CODE) — retrying in ${PUT_BACKOFF}s..." + sleep "$PUT_BACKOFF" + PUT_BACKOFF=$((PUT_BACKOFF * 2)) + [ "$PUT_BACKOFF" -gt 120 ] && PUT_BACKOFF=120 + ;; + *) + echo " ❌ $ANALYZER_ID — non-retryable error (HTTP $ANALYZER_HTTP_CODE)" + echo " Response: $ANALYZER_BODY" + break + ;; + esac + done + + if [ "$PUT_OK" = "true" ]; then + ANALYZER_SUCCESS=$((ANALYZER_SUCCESS + 1)) + else + echo " ❌ $ANALYZER_ID — failed after $PUT_MAX_RETRIES attempts" + ANALYZER_FAILED=$((ANALYZER_FAILED + 1)) + fi + done + + if [ "$ANALYZER_COUNT" -eq 0 ]; then + echo " ℹ️ No analyzer JSON files found in seed/analyzers/" + else + echo "" + echo " Analyzers processed: $ANALYZER_COUNT (✅ $ANALYZER_SUCCESS succeeded, ❌ $ANALYZER_FAILED failed)" + + # Fail deployment if ALL analyzers failed — partial success is allowed + if [ "$ANALYZER_FAILED" -gt 0 ] && [ "$ANALYZER_SUCCESS" -eq 0 ]; then + echo "" + echo "❌ All analyzer creations failed. Deployment cannot proceed." + echo " Re-run 'azd up' or 'azd hooks run postprovision' after verifying roles." + exit 1 + fi + fi + fi + fi +fi +echo "" echo "==================================================" echo "✓ Post-provision completed successfully" echo "==================================================" diff --git a/infra/scripts/pre-provision.sh b/infra/scripts/pre-provision.sh index b1309ba..f668f02 100755 --- a/infra/scripts/pre-provision.sh +++ b/infra/scripts/pre-provision.sh @@ -30,6 +30,12 @@ if ! az account show &> /dev/null; then exit 1 fi +# Note: ACR placeholder image import is NOT done here. +# In AILZ mode, ACR is fully private (publicNetworkAccess=Disabled). +# Use 'az acr build' (ACR Tasks) during 'azd deploy' to build and push images +# inside the ACR without requiring public access. +echo "✓ ACR image build will be handled by ACR Tasks during deployment phase." + echo "==================================================" echo "✓ Pre-provision checks completed successfully" echo "==================================================" diff --git a/rules.json b/rules.json new file mode 100644 index 0000000..75d9fdb --- /dev/null +++ b/rules.json @@ -0,0 +1,205 @@ +{ + "rules": [ + { + "documentType": "Driving License", + "validations": [ + { + "field": "ExpirationDate", + "validationType": "expiry_check", + "errorCode": "DL_EXPIRED", + "message_en": "Your driving license has expired. Please upload a valid driving license.", + "message_es": "Su licencia de conducir ha expirado. Por favor, cargue una licencia de conducir válida." + }, + { + "field": "FirstName", + "validationType": "exact_match", + "errorCode": "DL_FIRST_NAME_MISMATCH", + "message_en": "The first name on the driving license does not match the first name registered in your case.", + "message_es": "El nombre en la licencia de conducir no coincide con el nombre registrado en su caso." + }, + { + "field": "LastName", + "validationType": "exact_match", + "errorCode": "DL_LAST_NAME_MISMATCH", + "message_en": "The last name on the driving license does not match the fist last and second last name registered in your case.", + "message_es": "El apellido en la licencia de conducir no coincide con el primer y segundo apellido registrados en su caso." + }, + { + "field": "Address", + "validationType": "address_match", + "errorCode": "DL_ADDRESS_MISMATCH", + "message_en": "The mailing address on the driving license does not match the postal address registered in your case.", + "message_es": "La dirección postal en la licencia de conducir no coincide con la dirección postal registrada en su caso." + }, + { + "field": "BirthDate", + "validationType": "date_match", + "errorCode": "DL_DOB_MISMATCH", + "message_en": "The day of birth on the driving license does not match the birth date registered in your case.", + "message_es": "La fecha de nacimiento en la licencia de conducir no coincide con la fecha de nacimiento registrada en su caso." + } + ] + }, + { + "documentType": "Passport", + "validations": [ + { + "field": "ExpirationDate", + "validationType": "expiry_check", + "errorCode": "PP_EXPIRED", + "message_en": "Your passport has expired. Please upload a passport.", + "message_es": "Su pasaporte ha expirado. Por favor, cargue un pasaporte válido." + }, + { + "field": "LastName", + "validationType": "exact_match", + "errorCode": "PP_LAST_NAME_MISMATCH", + "message_en": "The last name on the passport does not match the fist last and second last name registered in your case.", + "message_es": "El apellido en el pasaporte no coincide con el primer y segundo apellido registrados en su caso." + }, + { + "field": "FirstName", + "validationType": "exact_match", + "errorCode": "PP_FIRST_NAME_MISMATCH", + "message_en": "The first name on the passport does not match the first name registered in your case.", + "message_es": "El nombre en el pasaporte no coincide con el nombre registrado en su caso." + }, + { + "field": "BirthDate", + "validationType": "date_match", + "errorCode": "PP_DOB_MISMATCH", + "message_en": "The day of birth on the passport does not match the birth date registered in your case.", + "message_es": "La fecha de nacimiento en el pasaporte no coincide con la fecha de nacimiento registrada en su caso." + } + ] + }, + { + "documentType": "Real ID", + "validations": [ + { + "field": "ExpirationDate", + "validationType": "expiry_check", + "errorCode": "RD_EXPIRED", + "message_en": "Your real ID has expired. Please upload a valid real ID.", + "message_es": "Su Real ID ha expirado. Por favor, cargue una Real ID válida." + }, + { + "field": "FirstName", + "validationType": "exact_match", + "errorCode": "RD_FIRST_NAME_MISMATCH", + "message_en": "The first name on the real ID does not match the first name registered in your case.", + "message_es": "El nombre en la Real ID no coincide con el nombre registrado en su caso." + }, + { + "field": "LastName", + "validationType": "exact_match", + "errorCode": "RD_LAST_NAME_MISMATCH", + "message_en": "The last name on the real ID does not match the first last and second last name registered in your case.", + "message_es": "El apellido en la Real ID no coincide con el primer y segundo apellido registrados en su caso." + }, + { + "field": "Address", + "validationType": "address_match", + "errorCode": "RD_ADDRESS_MISMATCH", + "message_en": "The mailing address on the real ID does not match the postal address registered in your case.", + "message_es": "La dirección postal en la Real ID no coincide con la dirección postal registrada en su caso." + } + ] + }, + { + "documentType": "Water utility bill", + "validations": [ + { + "field": "FullName", + "validationType": "name_match", + "errorCode": "WB_FIRST_LAST_NAME_MISMATCH", + "message_en": "The first and last name on the water utility bill does not match the first name, fist last and second last name registered in your case.", + "message_es": "El nombre y apellido en la factura de agua no coinciden con el nombre, primer apellido y segundo apellido registrados en su caso." + }, + { + "field": "Address", + "validationType": "address_match", + "errorCode": "WB_ADDRESS_MISMATCH", + "message_en": "The mailing address on the water utility bill does not match the postal address registered in your case.", + "message_es": "La dirección postal en la factura de agua no coincide con la dirección postal registrada en su caso." + } + ] + }, + { + "documentType": "Power utility bill", + "validations": [ + { + "field": "FullName", + "validationType": "name_match", + "errorCode": "PB_FIRST_LAST_NAME_MISMATCH", + "message_en": "The first and last name on the power utility bill does not match the first name, fist last and second last name registered in your case.", + "message_es": "El nombre y apellido en la factura de electricidad no coinciden con el nombre, primer apellido y segundo apellido registrados en su caso." + }, + { + "field": "Address", + "validationType": "address_match", + "errorCode": "PB_ADDRESS_MISMATCH", + "message_en": "The mailing address on the power utility bill does not match the postal address registered in your case.", + "message_es": "La dirección postal en la factura de electricidad no coincide con la dirección postal registrada en su caso." + } + ] + }, + { + "documentType": "Telecom utility bill_claro", + "validations": [ + { + "field": "FullName", + "validationType": "name_match", + "errorCode": "TC_FIRST_LAST_NAME_MISMATCH", + "message_en": "The first and last name on the telecom utility bill does not match the first name, fist last and second last name registered in your case.", + "message_es": "El nombre y apellido en la factura de telecomunicaciones no coinciden con el nombre, primer apellido y segundo apellido registrados en su caso." + }, + { + "field": "Address", + "validationType": "address_match", + "errorCode": "TC_ADDRESS_MISMATCH", + "message_en": "The mailing address on the telecom utility bill does not match the postal address registered in your case.", + "message_es": "La dirección postal en la factura de telecomunicaciones no coincide con la dirección postal registrada en su caso." + } + ] + }, + { + "documentType": "Telecom utility bill_liberty", + "validations": [ + { + "field": "FullName", + "validationType": "name_match", + "errorCode": "TL_FIRST_LAST_NAME_MISMATCH", + "message_en": "The first and last name on the telecom utility bill does not match the first name, fist last and second last name registered in your case.", + "message_es": "El nombre y apellido en la factura de telecomunicaciones no coinciden con el nombre, primer apellido y segundo apellido registrados en su caso." + }, + { + "field": "Address", + "validationType": "address_match", + "errorCode": "TL_ADDRESS_MISMATCH", + "message_en": "The mailing address on the telecom utility bill does not match the postal address registered in your case.", + "message_es": "La dirección postal en la factura de telecomunicaciones no coincide con la dirección postal registrada en su caso." + } + ] + }, + { + "documentType": "Telecom utility bill_tmobile", + "validations": [ + { + "field": "FullName", + "validationType": "name_match", + "errorCode": "TT_FIRST_LAST_NAME_MISMATCH", + "message_en": "The first and last name on the telecom utility bill does not match the firstname, fist last and second last name registered in your case.", + "message_es": "El nombre y apellido en la factura de telecomunicaciones no coinciden con el nombre, primer apellido y segundo apellido registrados en su caso." + }, + { + "field": "Address", + "validationType": "address_match", + "errorCode": "TT_ADDRESS_MISMATCH", + "message_en": "The mailing address on the telecom utility bill does not match the postal address registered in your case.", + "message_es": "La dirección postal en la factura de telecomunicaciones no coincide con la dirección postal registrada en su caso." + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/integration/test_ingest_api.py b/tests/integration/test_ingest_api.py new file mode 100644 index 0000000..45eb4bf --- /dev/null +++ b/tests/integration/test_ingest_api.py @@ -0,0 +1,336 @@ +""" +Test script for the POST /api/ingest multipart document submission endpoint. + +Usage: + 1. Place your test documents in a folder (default: ./input) + 2. Set BASE_URL to your API host (local or Azure) + 3. Set PIPELINE_ID to a valid pipeline ID from your Cosmos DB + 4. Run: + python test_ingest_api.py + +What this script does: + - Step 1: Sends a multipart POST to /api/ingest with case details (form fields) + and all document files found in the INPUT_DIR folder. + - Step 2: Prints the 202 Accepted response (execution_id, blob_prefix, etc.). + - Step 3: Polls GET /api/pipelines/executions/{execution_id} every few seconds + until the pipeline completes, fails, or the poll limit is reached. + - Step 4: Prints a final summary with the execution status. + +Where to check responses: + - CONSOLE: All responses are printed to stdout. + - SWAGGER: Open {BASE_URL}/docs in a browser to test interactively. + - COSMOS DB: Query the "executions" container for the returned execution_id. + - BLOB: Check Azure Blob Storage container → input/{caseId}_{executionId}/ + • ProvidedDetails.json — generated from form fields + • your uploaded documents — as submitted + • results.json — written by the validation executor +""" + +import os +import sys +import time +import json +import requests + +# ────────────────────────────────────────────── +# CONFIGURATION — edit these before running +# ────────────────────────────────────────────── + +# API base URL (no trailing slash) +# Local: http://localhost:8090 +# Azure: https://.azurecontainerapps.io +BASE_URL = os.environ.get("CONTENTFLOW_API_URL", "https://api-doa37xivbuto4.wittywave-d4ca8c39.eastus.azurecontainerapps.io") + +# A valid pipeline ID from Cosmos DB that is enabled and linked to the +# blob input discovery → content understanding → document validation flow. +# NOTE: pipeline_id is no longer sent by the client — the API resolves it +# internally by name. This variable is kept only for display/reference. +PIPELINE_ID = os.environ.get("CONTENTFLOW_PIPELINE_ID", "resolved-server-side") + +# Folder containing the test documents to upload (PDFs, images, etc.) +INPUT_DIR = os.environ.get("CONTENTFLOW_INPUT_DIR", os.path.join(os.path.dirname(__file__), "input")) + +# Polling settings +POLL_INTERVAL_SECONDS = 5 +POLL_MAX_ATTEMPTS = 60 # 5 minutes max + +# ────────────────────────────────────────────── +# SAMPLE CASE DATA — edit to match your rules +# ────────────────────────────────────────────── +CASE_DATA = { + "caseId": "20261117254", + "firstName": "ESTERBINA", + "lastName": "SIERRA RIOS", + "mailingAddress": "BO CIALITOS CRUCE CARR 608 KM 60 141 CIALES. PR 00638", + "dateOfBirth": "1983-03-16", +} + + +def collect_files(input_dir: str) -> list[str]: + """Collect all supported files from the input directory.""" + allowed = {".pdf", ".docx", ".pptx", ".xlsx", ".png", ".jpg", ".jpeg", ".tiff"} + if not os.path.isdir(input_dir): + print(f"ERROR: Input directory not found: {input_dir}") + print(f" Create this folder and place your test documents inside it.") + sys.exit(1) + + files = [] + for fname in sorted(os.listdir(input_dir)): + ext = os.path.splitext(fname)[1].lower() + if ext in allowed: + files.append(os.path.join(input_dir, fname)) + + if not files: + print(f"ERROR: No supported document files found in: {input_dir}") + print(f" Supported extensions: {allowed}") + sys.exit(1) + + return files + + +def submit_ingest(base_url: str, case_data: dict, file_paths: list[str]) -> dict: + """ + POST /api/ingest — multipart form data with case fields + files. + Pipeline is resolved server-side by name (not sent by client). + Returns the parsed JSON response (IngestResponse). + """ + url = f"{base_url}/api/ingest/" + + # Build form fields (sent as multipart form data, not JSON) + # NOTE: pipeline_id is NOT sent — the API resolves the pipeline internally by name + form_data = {**case_data} + + # Build file tuples: ("files", (filename, file_handle, content_type)) + file_handles = [] + files_payload = [] + for path in file_paths: + fname = os.path.basename(path) + fh = open(path, "rb") + file_handles.append(fh) + files_payload.append(("files", (fname, fh, "application/octet-stream"))) + + print("=" * 60) + print("STEP 1 — Submitting ingest request") + print("=" * 60) + print(f" URL: {url}") + print(f" Pipeline: (resolved server-side by name)") + print(f" Case ID: {case_data['caseId']}") + print(f" Files ({len(file_paths)}):") + for p in file_paths: + size_kb = os.path.getsize(p) / 1024 + print(f" • {os.path.basename(p)} ({size_kb:.1f} KB)") + print() + + try: + resp = requests.post(url, data=form_data, files=files_payload, timeout=120) + finally: + for fh in file_handles: + fh.close() + + print(f" HTTP Status: {resp.status_code}") + + if resp.status_code == 202: + result = resp.json() + print(f" Response:") + print(json.dumps(result, indent=4)) + return result + else: + print(f" ERROR RESPONSE:") + try: + print(json.dumps(resp.json(), indent=4)) + except Exception: + print(resp.text) + sys.exit(1) + + +def poll_execution(base_url: str, execution_id: str) -> dict: + """ + GET /api/pipelines/executions/{execution_id} — poll until terminal state. + Returns the final execution record. + """ + url = f"{base_url}/api/pipelines/executions/{execution_id}" + + print() + print("=" * 60) + print("STEP 2 — Polling execution status") + print("=" * 60) + print(f" URL: {url}") + print(f" Execution ID: {execution_id}") + print(f" Interval: {POLL_INTERVAL_SECONDS}s (max {POLL_MAX_ATTEMPTS} attempts)") + print() + + terminal_states = {"completed", "failed", "cancelled"} + + for attempt in range(1, POLL_MAX_ATTEMPTS + 1): + try: + resp = requests.get(url, timeout=30) + except requests.RequestException as e: + print(f" [{attempt:3d}] Connection error: {e}") + time.sleep(POLL_INTERVAL_SECONDS) + continue + + if resp.status_code == 200: + data = resp.json() + status = data.get("status", "unknown") + print(f" [{attempt:3d}] Status: {status}") + + if status in terminal_states: + print() + print(" Final execution record:") + print(json.dumps(data, indent=4, default=str)) + return data + elif resp.status_code == 404: + print(f" [{attempt:3d}] Execution not found (may still be creating)") + else: + print(f" [{attempt:3d}] HTTP {resp.status_code}: {resp.text[:200]}") + + time.sleep(POLL_INTERVAL_SECONDS) + + print() + print(" WARNING: Max poll attempts reached. The pipeline may still be running.") + print(f" Check manually: {url}") + return {} + + +def print_summary(ingest_response: dict, execution_result: dict): + """Print a final summary of the test run.""" + print() + print("=" * 60) + print("SUMMARY") + print("=" * 60) + print(f" Case ID: {ingest_response.get('case_id')}") + print(f" Execution ID: {ingest_response.get('execution_id')}") + print(f" Blob Prefix: {ingest_response.get('blob_prefix')}") + print(f" Files Uploaded: {ingest_response.get('files_uploaded')}") + print(f" Pipeline: {ingest_response.get('pipeline_name')} ({ingest_response.get('pipeline_id')})") + + final_status = execution_result.get("status", "unknown") + print(f" Final Status: {final_status}") + + if final_status == "completed": + print() + print(" SUCCESS — Pipeline completed. Check these locations:") + elif final_status == "failed": + error = execution_result.get("error", "") + print(f" Error: {error}") + print() + print(" FAILED — Check these locations for debugging:") + else: + print() + print(" Check these locations:") + + blob_prefix = ingest_response.get("blob_prefix", "input/_/") + print(f" Blob Storage: container → {blob_prefix}") + print(f" • ProvidedDetails.json (generated from form fields)") + print(f" • results.json (validation output)") + print(f" Cosmos DB: executions container → id = {ingest_response.get('execution_id')}") + print(f" Swagger UI: {BASE_URL}/docs") + print(f" Polling URL: {BASE_URL}/api/pipelines/executions/{ingest_response.get('execution_id')}") + print(f" Results URL: {BASE_URL}/api/ingest/{ingest_response.get('execution_id')}/results") + print() + + +def fetch_results(base_url: str, execution_id: str, save_dir: str = None) -> dict: + """ + GET /api/ingest/{execution_id}/results — fetch validation results. + Optionally saves the results JSON to save_dir for local verification. + Returns the parsed JSON response or empty dict on failure. + """ + url = f"{base_url}/api/ingest/{execution_id}/results" + + print() + print("=" * 60) + print("STEP 3 — Fetching validation results") + print("=" * 60) + print(f" URL: {url}") + print() + + try: + resp = requests.get(url, timeout=30) + except requests.RequestException as e: + print(f" Connection error: {e}") + return {} + + print(f" HTTP Status: {resp.status_code}") + + if resp.status_code == 200: + data = resp.json() + results = data.get("results", {}) + summary = results.get("summary", {}) + print(f" Overall status: {summary.get('overallStatus', 'N/A')}") + print(f" Total documents: {summary.get('totalDocuments', 'N/A')}") + print(f" Passed: {summary.get('passed', 'N/A')}") + print(f" Failed: {summary.get('failed', 'N/A')}") + print() + print(" Full results JSON:") + print(json.dumps(data, indent=4, default=str)) + + # Save to local file for verification + if save_dir: + os.makedirs(save_dir, exist_ok=True) + out_path = os.path.join(save_dir, f"results_{execution_id}.json") + with open(out_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=4, default=str) + print() + print(f" Saved results to: {out_path}") + + return data + elif resp.status_code == 202: + print(" Pipeline still running. Results not available yet.") + return {} + elif resp.status_code == 404: + print(" Execution or results not found.") + try: + print(f" Detail: {resp.json().get('detail', '')}") + except Exception: + pass + return {} + elif resp.status_code == 422: + print(" Execution did not complete successfully.") + try: + detail = resp.json().get("detail", {}) + print(f" Status: {detail.get('status', 'unknown')}") + print(f" Error: {detail.get('error', 'N/A')}") + except Exception: + print(f" Response: {resp.text[:300]}") + return {} + else: + print(f" Unexpected response: {resp.text[:300]}") + return {} + + +if __name__ == "__main__": + print() + print("ContentFlow Ingest API — Integration Test") + print("=" * 60) + + # ── GET-only mode: pass an execution_id as argument ── + # python test_ingest_api.py exec_abc123def456 + if len(sys.argv) > 1: + execution_id = sys.argv[1].strip() + print(f" Mode: GET results only") + print(f" Execution ID: {execution_id}") + fetch_results(BASE_URL, execution_id, save_dir=INPUT_DIR) + sys.exit(0) + + # ── Full mode: POST + poll + GET results ── + # Collect document files from input folder + file_paths = collect_files(INPUT_DIR) + + # Step 1: Submit ingest + ingest_response = submit_ingest(BASE_URL, CASE_DATA, file_paths) + + # Step 2: Poll until complete + execution_id = ingest_response["execution_id"] + execution_result = poll_execution(BASE_URL, execution_id) + + # Step 3: Fetch results + final_status = execution_result.get("status", "unknown") + if final_status == "completed": + results = fetch_results(BASE_URL, execution_id, save_dir=INPUT_DIR) + else: + print() + print(f" Skipping results fetch — execution status is '{final_status}'") + + # Step 4: Summary + print_summary(ingest_response, execution_result)