diff --git a/application/single_app/config.py b/application/single_app/config.py index 11f93c24..6250de22 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -96,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.047" +VERSION = "0.250.051" SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') SESSION_COOKIE_HTTPONLY = os.getenv('SESSION_COOKIE_HTTPONLY', 'true').lower() != 'false' diff --git a/application/single_app/functions_data_management.py b/application/single_app/functions_data_management.py index 4938f2c2..376b2f5d 100644 --- a/application/single_app/functions_data_management.py +++ b/application/single_app/functions_data_management.py @@ -2,10 +2,13 @@ """Data Management settings, schedules, and durable job records.""" import copy +import hashlib import json import logging import os +import re import socket +import time import tempfile import uuid from datetime import datetime, timedelta, timezone @@ -130,6 +133,10 @@ class DataManagementSettingsValidationError(ValueError): """Raised when Data Management settings fail admin-safe validation.""" + +class DataManagementCosmosEditorError(ValueError): + """Raised when Cosmos editor input is unsafe or incomplete.""" + DATA_MANAGEMENT_FRONTEND_SECRET_FIELDS = { "backup_storage_connection_string", "encryption_key_reference", @@ -194,6 +201,77 @@ class DataManagementSettingsValidationError(ValueError): {"name": "global_workspace_identities", "container_attr": "cosmos_global_workspace_identities_container", "container_name_attr": "cosmos_global_workspace_identities_container_name", "partition_key_path": "/id", "category": "identities"}, ] +DATA_MANAGEMENT_COSMOS_EDITOR_EMPTY_QUERY = "SELECT * FROM c" +DATA_MANAGEMENT_COSMOS_EDITOR_EMPTY_QUERY_LIMIT = 100 +DATA_MANAGEMENT_COSMOS_EDITOR_MAX_PAGE_SIZE = 100 +DATA_MANAGEMENT_COSMOS_EDITOR_MAX_QUERY_LENGTH = 4000 +DATA_MANAGEMENT_COSMOS_EDITOR_CONFIRMATION_PHRASE = "I understand this can damage system data" +DATA_MANAGEMENT_COSMOS_EDITOR_CONTAINER_DEFINITIONS = [ + ("conversations", "cosmos_conversations_container", "cosmos_conversations_container_name", "/id", "conversations"), + ("messages", "cosmos_messages_container", "cosmos_messages_container_name", "/conversation_id", "conversations"), + ("tabular_export_runs", "cosmos_tabular_export_runs_container", "cosmos_tabular_export_runs_container_name", "/user_id", "exports"), + ("data_management_jobs", "cosmos_data_management_jobs_container", "cosmos_data_management_jobs_container_name", "/id", "data_management"), + ("data_management_job_items", "cosmos_data_management_job_items_container", "cosmos_data_management_job_items_container_name", "/job_id", "data_management"), + ("personal_workflows", "cosmos_personal_workflows_container", "cosmos_personal_workflows_container_name", "/user_id", "workflows"), + ("personal_workflow_runs", "cosmos_personal_workflow_runs_container", "cosmos_personal_workflow_runs_container_name", "/user_id", "workflows"), + ("personal_workflow_run_items", "cosmos_personal_workflow_run_items_container", "cosmos_personal_workflow_run_items_container_name", "/run_id", "workflows"), + ("group_workflows", "cosmos_group_workflows_container", "cosmos_group_workflows_container_name", "/group_id", "workflows"), + ("group_workflow_runs", "cosmos_group_workflow_runs_container", "cosmos_group_workflow_runs_container_name", "/group_id", "workflows"), + ("group_workflow_run_items", "cosmos_group_workflow_run_items_container", "cosmos_group_workflow_run_items_container_name", "/run_id", "workflows"), + ("group_conversations", "cosmos_group_conversations_container", "cosmos_group_conversations_container_name", "/id", "conversations"), + ("group_messages", "cosmos_group_messages_container", "cosmos_group_messages_container_name", "/conversation_id", "conversations"), + ("collaboration_conversations", "cosmos_collaboration_conversations_container", "cosmos_collaboration_conversations_container_name", "/id", "collaboration"), + ("collaboration_messages", "cosmos_collaboration_messages_container", "cosmos_collaboration_messages_container_name", "/conversation_id", "collaboration"), + ("collaboration_user_state", "cosmos_collaboration_user_state_container", "cosmos_collaboration_user_state_container_name", "/user_id", "collaboration"), + ("settings", "cosmos_settings_container", "cosmos_settings_container_name", "/id", "settings"), + ("custom_pages", "cosmos_custom_pages_container", "cosmos_custom_pages_container_name", "/id", "settings"), + ("groups", "cosmos_groups_container", "cosmos_groups_container_name", "/id", "workspaces"), + ("public_workspaces", "cosmos_public_workspaces_container", "cosmos_public_workspaces_container_name", "/id", "workspaces"), + ("documents", "cosmos_user_documents_container", "cosmos_user_documents_container_name", "/id", "documents"), + ("group_documents", "cosmos_group_documents_container", "cosmos_group_documents_container_name", "/id", "documents"), + ("public_documents", "cosmos_public_documents_container", "cosmos_public_documents_container_name", "/id", "documents"), + ("document_access_index", "cosmos_document_access_index_container", "cosmos_document_access_index_container_name", "/scope_key", "documents"), + ("personal_file_sync_sources", "cosmos_personal_file_sync_sources_container", "cosmos_personal_file_sync_sources_container_name", "/user_id", "file_sync"), + ("group_file_sync_sources", "cosmos_group_file_sync_sources_container", "cosmos_group_file_sync_sources_container_name", "/group_id", "file_sync"), + ("public_file_sync_sources", "cosmos_public_file_sync_sources_container", "cosmos_public_file_sync_sources_container_name", "/public_workspace_id", "file_sync"), + ("personal_workspace_identities", "cosmos_personal_workspace_identities_container", "cosmos_personal_workspace_identities_container_name", "/user_id", "identities"), + ("group_workspace_identities", "cosmos_group_workspace_identities_container", "cosmos_group_workspace_identities_container_name", "/group_id", "identities"), + ("public_workspace_identities", "cosmos_public_workspace_identities_container", "cosmos_public_workspace_identities_container_name", "/public_workspace_id", "identities"), + ("global_workspace_identities", "cosmos_global_workspace_identities_container", "cosmos_global_workspace_identities_container_name", "/global_id", "identities"), + ("personal_file_sync_items", "cosmos_personal_file_sync_items_container", "cosmos_personal_file_sync_items_container_name", "/source_id", "file_sync"), + ("group_file_sync_items", "cosmos_group_file_sync_items_container", "cosmos_group_file_sync_items_container_name", "/source_id", "file_sync"), + ("public_file_sync_items", "cosmos_public_file_sync_items_container", "cosmos_public_file_sync_items_container_name", "/source_id", "file_sync"), + ("personal_file_sync_runs", "cosmos_personal_file_sync_runs_container", "cosmos_personal_file_sync_runs_container_name", "/source_id", "file_sync"), + ("group_file_sync_runs", "cosmos_group_file_sync_runs_container", "cosmos_group_file_sync_runs_container_name", "/source_id", "file_sync"), + ("public_file_sync_runs", "cosmos_public_file_sync_runs_container", "cosmos_public_file_sync_runs_container_name", "/source_id", "file_sync"), + ("user_settings", "cosmos_user_settings_container", "cosmos_user_settings_container_name", "/id", "settings"), + ("safety", "cosmos_safety_container", "cosmos_safety_container_name", "/id", "safety"), + ("feedback", "cosmos_feedback_container", "cosmos_feedback_container_name", "/id", "feedback"), + ("archived_conversations", "cosmos_archived_conversations_container", "cosmos_archived_conversations_container_name", "/id", "archive"), + ("archived_messages", "cosmos_archived_messages_container", "cosmos_archived_messages_container_name", "/conversation_id", "archive"), + ("prompts", "cosmos_user_prompts_container", "cosmos_user_prompts_container_name", "/id", "prompts"), + ("group_prompts", "cosmos_group_prompts_container", "cosmos_group_prompts_container_name", "/id", "prompts"), + ("public_prompts", "cosmos_public_prompts_container", "cosmos_public_prompts_container_name", "/id", "prompts"), + ("file_processing", "cosmos_file_processing_container", "cosmos_file_processing_container_name", "/document_id", "documents"), + ("personal_agents", "cosmos_personal_agents_container", "cosmos_personal_agents_container_name", "/user_id", "agents"), + ("personal_actions", "cosmos_personal_actions_container", "cosmos_personal_actions_container_name", "/user_id", "actions"), + ("group_agents", "cosmos_group_agents_container", "cosmos_group_agents_container_name", "/group_id", "agents"), + ("group_actions", "cosmos_group_actions_container", "cosmos_group_actions_container_name", "/group_id", "actions"), + ("global_agents", "cosmos_global_agents_container", "cosmos_global_agents_container_name", "/id", "agents"), + ("global_actions", "cosmos_global_actions_container", "cosmos_global_actions_container_name", "/id", "actions"), + ("governance_policies", "cosmos_governance_policies_container", "cosmos_governance_policies_container_name", "/id", "governance"), + ("governance_item_policies", "cosmos_governance_item_policies_container", "cosmos_governance_item_policies_container_name", "/id", "governance"), + ("agent_templates", "cosmos_agent_templates_container", "cosmos_agent_templates_container_name", "/id", "agents"), + ("agent_facts", "cosmos_agent_facts_container", "cosmos_agent_facts_container_name", "/scope_id", "agents"), + ("search_cache", "cosmos_search_cache_container", "cosmos_search_cache_container_name", "/user_id", "cache"), + ("activity_logs", "cosmos_activity_logs_container", "cosmos_activity_logs_container_name", "/user_id", "activity"), + ("notifications", "cosmos_notifications_container", "cosmos_notifications_container_name", "/user_id", "notifications"), + ("approvals", "cosmos_approvals_container", "cosmos_approvals_container_name", "/group_id", "approvals"), + ("msgraph_pending_actions", "cosmos_msgraph_pending_actions_container", "cosmos_msgraph_pending_actions_container_name", "/user_id", "actions"), + ("thoughts", "cosmos_thoughts_container", "cosmos_thoughts_container_name", "/user_id", "thoughts"), + ("archive_thoughts", "cosmos_archived_thoughts_container", "cosmos_archived_thoughts_container_name", "/user_id", "archive"), +] + DATA_MANAGEMENT_SEARCH_ARTIFACTS = [ {"name": "personal_ai_search", "client_key": "search_client_user", "index_name": "simplechat-user-index", "schema_file": "ai_search-index-user.json"}, {"name": "group_ai_search", "client_key": "search_client_group", "index_name": "simplechat-group-index", "schema_file": "ai_search-index-group.json"}, @@ -1640,6 +1718,402 @@ def _strip_cosmos_system_fields(document): } +def _format_cosmos_editor_label(container_name): + return _safe_text(container_name).replace("_", " ").title() + + +def _cosmos_editor_partition_key_field(partition_key_path): + normalized_path = _safe_text(partition_key_path) + if not normalized_path.startswith("/"): + raise DataManagementCosmosEditorError("Cosmos editor container metadata has an invalid partition key path.") + path_parts = [part for part in normalized_path.strip("/").split("/") if part] + if not path_parts: + raise DataManagementCosmosEditorError("Cosmos editor container metadata has an invalid partition key path.") + return path_parts[-1] + + +def _get_document_path_value(document, path): + if not isinstance(document, dict): + return None + path_parts = [part for part in _safe_text(path).strip("/").split("/") if part] + current_value = document + for path_part in path_parts: + if not isinstance(current_value, dict) or path_part not in current_value: + return None + current_value = current_value.get(path_part) + return current_value + + +def _build_cosmos_editor_container_metadata(): + containers = [] + seen_container_names = set() + for logical_name, container_attr, container_name_attr, partition_key_path, category in DATA_MANAGEMENT_COSMOS_EDITOR_CONTAINER_DEFINITIONS: + container_name = _safe_text(getattr(app_config, container_name_attr, logical_name)) + if not container_name or container_name in seen_container_names or not hasattr(app_config, container_attr): + continue + seen_container_names.add(container_name) + containers.append({ + "id": container_name, + "name": container_name, + "logical_name": logical_name, + "display_name": _format_cosmos_editor_label(container_name), + "category": category, + "container_attr": container_attr, + "container_name_attr": container_name_attr, + "partition_key_path": partition_key_path, + "partition_key_field": _cosmos_editor_partition_key_field(partition_key_path), + "max_page_size": DATA_MANAGEMENT_COSMOS_EDITOR_MAX_PAGE_SIZE, + "empty_query_limit": DATA_MANAGEMENT_COSMOS_EDITOR_EMPTY_QUERY_LIMIT, + "editable": True, + }) + return sorted(containers, key=lambda container: (container["category"], container["display_name"])) + + +def get_data_management_cosmos_editor_containers(): + return [ + { + "id": container["id"], + "name": container["name"], + "display_name": container["display_name"], + "category": container["category"], + "partition_key_path": container["partition_key_path"], + "partition_key_field": container["partition_key_field"], + "max_page_size": container["max_page_size"], + "empty_query_limit": container["empty_query_limit"], + "editable": container["editable"], + } + for container in _build_cosmos_editor_container_metadata() + ] + + +def _get_cosmos_editor_container_metadata(container_name): + safe_container_name = _safe_text(container_name) + if not safe_container_name: + raise DataManagementCosmosEditorError("Choose a Cosmos DB container.") + for container in _build_cosmos_editor_container_metadata(): + if safe_container_name in {container["id"], container["name"], container["logical_name"]}: + return container + raise DataManagementCosmosEditorError("The selected Cosmos DB container is not available for Data Management editing.") + + +def _get_cosmos_editor_container(container_name): + metadata = _get_cosmos_editor_container_metadata(container_name) + return metadata, getattr(app_config, metadata["container_attr"]) + + +def _hash_cosmos_editor_query(query_text): + if not query_text: + return "" + return hashlib.sha256(query_text.encode("utf-8")).hexdigest() + + +def _normalize_cosmos_editor_query(query_text): + query = _safe_text(query_text) + if not query: + return DATA_MANAGEMENT_COSMOS_EDITOR_EMPTY_QUERY, True + if len(query) > DATA_MANAGEMENT_COSMOS_EDITOR_MAX_QUERY_LENGTH: + raise DataManagementCosmosEditorError("Cosmos query text is too long.") + if ";" in query: + raise DataManagementCosmosEditorError("Run one SELECT query at a time without semicolons.") + if not re.match(r"^\s*SELECT\b", query, re.IGNORECASE): + raise DataManagementCosmosEditorError("Cosmos editor queries must start with SELECT.") + return query, False + + +def log_data_management_cosmos_editor_activity(admin_user_id, admin_email, action, status, message, details=None): + now = _now_iso() + safe_action = _safe_text(action) + safe_admin_user_id = _safe_text(admin_user_id, "unknown") or "unknown" + activity_record = { + "id": str(uuid.uuid4()), + "user_id": safe_admin_user_id, + "activity_type": "data_management", + "timestamp": now, + "created_at": now, + "action": safe_action, + "description": _safe_text(message) or safe_action, + "workspace_type": "admin", + "workspace_context": { + "action": safe_action, + "tool": "cosmos_json_editor", + }, + "additional_context": { + "tool": "cosmos_json_editor", + "status": _safe_text(status), + "details": _sanitize_activity_value(details if isinstance(details, dict) else {}), + }, + "admin": { + "user_id": safe_admin_user_id, + "email": _safe_text(admin_email, "unknown") or "unknown", + }, + } + + try: + app_config.cosmos_activity_logs_container.create_item(body=activity_record) + except Exception as exc: + log_event( + "[DataManagement] Failed to write Cosmos editor activity record.", + {"action": safe_action, "status": status, "error": str(exc)}, + level=logging.WARNING, + ) + + +def _summarize_cosmos_editor_item(item, container_metadata): + if not isinstance(item, dict): + preview_text = _safe_text(item) + return { + "id": None, + "partition_key": None, + "etag": None, + "timestamp": None, + "selectable": False, + "preview": preview_text[:200], + } + + document_id = item.get("id") + partition_key_value = _get_document_path_value(item, container_metadata["partition_key_path"]) + preview_fields = [] + for field_name in ("name", "display_name", "title", "type", "user_id", "group_id", "public_workspace_id", "created_at", "updated_at"): + field_value = item.get(field_name) + if field_value is not None: + preview_fields.append(f"{field_name}: {_safe_text(field_value)[:80]}") + if len(preview_fields) >= 3: + break + + return { + "id": _safe_text(document_id) if document_id is not None else None, + "partition_key": partition_key_value, + "etag": item.get("_etag"), + "timestamp": item.get("_ts"), + "selectable": document_id is not None and partition_key_value is not None, + "preview": "; ".join(preview_fields) if preview_fields else _safe_text(document_id)[:200], + } + + +def query_data_management_cosmos_editor_documents(container_name, query_text=None, page_size=None, continuation_token=None, admin_user_id=None, admin_email=None): + container_metadata, container = _get_cosmos_editor_container(container_name) + query, is_empty_query = _normalize_cosmos_editor_query(query_text) + safe_page_size = _safe_int( + page_size, + default=DATA_MANAGEMENT_COSMOS_EDITOR_MAX_PAGE_SIZE, + minimum=1, + maximum=DATA_MANAGEMENT_COSMOS_EDITOR_MAX_PAGE_SIZE, + ) + safe_continuation_token = None if is_empty_query else _safe_text(continuation_token) or None + started_at = time.perf_counter() + + query_iterable = container.query_items( + query=query, + enable_cross_partition_query=True, + max_item_count=safe_page_size, + ) + if hasattr(query_iterable, "by_page"): + page_iterator = query_iterable.by_page(continuation_token=safe_continuation_token) + try: + page_items = list(next(page_iterator)) + except StopIteration: + page_items = [] + next_continuation_token = None if is_empty_query else getattr(page_iterator, "continuation_token", None) + else: + page_items = list(query_iterable)[:safe_page_size] + next_continuation_token = None + + duration_ms = int((time.perf_counter() - started_at) * 1000) + items = [ + _summarize_cosmos_editor_item(item, container_metadata) + for item in page_items + ] + result = { + "container": get_data_management_cosmos_editor_container_public_metadata(container_metadata), + "query": { + "mode": "empty" if is_empty_query else "custom", + "page_size": safe_page_size, + "query_hash": _hash_cosmos_editor_query(query if not is_empty_query else ""), + "empty_query_limit_applied": is_empty_query, + }, + "items": items, + "count": len(items), + "continuation_token": next_continuation_token, + "has_more": bool(next_continuation_token), + "duration_ms": duration_ms, + } + log_data_management_cosmos_editor_activity( + admin_user_id, + admin_email, + "cosmos_editor_query_executed", + "success", + "Executed a Cosmos DB editor query.", + { + "container": container_metadata["name"], + "query_mode": result["query"]["mode"], + "page_size": safe_page_size, + "returned_count": len(items), + "has_more": bool(next_continuation_token), + "duration_ms": duration_ms, + "query_hash": result["query"]["query_hash"], + }, + ) + return result + + +def get_data_management_cosmos_editor_container_public_metadata(container_metadata): + return { + "id": container_metadata["id"], + "name": container_metadata["name"], + "display_name": container_metadata["display_name"], + "category": container_metadata["category"], + "partition_key_path": container_metadata["partition_key_path"], + "partition_key_field": container_metadata["partition_key_field"], + "max_page_size": container_metadata["max_page_size"], + "empty_query_limit": container_metadata["empty_query_limit"], + "editable": container_metadata["editable"], + } + + +def get_data_management_cosmos_editor_document(container_name, document_id, partition_key_value, admin_user_id=None, admin_email=None): + container_metadata, container = _get_cosmos_editor_container(container_name) + safe_document_id = _safe_text(document_id) + if not safe_document_id: + raise DataManagementCosmosEditorError("Choose a Cosmos DB document.") + if partition_key_value is None: + raise DataManagementCosmosEditorError("The selected document is missing its partition key value.") + + document = container.read_item(item=safe_document_id, partition_key=partition_key_value) + current_partition_key_value = _get_document_path_value(document, container_metadata["partition_key_path"]) + result = { + "container": get_data_management_cosmos_editor_container_public_metadata(container_metadata), + "document": document, + "id": safe_document_id, + "partition_key": current_partition_key_value, + "etag": document.get("_etag") if isinstance(document, dict) else None, + } + log_data_management_cosmos_editor_activity( + admin_user_id, + admin_email, + "cosmos_editor_document_opened", + "success", + "Opened a Cosmos DB document in the editor.", + { + "container": container_metadata["name"], + "document_id": safe_document_id, + "partition_key_path": container_metadata["partition_key_path"], + }, + ) + return result + + +def _summarize_cosmos_editor_changes(original_document, updated_document): + if not isinstance(original_document, dict) or not isinstance(updated_document, dict): + return {"changed_paths": [], "changed_count": 0, "added_count": 0, "removed_count": 0, "updated_count": 0} + + changed_paths = [] + added_count = 0 + removed_count = 0 + updated_count = 0 + + def compare_values(original_value, updated_value, path): + nonlocal added_count, removed_count, updated_count + if isinstance(original_value, dict) and isinstance(updated_value, dict): + all_keys = sorted(set(original_value.keys()) | set(updated_value.keys())) + for key in all_keys: + if key.startswith("_"): + continue + child_path = f"{path}.{key}" if path else key + if key not in original_value: + added_count += 1 + changed_paths.append(child_path) + continue + if key not in updated_value: + removed_count += 1 + changed_paths.append(child_path) + continue + compare_values(original_value.get(key), updated_value.get(key), child_path) + return + if original_value != updated_value: + updated_count += 1 + changed_paths.append(path) + + compare_values(original_document, updated_document, "") + return { + "changed_paths": changed_paths[:50], + "changed_count": len(changed_paths), + "added_count": added_count, + "removed_count": removed_count, + "updated_count": updated_count, + "truncated": len(changed_paths) > 50, + } + + +def _validate_cosmos_editor_save_confirmation(confirmation_accepted, confirmation_phrase): + if confirmation_accepted is not True: + raise DataManagementCosmosEditorError("Confirm that you understand this Cosmos DB edit can damage system data.") + if _safe_text(confirmation_phrase) != DATA_MANAGEMENT_COSMOS_EDITOR_CONFIRMATION_PHRASE: + raise DataManagementCosmosEditorError("Type the required confirmation phrase before saving this Cosmos DB document.") + + +def save_data_management_cosmos_editor_document(container_name, document_id, partition_key_value, etag, document, confirmation_accepted=False, confirmation_phrase="", admin_user_id=None, admin_email=None): + container_metadata, container = _get_cosmos_editor_container(container_name) + safe_document_id = _safe_text(document_id) + safe_etag = _safe_text(etag) + if not safe_document_id: + raise DataManagementCosmosEditorError("Choose a Cosmos DB document before saving.") + if partition_key_value is None: + raise DataManagementCosmosEditorError("The selected document is missing its partition key value.") + if not safe_etag: + raise DataManagementCosmosEditorError("The selected document is missing its ETag. Refresh the document before saving.") + if not isinstance(document, dict): + raise DataManagementCosmosEditorError("Cosmos DB document JSON must be an object.") + + _validate_cosmos_editor_save_confirmation(confirmation_accepted, confirmation_phrase) + + if _safe_text(document.get("id")) != safe_document_id: + raise DataManagementCosmosEditorError("Document id cannot be changed in the Cosmos DB editor.") + updated_partition_key_value = _get_document_path_value(document, container_metadata["partition_key_path"]) + if updated_partition_key_value != partition_key_value: + raise DataManagementCosmosEditorError("Document partition key value cannot be changed in the Cosmos DB editor.") + + original_document = container.read_item(item=safe_document_id, partition_key=partition_key_value) + change_summary = _summarize_cosmos_editor_changes(original_document, document) + clean_document = _strip_cosmos_system_fields(copy.deepcopy(document)) + replace_target = safe_document_id + if isinstance(original_document, dict) and original_document.get("_self"): + replace_target = original_document + saved_document = container.replace_item( + item=replace_target, + body=clean_document, + etag=safe_etag, + match_condition=MatchConditions.IfNotModified, + ) + + result = { + "container": get_data_management_cosmos_editor_container_public_metadata(container_metadata), + "document": saved_document, + "id": safe_document_id, + "partition_key": _get_document_path_value(saved_document, container_metadata["partition_key_path"]), + "etag": saved_document.get("_etag") if isinstance(saved_document, dict) else None, + "change_summary": change_summary, + } + log_data_management_cosmos_editor_activity( + admin_user_id, + admin_email, + "cosmos_editor_document_saved", + "success", + "Saved a Cosmos DB document from the Data Management editor.", + { + "container": container_metadata["name"], + "document_id": safe_document_id, + "partition_key_path": container_metadata["partition_key_path"], + "changed_count": change_summary["changed_count"], + "added_count": change_summary["added_count"], + "removed_count": change_summary["removed_count"], + "updated_count": change_summary["updated_count"], + "changed_paths": change_summary["changed_paths"], + "changed_paths_truncated": change_summary["truncated"], + }, + ) + return result + + def _save_data_management_job(job): body = _strip_cosmos_system_fields(job) saved = cosmos_data_management_jobs_container.upsert_item(body) diff --git a/application/single_app/route_backend_data_management.py b/application/single_app/route_backend_data_management.py index e4154132..e1e1de7b 100644 --- a/application/single_app/route_backend_data_management.py +++ b/application/single_app/route_backend_data_management.py @@ -13,16 +13,22 @@ DATA_MANAGEMENT_OPERATION_DRY_RUN, DATA_MANAGEMENT_OPERATION_MIGRATION, DATA_MANAGEMENT_OPERATION_RESTORE, + DataManagementCosmosEditorError, DataManagementSettingsValidationError, generate_data_management_encryption_key, + get_data_management_cosmos_editor_containers, + get_data_management_cosmos_editor_document, get_data_management_backup_summary, get_data_management_job_detail, get_data_management_jobs, get_data_management_migration_catalog, get_data_management_settings, + log_data_management_cosmos_editor_activity, queue_data_management_job, + query_data_management_cosmos_editor_documents, sanitize_data_management_job_for_admin, sanitize_data_management_settings_for_admin, + save_data_management_cosmos_editor_document, summarize_data_management_migration_plan, submit_data_management_job, test_backup_storage_connection, @@ -58,6 +64,32 @@ def _log_data_management_admin_action(action, description, additional_context=No ) +def _get_cosmos_editor_payload(): + payload = request.get_json(silent=True) or {} + return payload if isinstance(payload, dict) else {} + + +def _cosmos_editor_error_status(exc, default_status=400): + status_code = getattr(exc, "status_code", None) + if status_code == 404: + return 404 + if status_code == 412: + return 409 + return default_status + + +def _log_cosmos_editor_failure(action, message, details=None): + admin_user_id, admin_email = _get_admin_context() + log_data_management_cosmos_editor_activity( + admin_user_id, + admin_email, + action, + "failed", + message, + details=details or {}, + ) + + def register_route_backend_data_management(bp): @bp.route("/api/admin/data-management/settings", methods=["GET"]) @swagger_route(security=get_auth_security()) @@ -201,6 +233,160 @@ def test_admin_data_management_target_enhanced_citation_storage(): return jsonify({"success": False, "error": "Target Enhanced Citation Storage connection test failed."}), 400 return jsonify(result), 200 + @bp.route("/api/admin/data-management/cosmos-editor/containers", methods=["GET"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def list_admin_data_management_cosmos_editor_containers(): + return jsonify({ + "success": True, + "containers": get_data_management_cosmos_editor_containers(), + }), 200 + + @bp.route("/api/admin/data-management/cosmos-editor/danger-acknowledgement", methods=["POST"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def acknowledge_admin_data_management_cosmos_editor_danger(): + admin_user_id, admin_email = _get_admin_context() + log_data_management_cosmos_editor_activity( + admin_user_id, + admin_email, + "cosmos_editor_danger_acknowledged", + "success", + "Acknowledged the Cosmos DB editor danger prompt.", + {"prompt": "interface_entry"}, + ) + return jsonify({"success": True}), 200 + + @bp.route("/api/admin/data-management/cosmos-editor/query", methods=["POST"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def query_admin_data_management_cosmos_editor_documents(): + payload = _get_cosmos_editor_payload() + admin_user_id, admin_email = _get_admin_context() + try: + result = query_data_management_cosmos_editor_documents( + payload.get("container"), + query_text=payload.get("query"), + page_size=payload.get("page_size"), + continuation_token=payload.get("continuation_token"), + admin_user_id=admin_user_id, + admin_email=admin_email, + ) + except DataManagementCosmosEditorError as exc: + _log_cosmos_editor_failure( + "cosmos_editor_query_rejected", + "Rejected a Cosmos DB editor query.", + {"container": payload.get("container"), "error": str(exc)}, + ) + return jsonify({"success": False, "error": "Cosmos DB editor query was rejected."}), 400 + except Exception as exc: + log_event( + "[DataManagement] Cosmos editor query failed.", + {"container": payload.get("container"), "error": str(exc)}, + level=logging.WARNING, + ) + _log_cosmos_editor_failure( + "cosmos_editor_query_failed", + "Cosmos DB editor query failed.", + {"container": payload.get("container"), "status_code": getattr(exc, "status_code", None), "error": str(exc)}, + ) + return jsonify({"success": False, "error": "Cosmos DB editor query failed."}), _cosmos_editor_error_status(exc) + return jsonify({"success": True, **result}), 200 + + @bp.route("/api/admin/data-management/cosmos-editor/document", methods=["POST"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def get_admin_data_management_cosmos_editor_document(): + payload = _get_cosmos_editor_payload() + admin_user_id, admin_email = _get_admin_context() + try: + result = get_data_management_cosmos_editor_document( + payload.get("container"), + payload.get("id"), + payload.get("partition_key"), + admin_user_id=admin_user_id, + admin_email=admin_email, + ) + except DataManagementCosmosEditorError as exc: + log_event( + "[DataManagement] Cosmos editor document open rejected.", + {"container": payload.get("container"), "document_id": payload.get("id"), "error": str(exc)}, + level=logging.WARNING, + ) + _log_cosmos_editor_failure( + "cosmos_editor_document_rejected", + "Rejected a Cosmos DB editor document open request.", + {"container": payload.get("container"), "document_id": payload.get("id"), "error": str(exc)}, + ) + return jsonify({"success": False, "error": "Cosmos DB document request was invalid."}), 400 + except Exception as exc: + status_code = _cosmos_editor_error_status(exc, default_status=400) + log_event( + "[DataManagement] Cosmos editor document open failed.", + {"container": payload.get("container"), "document_id": payload.get("id"), "error": str(exc)}, + level=logging.WARNING, + ) + _log_cosmos_editor_failure( + "cosmos_editor_document_failed", + "Cosmos DB editor document open failed.", + {"container": payload.get("container"), "document_id": payload.get("id"), "status_code": getattr(exc, "status_code", None), "error": str(exc)}, + ) + error_message = "Cosmos DB document was not found." if status_code == 404 else "Cosmos DB document could not be opened." + return jsonify({"success": False, "error": error_message}), status_code + return jsonify({"success": True, **result}), 200 + + @bp.route("/api/admin/data-management/cosmos-editor/document", methods=["PUT"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def save_admin_data_management_cosmos_editor_document(): + payload = _get_cosmos_editor_payload() + admin_user_id, admin_email = _get_admin_context() + try: + result = save_data_management_cosmos_editor_document( + payload.get("container"), + payload.get("id"), + payload.get("partition_key"), + payload.get("etag"), + payload.get("document"), + confirmation_accepted=payload.get("confirmation_accepted") is True, + confirmation_phrase=payload.get("confirmation_phrase"), + admin_user_id=admin_user_id, + admin_email=admin_email, + ) + except DataManagementCosmosEditorError as exc: + _log_cosmos_editor_failure( + "cosmos_editor_save_rejected", + "Rejected a Cosmos DB editor save request.", + {"container": payload.get("container"), "document_id": payload.get("id"), "error": str(exc)}, + ) + return jsonify({"success": False, "error": "Cosmos DB document save request was rejected."}), 400 + except Exception as exc: + status_code = _cosmos_editor_error_status(exc, default_status=400) + log_event( + "[DataManagement] Cosmos editor save failed.", + {"container": payload.get("container"), "document_id": payload.get("id"), "error": str(exc)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + _log_cosmos_editor_failure( + "cosmos_editor_save_failed", + "Cosmos DB editor save failed.", + {"container": payload.get("container"), "document_id": payload.get("id"), "status_code": getattr(exc, "status_code", None), "error": str(exc)}, + ) + if status_code == 409: + error_message = "Cosmos DB document changed after it was opened. Refresh before saving again." + elif status_code == 404: + error_message = "Cosmos DB document was not found." + else: + error_message = "Cosmos DB document could not be saved." + return jsonify({"success": False, "error": error_message}), status_code + return jsonify({"success": True, **result}), 200 + @bp.route("/api/admin/data-management/jobs", methods=["GET"]) @swagger_route(security=get_auth_security()) @login_required diff --git a/application/single_app/static/css/styles.css b/application/single_app/static/css/styles.css index 180b970d..397b38a8 100644 --- a/application/single_app/static/css/styles.css +++ b/application/single_app/static/css/styles.css @@ -1240,3 +1240,19 @@ main { border-left-color: #495057; color: #adb5bd; } + +/* Cosmos editor modal */ +#data-management-cosmos-editor-results-modal .cosmos-editor-results-modal-body { + overflow-x: hidden; +} + +#data-management-cosmos-editor-results-modal .cosmos-editor-results-list { + max-height: calc(100vh - 18rem); + overflow-y: auto; +} + +@media (max-width: 991.98px) { + #data-management-cosmos-editor-results-modal .cosmos-editor-results-list { + max-height: 18rem; + } +} diff --git a/application/single_app/static/js/admin/admin_data_management.js b/application/single_app/static/js/admin/admin_data_management.js index c6ae959a..6476ec03 100644 --- a/application/single_app/static/js/admin/admin_data_management.js +++ b/application/single_app/static/js/admin/admin_data_management.js @@ -6,6 +6,7 @@ const redactedValue = "***REDACTED***"; const backupStorageAuthManagedIdentity = "managed_identity"; const backupStorageAuthConnectionString = "connection_string"; const targetCosmosDatabaseName = "SimpleChat"; +const cosmosEditorConfirmationPhrase = "I understand this can damage system data"; const elements = {}; let dataManagementModified = false; let storedBackupConnectionStringAvailable = false; @@ -15,6 +16,12 @@ let currentJobDetailId = null; let jobDetailRefreshTimer = null; let jobDetailRefreshInFlight = false; let refreshListsWhenJobCompletes = false; +let cosmosEditorUnlocked = false; +let cosmosEditorContainers = []; +let cosmosEditorContinuationToken = null; +let cosmosEditorResultCount = 0; +let cosmosEditorSelectedDocument = null; +let cosmosEditorPendingDocument = null; const jobDetailRefreshIntervalMs = 4000; const activeJobStatuses = new Set(["queued", "running"]); @@ -35,6 +42,8 @@ document.addEventListener("DOMContentLoaded", () => { loadDataManagementSettings(); loadDataManagementBackups(); loadDataManagementJobs(); + initializeCosmosEditorLockedState(); + loadCosmosEditorContainers(); }); function bindElements() { @@ -135,6 +144,35 @@ function bindElements() { "data-management-job-artifacts-tbody", "data-management-job-manifest-detail", "data-management-job-warnings", + "data-management-cosmos-editor-section", + "data-management-cosmos-editor-open-danger-btn", + "data-management-cosmos-editor-locked-message", + "data-management-cosmos-editor-workspace", + "data_management_cosmos_editor_container", + "data-management-cosmos-editor-container-help", + "data_management_cosmos_editor_page_size", + "data_management_cosmos_editor_query", + "data-management-cosmos-editor-run-query-btn", + "data-management-cosmos-editor-next-page-btn", + "data-management-cosmos-editor-refresh-document-btn", + "data-management-cosmos-editor-query-status", + "data-management-cosmos-editor-results-list", + "data-management-cosmos-editor-document-meta", + "data-management-cosmos-editor-save-btn", + "data_management_cosmos_editor_document_json", + "data-management-cosmos-editor-document-help", + "data-management-cosmos-editor-danger-modal", + "data_management_cosmos_editor_danger_accept", + "data-management-cosmos-editor-accept-danger-btn", + "data-management-cosmos-editor-save-modal", + "data-management-cosmos-editor-save-subtitle", + "data-management-cosmos-editor-save-summary", + "data_management_cosmos_editor_confirmation_phrase", + "data-management-cosmos-editor-confirm-save-btn", + "data-management-cosmos-editor-results-modal", + "data-management-cosmos-editor-modal-title", + "data-management-cosmos-editor-modal-subtitle", + "data-management-cosmos-editor-modal-status", ]; ids.forEach((id) => { @@ -182,6 +220,17 @@ function bindEvents() { elements.dataManagementRefreshJobsBtn?.addEventListener("click", loadDataManagementJobs); elements.dataManagementJobDetailModal?.addEventListener("hidden.bs.modal", () => stopJobDetailAutoRefresh({ clearJob: true })); elements.dataManagementKeyVaultLink?.addEventListener("click", openKeyVaultSettings); + elements.dataManagementCosmosEditorOpenDangerBtn?.addEventListener("click", showCosmosEditorDangerModal); + elements.datamanagementcosmoseditordangeraccept?.addEventListener("change", updateCosmosEditorDangerAcceptState); + elements.dataManagementCosmosEditorAcceptDangerBtn?.addEventListener("click", acceptCosmosEditorDanger); + elements.datamanagementcosmoseditorcontainer?.addEventListener("change", () => resetCosmosEditorQueryState({ clearResults: true, clearDocument: true })); + elements.datamanagementcosmoseditorquery?.addEventListener("input", () => resetCosmosEditorQueryState({ clearResults: false, clearDocument: true })); + elements.dataManagementCosmosEditorRunQueryBtn?.addEventListener("click", () => queryCosmosEditorDocuments(false)); + elements.dataManagementCosmosEditorNextPageBtn?.addEventListener("click", () => queryCosmosEditorDocuments(true)); + elements.dataManagementCosmosEditorRefreshDocumentBtn?.addEventListener("click", refreshCosmosEditorDocument); + elements.dataManagementCosmosEditorSaveBtn?.addEventListener("click", openCosmosEditorSaveModal); + elements.datamanagementcosmoseditorconfirmationphrase?.addEventListener("input", updateCosmosEditorConfirmSaveState); + elements.dataManagementCosmosEditorConfirmSaveBtn?.addEventListener("click", saveCosmosEditorDocument); bindDataManagementChangeTracking(); setStorageAuthVisibility(); setMigrationTargetVisibility(); @@ -210,6 +259,9 @@ function bindMigrationPickerEvents() { function bindDataManagementChangeTracking() { elements.tabPane?.querySelectorAll("input, select, textarea").forEach((element) => { + if (element.closest("[data-ignore-data-management-change='true']")) { + return; + } const eventName = element.type === "checkbox" || element.type === "radio" || element.tagName === "SELECT" ? "change" : "input"; element.addEventListener(eventName, markDataManagementModified); }); @@ -268,6 +320,14 @@ function setBusy(button, isBusy, busyLabel = "Working...") { button.disabled = false; } +function setButtonDisabled(button, disabled) { + if (!button) { + return; + } + button.disabled = Boolean(disabled); + button.setAttribute("aria-disabled", String(Boolean(disabled))); +} + function setValue(element, value) { if (!element) { return; @@ -560,6 +620,475 @@ async function requestJson(url, options = {}) { return data; } +function initializeCosmosEditorLockedState() { + setElementVisible(elements.dataManagementCosmosEditorWorkspace, cosmosEditorUnlocked); + setElementVisible(elements.dataManagementCosmosEditorLockedMessage, !cosmosEditorUnlocked); + setButtonDisabled(elements.dataManagementCosmosEditorRunQueryBtn, !cosmosEditorUnlocked || cosmosEditorContainers.length === 0); + setButtonDisabled(elements.dataManagementCosmosEditorNextPageBtn, true); + setButtonDisabled(elements.dataManagementCosmosEditorRefreshDocumentBtn, true); + setButtonDisabled(elements.dataManagementCosmosEditorSaveBtn, true); + if (elements.datamanagementcosmoseditorcontainer) { + elements.datamanagementcosmoseditorcontainer.disabled = !cosmosEditorUnlocked; + } + if (elements.datamanagementcosmoseditorpagesize) { + elements.datamanagementcosmoseditorpagesize.disabled = !cosmosEditorUnlocked; + } + if (elements.datamanagementcosmoseditorquery) { + elements.datamanagementcosmoseditorquery.disabled = !cosmosEditorUnlocked; + } + if (elements.datamanagementcosmoseditordocumentjson) { + elements.datamanagementcosmoseditordocumentjson.disabled = true; + } +} + +function showCosmosEditorDangerModal() { + if (cosmosEditorUnlocked) { + setElementVisible(elements.dataManagementCosmosEditorWorkspace, true); + return; + } + if (elements.datamanagementcosmoseditordangeraccept) { + elements.datamanagementcosmoseditordangeraccept.checked = false; + } + updateCosmosEditorDangerAcceptState(); + if (elements.dataManagementCosmosEditorDangerModal && window.bootstrap?.Modal) { + window.bootstrap.Modal.getOrCreateInstance(elements.dataManagementCosmosEditorDangerModal).show(); + } +} + +function updateCosmosEditorDangerAcceptState() { + setButtonDisabled( + elements.dataManagementCosmosEditorAcceptDangerBtn, + !elements.datamanagementcosmoseditordangeraccept?.checked + ); +} + +async function acceptCosmosEditorDanger() { + if (!elements.datamanagementcosmoseditordangeraccept?.checked) { + return; + } + setBusy(elements.dataManagementCosmosEditorAcceptDangerBtn, true, "Unlocking..."); + try { + await requestJson("/api/admin/data-management/cosmos-editor/danger-acknowledgement", { + method: "POST", + body: "{}", + }); + cosmosEditorUnlocked = true; + initializeCosmosEditorLockedState(); + setStatus("Cosmos DB JSON editor unlocked for this page session.", "warning"); + showToast("Cosmos DB JSON editor unlocked.", "warning"); + if (elements.dataManagementCosmosEditorDangerModal && window.bootstrap?.Modal) { + window.bootstrap.Modal.getOrCreateInstance(elements.dataManagementCosmosEditorDangerModal).hide(); + } + } catch (error) { + setStatus(error.message || "Cosmos DB editor could not be unlocked.", "danger"); + showToast(error.message || "Cosmos DB editor could not be unlocked.", "danger"); + } finally { + setBusy(elements.dataManagementCosmosEditorAcceptDangerBtn, false); + updateCosmosEditorDangerAcceptState(); + } +} + +async function loadCosmosEditorContainers() { + const select = elements.datamanagementcosmoseditorcontainer; + if (!select) { + return; + } + try { + const data = await requestJson("/api/admin/data-management/cosmos-editor/containers", { method: "GET" }); + cosmosEditorContainers = Array.isArray(data.containers) ? data.containers : []; + renderCosmosEditorContainerOptions(); + } catch (error) { + cosmosEditorContainers = []; + renderCosmosEditorContainerOptions(error.message || "Cosmos DB containers could not be loaded."); + } +} + +function renderCosmosEditorContainerOptions(errorMessage = "") { + const select = elements.datamanagementcosmoseditorcontainer; + if (!select) { + return; + } + select.replaceChildren(); + const placeholder = document.createElement("option"); + placeholder.value = ""; + placeholder.textContent = errorMessage || "Choose a container"; + select.appendChild(placeholder); + cosmosEditorContainers.forEach((container) => { + const option = document.createElement("option"); + option.value = container.name; + option.textContent = `${container.display_name || container.name} (${container.partition_key_path})`; + select.appendChild(option); + }); + setButtonDisabled(elements.dataManagementCosmosEditorRunQueryBtn, !cosmosEditorUnlocked || cosmosEditorContainers.length === 0); + updateCosmosEditorContainerHelp(); +} + +function updateCosmosEditorContainerHelp() { + const container = getSelectedCosmosEditorContainer(); + if (!container) { + setText(elements.dataManagementCosmosEditorContainerHelp, "Choose a known SimpleChat Cosmos DB container."); + return; + } + setText( + elements.dataManagementCosmosEditorContainerHelp, + `Category: ${formatActivityLabel(container.category)}. Partition key: ${container.partition_key_path}.` + ); +} + +function getSelectedCosmosEditorContainer() { + const selectedName = getValue(elements.datamanagementcosmoseditorcontainer); + return cosmosEditorContainers.find((container) => container.name === selectedName) || null; +} + +function setCosmosEditorQueryStatus(message) { + setText(elements.dataManagementCosmosEditorQueryStatus, message); + setText(elements.dataManagementCosmosEditorModalStatus, message); +} + +function showCosmosEditorResultsModal() { + const selectedContainer = getSelectedCosmosEditorContainer(); + if (selectedContainer) { + setText(elements.dataManagementCosmosEditorModalTitle, `${selectedContainer.display_name || selectedContainer.name} Results`); + setText( + elements.dataManagementCosmosEditorModalSubtitle, + `Container: ${selectedContainer.name}; partition key: ${selectedContainer.partition_key_path}` + ); + } + if (elements.dataManagementCosmosEditorResultsModal && window.bootstrap?.Modal) { + window.bootstrap.Modal.getOrCreateInstance(elements.dataManagementCosmosEditorResultsModal).show(); + } +} + +function resetCosmosEditorQueryState(options = {}) { + cosmosEditorContinuationToken = null; + if (options.clearResults) { + cosmosEditorResultCount = 0; + renderCosmosEditorResultMessage("Run a query to list documents."); + } + if (options.clearDocument) { + clearCosmosEditorDocument(); + } + updateCosmosEditorContainerHelp(); + setButtonDisabled(elements.dataManagementCosmosEditorNextPageBtn, true); +} + +function clearCosmosEditorDocument() { + cosmosEditorSelectedDocument = null; + cosmosEditorPendingDocument = null; + setValue(elements.datamanagementcosmoseditordocumentjson, ""); + if (elements.datamanagementcosmoseditordocumentjson) { + elements.datamanagementcosmoseditordocumentjson.disabled = true; + elements.datamanagementcosmoseditordocumentjson.setAttribute("aria-disabled", "true"); + } + setText(elements.dataManagementCosmosEditorDocumentMeta, "Select a result to load JSON."); + setButtonDisabled(elements.dataManagementCosmosEditorRefreshDocumentBtn, true); + setButtonDisabled(elements.dataManagementCosmosEditorSaveBtn, true); +} + +async function queryCosmosEditorDocuments(useContinuation) { + if (!cosmosEditorUnlocked) { + showCosmosEditorDangerModal(); + return; + } + const selectedContainer = getSelectedCosmosEditorContainer(); + if (!selectedContainer) { + showToast("Choose a Cosmos DB container first.", "warning"); + return; + } + if (useContinuation && !cosmosEditorContinuationToken) { + return; + } + + const triggerButton = useContinuation ? elements.dataManagementCosmosEditorNextPageBtn : elements.dataManagementCosmosEditorRunQueryBtn; + setBusy(triggerButton, true, useContinuation ? "Loading..." : "Querying..."); + setCosmosEditorQueryStatus(useContinuation ? "Loading next page..." : "Running query..."); + try { + const data = await requestJson("/api/admin/data-management/cosmos-editor/query", { + method: "POST", + body: JSON.stringify({ + container: selectedContainer.name, + query: getValue(elements.datamanagementcosmoseditorquery), + page_size: getNumberValue(elements.datamanagementcosmoseditorpagesize, 100), + continuation_token: useContinuation ? cosmosEditorContinuationToken : null, + }), + }); + if (!useContinuation) { + clearCosmosEditorDocument(); + cosmosEditorResultCount = 0; + } + renderCosmosEditorResults(data, useContinuation); + cosmosEditorContinuationToken = data.continuation_token || null; + setButtonDisabled(elements.dataManagementCosmosEditorNextPageBtn, !cosmosEditorContinuationToken); + const modeLabel = data.query?.mode === "empty" ? "empty browse" : "custom SELECT"; + const pageNote = data.has_more ? "More results are available." : "No more results returned."; + setCosmosEditorQueryStatus(`${formatNumber(cosmosEditorResultCount)} loaded from ${modeLabel} query. ${pageNote}`); + showCosmosEditorResultsModal(); + } catch (error) { + setCosmosEditorQueryStatus(error.message || "Cosmos DB query failed."); + showToast(error.message || "Cosmos DB query failed.", "danger"); + } finally { + setBusy(triggerButton, false); + setButtonDisabled(elements.dataManagementCosmosEditorRunQueryBtn, !cosmosEditorUnlocked || cosmosEditorContainers.length === 0); + setButtonDisabled(elements.dataManagementCosmosEditorNextPageBtn, !cosmosEditorContinuationToken); + } +} + +function renderCosmosEditorResults(data, append) { + const resultsList = elements.dataManagementCosmosEditorResultsList; + if (!resultsList) { + return; + } + const items = Array.isArray(data.items) ? data.items : []; + if (!append) { + resultsList.replaceChildren(); + } + if (!items.length && !append) { + renderCosmosEditorResultMessage("No documents matched this query."); + return; + } + items.forEach((item) => { + cosmosEditorResultCount += 1; + resultsList.appendChild(createCosmosEditorResultButton(item, cosmosEditorResultCount)); + }); +} + +function renderCosmosEditorResultMessage(message) { + const resultsList = elements.dataManagementCosmosEditorResultsList; + if (!resultsList) { + return; + } + const messageElement = document.createElement("div"); + messageElement.className = "list-group-item text-muted"; + messageElement.textContent = message; + resultsList.replaceChildren(messageElement); +} + +function createCosmosEditorResultButton(item, index) { + const button = document.createElement("button"); + button.type = "button"; + button.className = "list-group-item list-group-item-action py-2"; + button.disabled = item.selectable !== true; + + const header = document.createElement("div"); + header.className = "d-flex justify-content-between align-items-start gap-2"; + const title = document.createElement("div"); + title.className = "small fw-semibold text-break"; + title.textContent = item.id || "Non-document projection"; + const indexBadge = createBadge(formatNumber(index), item.selectable === true ? "bg-light text-dark border" : "bg-secondary"); + header.append(title, indexBadge); + const partition = document.createElement("div"); + partition.className = "small text-muted text-break"; + partition.textContent = item.partition_key !== null && item.partition_key !== undefined + ? `Partition key: ${formatDetailValue(item.partition_key)}` + : "Projection is missing the partition key."; + const preview = document.createElement("div"); + preview.className = "small text-muted"; + const previewText = item.preview || "No preview fields available."; + preview.textContent = previewText.length > 120 ? `${previewText.slice(0, 117)}...` : previewText; + button.append(header, partition, preview); + if (item.selectable === true) { + button.addEventListener("click", () => openCosmosEditorDocument(item)); + } + return button; +} + +async function openCosmosEditorDocument(item) { + const selectedContainer = getSelectedCosmosEditorContainer(); + if (!selectedContainer || !item?.id) { + return; + } + setText(elements.dataManagementCosmosEditorDocumentMeta, "Loading selected document..."); + try { + const data = await requestJson("/api/admin/data-management/cosmos-editor/document", { + method: "POST", + body: JSON.stringify({ + container: selectedContainer.name, + id: item.id, + partition_key: item.partition_key, + }), + }); + renderCosmosEditorDocument(data); + } catch (error) { + clearCosmosEditorDocument(); + setText(elements.dataManagementCosmosEditorDocumentMeta, error.message || "Document could not be opened."); + showToast(error.message || "Document could not be opened.", "danger"); + } +} + +function renderCosmosEditorDocument(data) { + const documentJson = data.document && typeof data.document === "object" ? data.document : {}; + cosmosEditorSelectedDocument = { + container: data.container?.name || getValue(elements.datamanagementcosmoseditorcontainer), + id: data.id, + partitionKey: data.partition_key, + etag: data.etag, + originalDocument: documentJson, + }; + setValue(elements.datamanagementcosmoseditordocumentjson, JSON.stringify(documentJson, null, 2)); + if (elements.datamanagementcosmoseditordocumentjson) { + elements.datamanagementcosmoseditordocumentjson.disabled = false; + elements.datamanagementcosmoseditordocumentjson.setAttribute("aria-disabled", "false"); + } + setText( + elements.dataManagementCosmosEditorDocumentMeta, + `Container: ${cosmosEditorSelectedDocument.container}; id: ${cosmosEditorSelectedDocument.id}; ETag: ${cosmosEditorSelectedDocument.etag || "not returned"}` + ); + setButtonDisabled(elements.dataManagementCosmosEditorRefreshDocumentBtn, false); + setButtonDisabled(elements.dataManagementCosmosEditorSaveBtn, false); +} + +function refreshCosmosEditorDocument() { + if (!cosmosEditorSelectedDocument) { + return; + } + openCosmosEditorDocument({ + id: cosmosEditorSelectedDocument.id, + partition_key: cosmosEditorSelectedDocument.partitionKey, + selectable: true, + }); +} + +function parseCosmosEditorDocumentJson() { + const rawJson = getValue(elements.datamanagementcosmoseditordocumentjson); + const parsedDocument = JSON.parse(rawJson); + if (!parsedDocument || typeof parsedDocument !== "object" || Array.isArray(parsedDocument)) { + throw new Error("Cosmos DB document JSON must be an object."); + } + return parsedDocument; +} + +function openCosmosEditorSaveModal() { + if (!cosmosEditorSelectedDocument) { + return; + } + let parsedDocument; + try { + parsedDocument = parseCosmosEditorDocumentJson(); + } catch (error) { + showToast(error.message || "Document JSON is invalid.", "danger"); + return; + } + + cosmosEditorPendingDocument = parsedDocument; + const summary = summarizeCosmosEditorChanges(cosmosEditorSelectedDocument.originalDocument, parsedDocument); + renderCosmosEditorSaveSummary(summary); + setText( + elements.dataManagementCosmosEditorSaveSubtitle, + `Container: ${cosmosEditorSelectedDocument.container}; id: ${cosmosEditorSelectedDocument.id}` + ); + setValue(elements.datamanagementcosmoseditorconfirmationphrase, ""); + updateCosmosEditorConfirmSaveState(); + if (elements.dataManagementCosmosEditorSaveModal && window.bootstrap?.Modal) { + window.bootstrap.Modal.getOrCreateInstance(elements.dataManagementCosmosEditorSaveModal).show(); + } +} + +function summarizeCosmosEditorChanges(originalDocument, updatedDocument) { + const changedPaths = []; + let addedCount = 0; + let removedCount = 0; + let updatedCount = 0; + + function compareValues(originalValue, updatedValue, path) { + if (isPlainObject(originalValue) && isPlainObject(updatedValue)) { + const keys = Array.from(new Set([...Object.keys(originalValue), ...Object.keys(updatedValue)])).sort(); + keys.forEach((key) => { + if (key.startsWith("_")) { + return; + } + const childPath = path ? `${path}.${key}` : key; + if (!Object.prototype.hasOwnProperty.call(originalValue, key)) { + addedCount += 1; + changedPaths.push(childPath); + return; + } + if (!Object.prototype.hasOwnProperty.call(updatedValue, key)) { + removedCount += 1; + changedPaths.push(childPath); + return; + } + compareValues(originalValue[key], updatedValue[key], childPath); + }); + return; + } + if (JSON.stringify(originalValue) !== JSON.stringify(updatedValue)) { + updatedCount += 1; + changedPaths.push(path); + } + } + + compareValues(originalDocument || {}, updatedDocument || {}, ""); + return { + changedPaths, + changedCount: changedPaths.length, + addedCount, + removedCount, + updatedCount, + }; +} + +function isPlainObject(value) { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function renderCosmosEditorSaveSummary(summary) { + const container = elements.dataManagementCosmosEditorSaveSummary; + if (!container) { + return; + } + const wrapper = document.createElement("div"); + wrapper.className = "vstack gap-2"; + wrapper.append( + createDetailBlock("Changed paths", formatNumber(summary.changedCount)), + createDetailBlock("Added / removed / updated", `${formatNumber(summary.addedCount)} / ${formatNumber(summary.removedCount)} / ${formatNumber(summary.updatedCount)}`) + ); + const paths = document.createElement("div"); + paths.className = "small text-muted text-break"; + paths.textContent = summary.changedPaths.length + ? summary.changedPaths.slice(0, 20).join(", ") + : "No value changes detected. Saving will still refresh the document ETag."; + wrapper.appendChild(paths); + container.replaceChildren(wrapper); +} + +function updateCosmosEditorConfirmSaveState() { + const phraseMatches = getValue(elements.datamanagementcosmoseditorconfirmationphrase) === cosmosEditorConfirmationPhrase; + setButtonDisabled(elements.dataManagementCosmosEditorConfirmSaveBtn, !phraseMatches); +} + +async function saveCosmosEditorDocument() { + if (!cosmosEditorSelectedDocument || !cosmosEditorPendingDocument) { + return; + } + setBusy(elements.dataManagementCosmosEditorConfirmSaveBtn, true, "Saving..."); + try { + const data = await requestJson("/api/admin/data-management/cosmos-editor/document", { + method: "PUT", + body: JSON.stringify({ + container: cosmosEditorSelectedDocument.container, + id: cosmosEditorSelectedDocument.id, + partition_key: cosmosEditorSelectedDocument.partitionKey, + etag: cosmosEditorSelectedDocument.etag, + document: cosmosEditorPendingDocument, + confirmation_accepted: true, + confirmation_phrase: cosmosEditorConfirmationPhrase, + }), + }); + if (elements.dataManagementCosmosEditorSaveModal && window.bootstrap?.Modal) { + window.bootstrap.Modal.getOrCreateInstance(elements.dataManagementCosmosEditorSaveModal).hide(); + } + renderCosmosEditorDocument(data); + setStatus("Cosmos DB document saved. The edit was recorded in Activity Logs.", "success"); + showToast("Cosmos DB document saved.", "success"); + } catch (error) { + setStatus(error.message || "Cosmos DB document could not be saved.", "danger"); + showToast(error.message || "Cosmos DB document could not be saved.", "danger"); + } finally { + setBusy(elements.dataManagementCosmosEditorConfirmSaveBtn, false); + updateCosmosEditorConfirmSaveState(); + } +} + async function loadDataManagementSettings() { try { const data = await requestJson("/api/admin/data-management/settings", { method: "GET" }); diff --git a/application/single_app/templates/_sidebar_nav.html b/application/single_app/templates/_sidebar_nav.html index 9e80ba11..61e682e5 100644 --- a/application/single_app/templates/_sidebar_nav.html +++ b/application/single_app/templates/_sidebar_nav.html @@ -646,6 +646,11 @@ Migration +