diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..af044401 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,103 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL Advanced" + +on: + push: + branches: [ "Development" ] + pull_request: + branches: [ "Development" ] + workflow_dispatch: + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: actions + build-mode: none + - language: javascript-typescript + build-mode: none + - language: python + build-mode: none + # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Add any setup steps before running the `github/codeql-action/init` action. + # This includes steps like installing compilers or runtimes (`actions/setup-node` + # or others). This is typically only required for manual builds. + # - name: Setup runtime (example) + # uses: actions/setup-example@v1 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + queries: +security-and-quality + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - name: Run manual build steps + if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 + with: + category: "/language:${{matrix.language}}" diff --git a/.gitignore b/.gitignore index e8e2d544..969720f2 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,7 @@ scripts/agent.json scripts/me.json .github/instructions/python-venv-path.instructions.md .github/instructions/local.instructions.md +/venv deploy.zip diff --git a/application/single_app/app_settings_cache.py b/application/single_app/app_settings_cache.py index 90dbc4a6..7067bfb3 100644 --- a/application/single_app/app_settings_cache.py +++ b/application/single_app/app_settings_cache.py @@ -11,7 +11,7 @@ import os import threading import time -from datetime import datetime +from datetime import datetime, timedelta from redis import Redis from redis.credentials import CredentialProvider from azure.identity import DefaultAzureCredential @@ -32,6 +32,7 @@ APP_GOVERNANCE_CACHE_VERSION = 0 APP_SETTINGS_SHARED_VERSION_CACHE = {'value': 0, 'expires_at': 0} APP_GOVERNANCE_SHARED_VERSION_CACHE = {'value': 0, 'expires_at': 0} +APP_REDIS_CLIENT = None APP_SETTINGS_CACHE_KEY = 'APP_SETTINGS_CACHE' APP_SETTINGS_CACHE_VERSION_KEY = 'APP_SETTINGS_CACHE_VERSION' APP_SETTINGS_CACHE_VERSION_DOC_ID = 'app_settings_cache_version' @@ -41,6 +42,8 @@ GOVERNANCE_CACHE_VERSION_DOC_ID = 'governance_cache_version' CACHE_VERSION_DOC_TYPE = 'cache_version' CACHE_VERSION_READ_TTL_SECONDS = 15 +COSMOS_CACHE_ENTRY_DOC_TYPE = 'app_cache_entry' +COSMOS_CACHE_ENTRY_PREFIX = 'app_cache_entry:' update_settings_cache = None get_settings_cache = None get_app_settings_cache_version = None @@ -188,6 +191,495 @@ def _set_ttl_cached_version(version_cache, version): version_cache['value'] = _normalize_cache_version(version) version_cache['expires_at'] = time.time() + CACHE_VERSION_READ_TTL_SECONDS + +def _log_cache_fallback(operation, exception, log_event_func=None): + message = f"[ASC] Redis cache operation failed; using Cosmos/local fallback for {operation}." + _logger.warning("%s Error: %s", message, exception) + if callable(log_event_func): + log_event_func( + message, + extra={ + 'operation': operation, + 'error': str(exception), + }, + level=logging.WARNING, + ) + + +def _build_cosmos_cache_doc_id(cache_key): + return f"{COSMOS_CACHE_ENTRY_PREFIX}{cache_key}" + + +def _get_cosmos_cache_container(): + from config import cosmos_settings_container + return cosmos_settings_container + + +def _serialize_datetime(value): + if isinstance(value, datetime): + return value.isoformat() + return value + + +def _set_cosmos_cache_entry(cache_key, payload, ttl_seconds=None, log_event_func=None): + try: + now = datetime.utcnow() + expires_at = None + if ttl_seconds is not None: + expires_at = now + timedelta(seconds=max(int(ttl_seconds), 0)) + doc_id = _build_cosmos_cache_doc_id(cache_key) + _get_cosmos_cache_container().upsert_item({ + 'id': doc_id, + 'type': COSMOS_CACHE_ENTRY_DOC_TYPE, + 'cache_key': cache_key, + 'payload': copy.deepcopy(payload), + 'created_at': now.isoformat(), + 'updated_at': now.isoformat(), + 'expires_at': _serialize_datetime(expires_at), + }) + return True + except Exception as ex: + _logger.warning("[ASC] Cosmos cache fallback write failed for %s: %s", cache_key, ex) + if callable(log_event_func): + log_event_func( + "[ASC] Cosmos cache fallback write failed.", + extra={'cache_key': cache_key, 'error': str(ex)}, + level=logging.WARNING, + ) + return False + + +def _get_cosmos_cache_entry(cache_key, log_event_func=None): + try: + doc_id = _build_cosmos_cache_doc_id(cache_key) + doc = _get_cosmos_cache_container().read_item(item=doc_id, partition_key=doc_id) + expires_at = doc.get('expires_at') + if expires_at: + try: + if datetime.fromisoformat(expires_at) <= datetime.utcnow(): + return None + except (TypeError, ValueError): + return None + return copy.deepcopy(doc.get('payload')) + except Exception as ex: + status_code = getattr(ex, 'status_code', None) + if status_code != 404: + _logger.warning("[ASC] Cosmos cache fallback read failed for %s: %s", cache_key, ex) + if callable(log_event_func): + log_event_func( + "[ASC] Cosmos cache fallback read failed.", + extra={'cache_key': cache_key, 'error': str(ex), 'status_code': status_code}, + level=logging.WARNING, + ) + return None + + +def _delete_cosmos_cache_entry(cache_key, log_event_func=None): + try: + doc_id = _build_cosmos_cache_doc_id(cache_key) + _get_cosmos_cache_container().delete_item(item=doc_id, partition_key=doc_id) + return True + except Exception as ex: + status_code = getattr(ex, 'status_code', None) + if status_code != 404: + _logger.warning("[ASC] Cosmos cache fallback delete failed for %s: %s", cache_key, ex) + if callable(log_event_func): + log_event_func( + "[ASC] Cosmos cache fallback delete failed.", + extra={'cache_key': cache_key, 'error': str(ex), 'status_code': status_code}, + level=logging.WARNING, + ) + return False + + +def _get_app_settings_cache_version_fallback(log_event_func=None): + global APP_SETTINGS_CACHE_VERSION + try: + from config import cosmos_settings_container + return _get_ttl_cached_cosmos_version( + APP_SETTINGS_SHARED_VERSION_CACHE, + cosmos_settings_container, + APP_SETTINGS_CACHE_VERSION_DOC_ID, + APP_SETTINGS_CACHE_VERSION, + log_event_func=log_event_func, + ) + except Exception as ex: + _logger.warning("[ASC] Shared cache version read failed; using local version fallback: %s", ex) + if callable(log_event_func): + log_event_func( + "[ASC] Shared cache version read failed; using local version fallback.", + extra={'version_doc_id': APP_SETTINGS_CACHE_VERSION_DOC_ID, 'error': str(ex)}, + level=logging.WARNING, + ) + with _app_cache_lock: + return APP_SETTINGS_CACHE_VERSION + + +def _bump_app_settings_cache_version_fallback(log_event_func=None): + global APP_SETTINGS_CACHE_VERSION + try: + from config import cosmos_settings_container + bumped_version = _bump_cosmos_cache_version( + cosmos_settings_container, + APP_SETTINGS_CACHE_VERSION_DOC_ID, + log_event_func=log_event_func, + ) + if bumped_version is not None: + with _app_cache_lock: + APP_SETTINGS_CACHE_VERSION = bumped_version + _set_ttl_cached_version(APP_SETTINGS_SHARED_VERSION_CACHE, bumped_version) + return bumped_version + except Exception as ex: + _logger.warning("[ASC] Shared cache version bump failed; using local version fallback: %s", ex) + if callable(log_event_func): + log_event_func( + "[ASC] Shared cache version bump failed; using local version fallback.", + extra={'version_doc_id': APP_SETTINGS_CACHE_VERSION_DOC_ID, 'error': str(ex)}, + level=logging.WARNING, + ) + + with _app_cache_lock: + APP_SETTINGS_CACHE_VERSION += 1 + fallback_version = APP_SETTINGS_CACHE_VERSION + _set_ttl_cached_version(APP_SETTINGS_SHARED_VERSION_CACHE, fallback_version) + return fallback_version + + +def _update_settings_cache_fallback(new_settings, log_event_func=None): + global APP_SETTINGS_CACHE, APP_SETTINGS_CACHE_VERSION + shared_version = _get_app_settings_cache_version_fallback(log_event_func=log_event_func) + with _app_cache_lock: + APP_SETTINGS_CACHE = copy.deepcopy(new_settings or {}) + APP_SETTINGS_CACHE_VERSION = shared_version + + +def _get_settings_cache_fallback(log_event_func=None): + global APP_SETTINGS_CACHE, APP_SETTINGS_CACHE_VERSION + shared_version = _get_app_settings_cache_version_fallback(log_event_func=log_event_func) + with _app_cache_lock: + if APP_SETTINGS_CACHE and APP_SETTINGS_CACHE_VERSION == shared_version: + return copy.deepcopy(APP_SETTINGS_CACHE) + + try: + from config import cosmos_settings_container + loaded_settings = cosmos_settings_container.read_item( + item='app_settings', + partition_key='app_settings', + ) + with _app_cache_lock: + APP_SETTINGS_CACHE = copy.deepcopy(loaded_settings or {}) + APP_SETTINGS_CACHE_VERSION = shared_version + return copy.deepcopy(loaded_settings or {}) + except Exception as ex: + _logger.warning("[ASC] Failed to refresh app settings cache from Cosmos; using local cache fallback: %s", ex) + if callable(log_event_func): + log_event_func( + "[ASC] Failed to refresh app settings cache from Cosmos; using local cache fallback.", + extra={'error': str(ex)}, + level=logging.WARNING, + ) + with _app_cache_lock: + return copy.deepcopy(APP_SETTINGS_CACHE) + + +def _get_governance_cache_version_fallback(log_event_func=None): + global APP_GOVERNANCE_CACHE_VERSION + try: + from config import cosmos_governance_policies_container + return _get_ttl_cached_cosmos_version( + APP_GOVERNANCE_SHARED_VERSION_CACHE, + cosmos_governance_policies_container, + GOVERNANCE_CACHE_VERSION_DOC_ID, + APP_GOVERNANCE_CACHE_VERSION, + log_event_func=log_event_func, + ) + except Exception as ex: + _logger.warning("[ASC] Governance cache version read failed; using local version fallback: %s", ex) + if callable(log_event_func): + log_event_func( + "[ASC] Governance cache version read failed; using local version fallback.", + extra={'version_doc_id': GOVERNANCE_CACHE_VERSION_DOC_ID, 'error': str(ex)}, + level=logging.WARNING, + ) + with _app_cache_lock: + return APP_GOVERNANCE_CACHE_VERSION + + +def _bump_governance_cache_version_fallback(log_event_func=None): + global APP_GOVERNANCE_CACHE_VERSION + try: + from config import cosmos_governance_policies_container + bumped_version = _bump_cosmos_cache_version( + cosmos_governance_policies_container, + GOVERNANCE_CACHE_VERSION_DOC_ID, + log_event_func=log_event_func, + ) + if bumped_version is not None: + with _app_cache_lock: + APP_GOVERNANCE_CACHE_VERSION = bumped_version + _set_ttl_cached_version(APP_GOVERNANCE_SHARED_VERSION_CACHE, bumped_version) + return bumped_version + except Exception as ex: + _logger.warning("[ASC] Governance cache version bump failed; using local version fallback: %s", ex) + if callable(log_event_func): + log_event_func( + "[ASC] Governance cache version bump failed; using local version fallback.", + extra={'version_doc_id': GOVERNANCE_CACHE_VERSION_DOC_ID, 'error': str(ex)}, + level=logging.WARNING, + ) + + with _app_cache_lock: + APP_GOVERNANCE_CACHE_VERSION += 1 + fallback_version = APP_GOVERNANCE_CACHE_VERSION + _set_ttl_cached_version(APP_GOVERNANCE_SHARED_VERSION_CACHE, fallback_version) + return fallback_version + + +def _get_stream_session_metadata_key(cache_key): + return f'STREAM_SESSION_META:{cache_key}' + + +def _get_stream_session_events_key(cache_key): + return f'STREAM_SESSION_EVENTS:{cache_key}' + + +def _initialize_stream_session_cache_fallback(cache_key, metadata, ttl_seconds=None, log_event_func=None): + expiration_timestamp = _get_expiration_timestamp(ttl_seconds) + with _app_cache_lock: + APP_STREAM_SESSION_METADATA[cache_key] = { + 'value': copy.deepcopy(metadata or {}), + 'expires_at': expiration_timestamp, + } + APP_STREAM_SESSION_EVENTS[cache_key] = { + 'value': [], + 'expires_at': expiration_timestamp, + } + _set_cosmos_cache_entry( + _get_stream_session_metadata_key(cache_key), + metadata or {}, + ttl_seconds=ttl_seconds, + log_event_func=log_event_func, + ) + _set_cosmos_cache_entry( + _get_stream_session_events_key(cache_key), + [], + ttl_seconds=ttl_seconds, + log_event_func=log_event_func, + ) + + +def _set_stream_session_meta_fallback(cache_key, metadata, ttl_seconds=None, log_event_func=None): + expiration_timestamp = _get_expiration_timestamp(ttl_seconds) + with _app_cache_lock: + APP_STREAM_SESSION_METADATA[cache_key] = { + 'value': copy.deepcopy(metadata or {}), + 'expires_at': expiration_timestamp, + } + _set_cosmos_cache_entry( + _get_stream_session_metadata_key(cache_key), + metadata or {}, + ttl_seconds=ttl_seconds, + log_event_func=log_event_func, + ) + + +def _get_stream_session_meta_fallback(cache_key, log_event_func=None): + with _app_cache_lock: + entry = APP_STREAM_SESSION_METADATA.get(cache_key) + if entry and not _is_expired(entry): + return copy.deepcopy(entry.get('value') or {}) + if entry: + APP_STREAM_SESSION_METADATA.pop(cache_key, None) + + cached = _get_cosmos_cache_entry( + _get_stream_session_metadata_key(cache_key), + log_event_func=log_event_func, + ) + return copy.deepcopy(cached or {}) if cached is not None else None + + +def _append_stream_session_event_fallback(cache_key, event_text, ttl_seconds=None, log_event_func=None): + expiration_timestamp = _get_expiration_timestamp(ttl_seconds) + with _app_cache_lock: + entry = APP_STREAM_SESSION_EVENTS.get(cache_key) + if _is_expired(entry): + entry = {'value': [], 'expires_at': expiration_timestamp} + APP_STREAM_SESSION_EVENTS[cache_key] = entry + entry['value'].append(event_text) + if expiration_timestamp is not None: + entry['expires_at'] = expiration_timestamp + events_payload = list(entry.get('value') or []) + _set_cosmos_cache_entry( + _get_stream_session_events_key(cache_key), + events_payload, + ttl_seconds=ttl_seconds, + log_event_func=log_event_func, + ) + + +def _get_stream_session_events_fallback(cache_key, start_index=0, log_event_func=None): + start = int(start_index or 0) + with _app_cache_lock: + entry = APP_STREAM_SESSION_EVENTS.get(cache_key) + if entry and not _is_expired(entry): + return list((entry.get('value') or [])[start:]) + if entry: + APP_STREAM_SESSION_EVENTS.pop(cache_key, None) + + cached = _get_cosmos_cache_entry( + _get_stream_session_events_key(cache_key), + log_event_func=log_event_func, + ) + return list((cached or [])[start:]) + + +def _delete_stream_session_cache_fallback(cache_key, log_event_func=None): + with _app_cache_lock: + APP_STREAM_SESSION_METADATA.pop(cache_key, None) + APP_STREAM_SESSION_EVENTS.pop(cache_key, None) + _delete_cosmos_cache_entry( + _get_stream_session_metadata_key(cache_key), + log_event_func=log_event_func, + ) + _delete_cosmos_cache_entry( + _get_stream_session_events_key(cache_key), + log_event_func=log_event_func, + ) + + +def _get_user_ui_settings_cache_key(user_id): + return f'{USER_UI_SETTINGS_CACHE_KEY_PREFIX}:{user_id}' + + +def _get_user_ui_settings_cache_fallback(user_id, log_event_func=None): + with _app_cache_lock: + entry = APP_USER_UI_SETTINGS_CACHE.get(user_id) + if entry and not _is_expired(entry): + return copy.deepcopy(entry.get('value') or {}) + if entry: + APP_USER_UI_SETTINGS_CACHE.pop(user_id, None) + + cached = _get_cosmos_cache_entry( + _get_user_ui_settings_cache_key(user_id), + log_event_func=log_event_func, + ) + return copy.deepcopy(cached) if cached is not None else None + + +def _set_user_ui_settings_cache_fallback(user_id, ui_settings, ttl_seconds=None, log_event_func=None): + ttl = int(ttl_seconds or USER_UI_SETTINGS_CACHE_TTL_SECONDS) + expiration_timestamp = _get_expiration_timestamp(ttl) + with _app_cache_lock: + APP_USER_UI_SETTINGS_CACHE[user_id] = { + 'value': copy.deepcopy(ui_settings or {}), + 'expires_at': expiration_timestamp, + } + _set_cosmos_cache_entry( + _get_user_ui_settings_cache_key(user_id), + ui_settings or {}, + ttl_seconds=ttl, + log_event_func=log_event_func, + ) + + +def _delete_user_ui_settings_cache_fallback(user_id, log_event_func=None): + with _app_cache_lock: + APP_USER_UI_SETTINGS_CACHE.pop(user_id, None) + _delete_cosmos_cache_entry( + _get_user_ui_settings_cache_key(user_id), + log_event_func=log_event_func, + ) + + +def _assign_fallback_cache_functions(log_event_func=None): + global update_settings_cache, get_settings_cache + global initialize_stream_session_cache, set_stream_session_meta, get_stream_session_meta + global append_stream_session_event, get_stream_session_events, delete_stream_session_cache + global get_user_ui_settings_cache, set_user_ui_settings_cache, delete_user_ui_settings_cache + global get_app_settings_cache_version, bump_app_settings_cache_version + global get_governance_cache_version, bump_governance_cache_version + global app_cache_is_using_redis + global APP_REDIS_CLIENT + + app_cache_is_using_redis = False + APP_REDIS_CLIENT = None + update_settings_cache = lambda new_settings: _update_settings_cache_fallback( + new_settings, + log_event_func=log_event_func, + ) + get_settings_cache = lambda: _get_settings_cache_fallback(log_event_func=log_event_func) + get_app_settings_cache_version = lambda: _get_app_settings_cache_version_fallback( + log_event_func=log_event_func, + ) + bump_app_settings_cache_version = lambda: _bump_app_settings_cache_version_fallback( + log_event_func=log_event_func, + ) + initialize_stream_session_cache = lambda cache_key, metadata, ttl_seconds=None: ( + _initialize_stream_session_cache_fallback( + cache_key, + metadata, + ttl_seconds=ttl_seconds, + log_event_func=log_event_func, + ) + ) + set_stream_session_meta = lambda cache_key, metadata, ttl_seconds=None: ( + _set_stream_session_meta_fallback( + cache_key, + metadata, + ttl_seconds=ttl_seconds, + log_event_func=log_event_func, + ) + ) + get_stream_session_meta = lambda cache_key: _get_stream_session_meta_fallback( + cache_key, + log_event_func=log_event_func, + ) + append_stream_session_event = lambda cache_key, event_text, ttl_seconds=None: ( + _append_stream_session_event_fallback( + cache_key, + event_text, + ttl_seconds=ttl_seconds, + log_event_func=log_event_func, + ) + ) + get_stream_session_events = lambda cache_key, start_index=0: _get_stream_session_events_fallback( + cache_key, + start_index=start_index, + log_event_func=log_event_func, + ) + delete_stream_session_cache = lambda cache_key: _delete_stream_session_cache_fallback( + cache_key, + log_event_func=log_event_func, + ) + get_user_ui_settings_cache = lambda user_id: _get_user_ui_settings_cache_fallback( + user_id, + log_event_func=log_event_func, + ) + set_user_ui_settings_cache = lambda user_id, ui_settings, ttl_seconds=None: ( + _set_user_ui_settings_cache_fallback( + user_id, + ui_settings, + ttl_seconds=ttl_seconds, + log_event_func=log_event_func, + ) + ) + delete_user_ui_settings_cache = lambda user_id: _delete_user_ui_settings_cache_fallback( + user_id, + log_event_func=log_event_func, + ) + get_governance_cache_version = lambda: _get_governance_cache_version_fallback( + log_event_func=log_event_func, + ) + bump_governance_cache_version = lambda: _bump_governance_cache_version_fallback( + log_event_func=log_event_func, + ) + + +def get_app_cache_redis_client(): + """Return the configured Redis client for shared cache consumers, when available.""" + return APP_REDIS_CLIENT if app_cache_is_using_redis else None + + def configure_app_cache(settings, redis_cache_endpoint=None): global _settings, update_settings_cache, get_settings_cache, APP_SETTINGS_CACHE global APP_USER_UI_SETTINGS_CACHE, APP_STREAM_SESSION_METADATA, APP_STREAM_SESSION_EVENTS @@ -199,65 +691,79 @@ def configure_app_cache(settings, redis_cache_endpoint=None): global get_app_settings_cache_version, bump_app_settings_cache_version global get_governance_cache_version, bump_governance_cache_version global app_cache_is_using_redis + global APP_REDIS_CLIENT # Local import to avoid circular dependency: functions_keyvault imports app_settings_cache. from functions_appinsights import log_event _settings = settings use_redis = _settings.get('enable_redis_cache', False) app_cache_is_using_redis = False + APP_REDIS_CLIENT = None if use_redis: - app_cache_is_using_redis = True redis_url = settings.get('redis_url', '').strip() redis_auth_type = settings.get('redis_auth_type', 'key').strip().lower() - if redis_auth_type == 'managed_identity': - log_event("[ASC] Redis enabled using Managed Identity", level=logging.INFO) - redis_client = create_redis_managed_identity_client( - redis_url, - settings=settings - ) - elif redis_auth_type == 'key_vault': - log_event("[ASC] Redis enabled using Key Vault Secret", level=logging.INFO) - # Local import to avoid circular dependency: functions_keyvault imports app_settings_cache. - from functions_keyvault import retrieve_secret_direct - redis_key_secret_name = settings.get('redis_key', '').strip() - try: + try: + if not redis_url: + raise ValueError('Redis cache is enabled but redis_url is empty.') + if redis_auth_type == 'managed_identity': + log_event("[ASC] Redis enabled using Managed Identity", level=logging.INFO) + redis_client = create_redis_managed_identity_client( + redis_url, + settings=settings + ) + elif redis_auth_type == 'key_vault': + log_event("[ASC] Redis enabled using Key Vault Secret", level=logging.INFO) + # Local import to avoid circular dependency: functions_keyvault imports app_settings_cache. + from functions_keyvault import retrieve_secret_direct + redis_key_secret_name = settings.get('redis_key', '').strip() # Pass settings directly: get_settings_cache() is still None at this point # because configure_app_cache has not finished initialising the cache yet. redis_password = retrieve_secret_direct(redis_key_secret_name, settings=settings) if redis_password: redis_password = redis_password.strip() log_event("[ASC] Redis key retrieved from Key Vault successfully", level=logging.INFO) - except Exception as kv_err: - log_event(f"[ASC] ERROR: Failed to retrieve Redis key from Key Vault: {kv_err}", level=logging.ERROR, exceptionTraceback=True) - raise - - redis_client = Redis( - host=redis_url, - port=6380, - db=0, - password=redis_password, - ssl=True - ) - else: - redis_key = settings.get('redis_key', '').strip() - log_event("[ASC] Redis enabled using Access Key", level=logging.INFO) - redis_client = Redis( - host=redis_url, - port=6380, - db=0, - password=redis_key, - ssl=True - ) + + redis_client = Redis( + host=redis_url, + port=6380, + db=0, + password=redis_password, + ssl=True + ) + else: + redis_key = settings.get('redis_key', '').strip() + log_event("[ASC] Redis enabled using Access Key", level=logging.INFO) + redis_client = Redis( + host=redis_url, + port=6380, + db=0, + password=redis_key, + ssl=True + ) + app_cache_is_using_redis = True + APP_REDIS_CLIENT = redis_client + except Exception as redis_init_error: + _log_cache_fallback('redis_initialization', redis_init_error, log_event_func=log_event) + _assign_fallback_cache_functions(log_event_func=log_event) + return def get_app_settings_cache_version_redis(): - cached = redis_client.get(APP_SETTINGS_CACHE_VERSION_KEY) - if cached is None: - redis_client.setnx(APP_SETTINGS_CACHE_VERSION_KEY, 0) - return 0 - return _normalize_cache_version(cached) + try: + cached = redis_client.get(APP_SETTINGS_CACHE_VERSION_KEY) + if cached is None: + redis_client.setnx(APP_SETTINGS_CACHE_VERSION_KEY, 0) + return 0 + return _normalize_cache_version(cached) + except Exception as ex: + _log_cache_fallback('get_app_settings_cache_version', ex, log_event_func=log_event) + return _get_app_settings_cache_version_fallback(log_event_func=log_event) def bump_app_settings_cache_version_redis(): - return _normalize_cache_version(redis_client.incr(APP_SETTINGS_CACHE_VERSION_KEY)) + try: + return _normalize_cache_version(redis_client.incr(APP_SETTINGS_CACHE_VERSION_KEY)) + except Exception as ex: + _log_cache_fallback('bump_app_settings_cache_version', ex, log_event_func=log_event) + return _bump_app_settings_cache_version_fallback(log_event_func=log_event) def get_ttl_cached_app_settings_version_redis(): now = time.time() @@ -271,26 +777,36 @@ def get_ttl_cached_app_settings_version_redis(): def update_settings_cache_redis(new_settings): global APP_SETTINGS_CACHE, APP_SETTINGS_CACHE_VERSION - redis_client.set(APP_SETTINGS_CACHE_KEY, json.dumps(new_settings)) - shared_version = get_app_settings_cache_version_redis() - with _app_cache_lock: - APP_SETTINGS_CACHE = new_settings - APP_SETTINGS_CACHE_VERSION = shared_version - _set_ttl_cached_version(APP_SETTINGS_SHARED_VERSION_CACHE, shared_version) + try: + redis_client.set(APP_SETTINGS_CACHE_KEY, json.dumps(new_settings)) + shared_version = get_app_settings_cache_version_redis() + with _app_cache_lock: + APP_SETTINGS_CACHE = copy.deepcopy(new_settings or {}) + APP_SETTINGS_CACHE_VERSION = shared_version + _set_ttl_cached_version(APP_SETTINGS_SHARED_VERSION_CACHE, shared_version) + except Exception as ex: + _log_cache_fallback('update_settings_cache', ex, log_event_func=log_event) + _update_settings_cache_fallback(new_settings, log_event_func=log_event) def get_settings_cache_redis(): global APP_SETTINGS_CACHE, APP_SETTINGS_CACHE_VERSION - shared_version = get_ttl_cached_app_settings_version_redis() - with _app_cache_lock: - if APP_SETTINGS_CACHE and APP_SETTINGS_CACHE_VERSION == shared_version: - return APP_SETTINGS_CACHE + try: + shared_version = get_ttl_cached_app_settings_version_redis() + with _app_cache_lock: + if APP_SETTINGS_CACHE and APP_SETTINGS_CACHE_VERSION == shared_version: + return copy.deepcopy(APP_SETTINGS_CACHE) - cached = redis_client.get(APP_SETTINGS_CACHE_KEY) - loaded_settings = json.loads(cached) if cached else {} - with _app_cache_lock: - APP_SETTINGS_CACHE = loaded_settings - APP_SETTINGS_CACHE_VERSION = shared_version - return loaded_settings + cached = redis_client.get(APP_SETTINGS_CACHE_KEY) + if cached is None: + return _get_settings_cache_fallback(log_event_func=log_event) + loaded_settings = json.loads(cached) + with _app_cache_lock: + APP_SETTINGS_CACHE = copy.deepcopy(loaded_settings or {}) + APP_SETTINGS_CACHE_VERSION = shared_version + return copy.deepcopy(loaded_settings or {}) + except Exception as ex: + _log_cache_fallback('get_settings_cache', ex, log_event_func=log_event) + return _get_settings_cache_fallback(log_event_func=log_event) def get_stream_session_metadata_key(cache_key): return f'STREAM_SESSION_META:{cache_key}' @@ -299,88 +815,156 @@ def get_stream_session_events_key(cache_key): return f'STREAM_SESSION_EVENTS:{cache_key}' def initialize_stream_session_cache_redis(cache_key, metadata, ttl_seconds=None): - metadata_key = get_stream_session_metadata_key(cache_key) - events_key = get_stream_session_events_key(cache_key) - pipeline = redis_client.pipeline() - pipeline.delete(events_key) - pipeline.set(metadata_key, json.dumps(metadata)) - if ttl_seconds is not None: - pipeline.expire(metadata_key, int(ttl_seconds)) - pipeline.execute() + try: + metadata_key = get_stream_session_metadata_key(cache_key) + events_key = get_stream_session_events_key(cache_key) + pipeline = redis_client.pipeline() + pipeline.delete(events_key) + pipeline.set(metadata_key, json.dumps(metadata)) + if ttl_seconds is not None: + pipeline.expire(metadata_key, int(ttl_seconds)) + pipeline.execute() + except Exception as ex: + _log_cache_fallback('initialize_stream_session_cache', ex, log_event_func=log_event) + _initialize_stream_session_cache_fallback( + cache_key, + metadata, + ttl_seconds=ttl_seconds, + log_event_func=log_event, + ) def set_stream_session_meta_redis(cache_key, metadata, ttl_seconds=None): - metadata_key = get_stream_session_metadata_key(cache_key) - events_key = get_stream_session_events_key(cache_key) - pipeline = redis_client.pipeline() - pipeline.set(metadata_key, json.dumps(metadata)) - if ttl_seconds is not None: - pipeline.expire(metadata_key, int(ttl_seconds)) - if redis_client.exists(events_key): - pipeline.expire(events_key, int(ttl_seconds)) - pipeline.execute() + try: + metadata_key = get_stream_session_metadata_key(cache_key) + events_key = get_stream_session_events_key(cache_key) + pipeline = redis_client.pipeline() + pipeline.set(metadata_key, json.dumps(metadata)) + if ttl_seconds is not None: + pipeline.expire(metadata_key, int(ttl_seconds)) + if redis_client.exists(events_key): + pipeline.expire(events_key, int(ttl_seconds)) + pipeline.execute() + except Exception as ex: + _log_cache_fallback('set_stream_session_meta', ex, log_event_func=log_event) + _set_stream_session_meta_fallback( + cache_key, + metadata, + ttl_seconds=ttl_seconds, + log_event_func=log_event, + ) def get_stream_session_meta_redis(cache_key): - cached = redis_client.get(get_stream_session_metadata_key(cache_key)) - return json.loads(cached) if cached else None + try: + cached = redis_client.get(get_stream_session_metadata_key(cache_key)) + return json.loads(cached) if cached else None + except Exception as ex: + _log_cache_fallback('get_stream_session_meta', ex, log_event_func=log_event) + return _get_stream_session_meta_fallback(cache_key, log_event_func=log_event) def append_stream_session_event_redis(cache_key, event_text, ttl_seconds=None): - metadata_key = get_stream_session_metadata_key(cache_key) - events_key = get_stream_session_events_key(cache_key) - pipeline = redis_client.pipeline() - pipeline.rpush(events_key, event_text) - if ttl_seconds is not None: - pipeline.expire(events_key, int(ttl_seconds)) - if redis_client.exists(metadata_key): - pipeline.expire(metadata_key, int(ttl_seconds)) - pipeline.execute() + try: + metadata_key = get_stream_session_metadata_key(cache_key) + events_key = get_stream_session_events_key(cache_key) + pipeline = redis_client.pipeline() + pipeline.rpush(events_key, event_text) + if ttl_seconds is not None: + pipeline.expire(events_key, int(ttl_seconds)) + if redis_client.exists(metadata_key): + pipeline.expire(metadata_key, int(ttl_seconds)) + pipeline.execute() + except Exception as ex: + _log_cache_fallback('append_stream_session_event', ex, log_event_func=log_event) + _append_stream_session_event_fallback( + cache_key, + event_text, + ttl_seconds=ttl_seconds, + log_event_func=log_event, + ) def get_stream_session_events_redis(cache_key, start_index=0): - cached_events = redis_client.lrange( - get_stream_session_events_key(cache_key), - int(start_index or 0), - -1, - ) - normalized_events = [] - for event in cached_events: - if isinstance(event, bytes): - normalized_events.append(event.decode('utf-8')) - else: - normalized_events.append(event) - return normalized_events + try: + cached_events = redis_client.lrange( + get_stream_session_events_key(cache_key), + int(start_index or 0), + -1, + ) + normalized_events = [] + for event in cached_events: + if isinstance(event, bytes): + normalized_events.append(event.decode('utf-8')) + else: + normalized_events.append(event) + return normalized_events + except Exception as ex: + _log_cache_fallback('get_stream_session_events', ex, log_event_func=log_event) + return _get_stream_session_events_fallback( + cache_key, + start_index=start_index, + log_event_func=log_event, + ) def delete_stream_session_cache_redis(cache_key): - redis_client.delete( - get_stream_session_metadata_key(cache_key), - get_stream_session_events_key(cache_key), - ) + try: + redis_client.delete( + get_stream_session_metadata_key(cache_key), + get_stream_session_events_key(cache_key), + ) + except Exception as ex: + _log_cache_fallback('delete_stream_session_cache', ex, log_event_func=log_event) + _delete_stream_session_cache_fallback(cache_key, log_event_func=log_event) def get_user_ui_settings_cache_key(user_id): return f'{USER_UI_SETTINGS_CACHE_KEY_PREFIX}:{user_id}' def get_user_ui_settings_cache_redis(user_id): - cached = redis_client.get(get_user_ui_settings_cache_key(user_id)) - return json.loads(cached) if cached else None + try: + cached = redis_client.get(get_user_ui_settings_cache_key(user_id)) + return json.loads(cached) if cached else None + except Exception as ex: + _log_cache_fallback('get_user_ui_settings_cache', ex, log_event_func=log_event) + return _get_user_ui_settings_cache_fallback(user_id, log_event_func=log_event) def set_user_ui_settings_cache_redis(user_id, ui_settings, ttl_seconds=None): ttl = int(ttl_seconds or USER_UI_SETTINGS_CACHE_TTL_SECONDS) - redis_client.setex( - get_user_ui_settings_cache_key(user_id), - ttl, - json.dumps(ui_settings or {}) - ) + try: + redis_client.setex( + get_user_ui_settings_cache_key(user_id), + ttl, + json.dumps(ui_settings or {}) + ) + except Exception as ex: + _log_cache_fallback('set_user_ui_settings_cache', ex, log_event_func=log_event) + _set_user_ui_settings_cache_fallback( + user_id, + ui_settings, + ttl_seconds=ttl, + log_event_func=log_event, + ) def delete_user_ui_settings_cache_redis(user_id): - redis_client.delete(get_user_ui_settings_cache_key(user_id)) + try: + redis_client.delete(get_user_ui_settings_cache_key(user_id)) + except Exception as ex: + _log_cache_fallback('delete_user_ui_settings_cache', ex, log_event_func=log_event) + _delete_user_ui_settings_cache_fallback(user_id, log_event_func=log_event) def get_governance_cache_version_redis(): - cached = redis_client.get(GOVERNANCE_CACHE_VERSION_KEY) - if cached is None: - redis_client.setnx(GOVERNANCE_CACHE_VERSION_KEY, 0) - return 0 - return _normalize_cache_version(cached) + try: + cached = redis_client.get(GOVERNANCE_CACHE_VERSION_KEY) + if cached is None: + redis_client.setnx(GOVERNANCE_CACHE_VERSION_KEY, 0) + return 0 + return _normalize_cache_version(cached) + except Exception as ex: + _log_cache_fallback('get_governance_cache_version', ex, log_event_func=log_event) + return _get_governance_cache_version_fallback(log_event_func=log_event) def bump_governance_cache_version_redis(): - return _normalize_cache_version(redis_client.incr(GOVERNANCE_CACHE_VERSION_KEY)) + try: + return _normalize_cache_version(redis_client.incr(GOVERNANCE_CACHE_VERSION_KEY)) + except Exception as ex: + _log_cache_fallback('bump_governance_cache_version', ex, log_event_func=log_event) + return _bump_governance_cache_version_fallback(log_event_func=log_event) update_settings_cache = update_settings_cache_redis get_settings_cache = get_settings_cache_redis @@ -528,7 +1112,13 @@ def get_app_settings_cache_version_mem(): APP_SETTINGS_CACHE_VERSION, log_event_func=log_event, ) - except Exception: + except Exception as ex: + _logger.warning("[ASC] Shared cache version read failed; using local version fallback: %s", ex) + log_event( + "[ASC] Shared cache version read failed; using local version fallback.", + extra={'version_doc_id': APP_SETTINGS_CACHE_VERSION_DOC_ID, 'error': str(ex)}, + level=logging.WARNING, + ) with _app_cache_lock: return APP_SETTINGS_CACHE_VERSION @@ -546,8 +1136,13 @@ def bump_app_settings_cache_version_mem(): APP_SETTINGS_CACHE_VERSION = bumped_version _set_ttl_cached_version(APP_SETTINGS_SHARED_VERSION_CACHE, bumped_version) return bumped_version - except Exception: - pass + except Exception as ex: + _logger.warning("[ASC] Shared cache version bump failed; using local version fallback: %s", ex) + log_event( + "[ASC] Shared cache version bump failed; using local version fallback.", + extra={'version_doc_id': APP_SETTINGS_CACHE_VERSION_DOC_ID, 'error': str(ex)}, + level=logging.WARNING, + ) with _app_cache_lock: APP_SETTINGS_CACHE_VERSION += 1 @@ -566,7 +1161,13 @@ def get_governance_cache_version_mem(): APP_GOVERNANCE_CACHE_VERSION, log_event_func=log_event, ) - except Exception: + except Exception as ex: + _logger.warning("[ASC] Governance cache version read failed; using local version fallback: %s", ex) + log_event( + "[ASC] Governance cache version read failed; using local version fallback.", + extra={'version_doc_id': GOVERNANCE_CACHE_VERSION_DOC_ID, 'error': str(ex)}, + level=logging.WARNING, + ) with _app_cache_lock: return APP_GOVERNANCE_CACHE_VERSION @@ -584,8 +1185,13 @@ def bump_governance_cache_version_mem(): APP_GOVERNANCE_CACHE_VERSION = bumped_version _set_ttl_cached_version(APP_GOVERNANCE_SHARED_VERSION_CACHE, bumped_version) return bumped_version - except Exception: - pass + except Exception as ex: + _logger.warning("[ASC] Governance cache version bump failed; using local version fallback: %s", ex) + log_event( + "[ASC] Governance cache version bump failed; using local version fallback.", + extra={'version_doc_id': GOVERNANCE_CACHE_VERSION_DOC_ID, 'error': str(ex)}, + level=logging.WARNING, + ) with _app_cache_lock: APP_GOVERNANCE_CACHE_VERSION += 1 diff --git a/application/single_app/background_tasks.py b/application/single_app/background_tasks.py index 109b4a43..464255dd 100644 --- a/application/single_app/background_tasks.py +++ b/application/single_app/background_tasks.py @@ -25,6 +25,12 @@ calculate_cosmos_throughput_autoscale_interval_seconds, evaluate_and_apply_cosmos_throughput_scaling, ) +from functions_app_maintenance import ( + APP_MAINTENANCE_LOCK_NAME, + calculate_app_maintenance_sleep_seconds, + get_app_maintenance_settings, + run_app_maintenance_once, +) from functions_debug import debug_print from functions_data_management import check_due_data_management_jobs_once from functions_file_sync import check_due_file_sync_sources_once @@ -696,6 +702,40 @@ def run_data_management_scheduler_loop(): time.sleep(60) +def run_app_maintenance_loop(): + """Run idempotent app maintenance tasks under a distributed lock.""" + while True: + lock_document = None + sleep_seconds = 3600 + try: + settings = get_settings() + maintenance_settings = get_app_maintenance_settings(settings) + sleep_seconds = maintenance_settings.get('check_interval_seconds', 3600) + if maintenance_settings.get('enabled') and maintenance_settings.get('run_on_startup'): + lock_document = acquire_distributed_task_lock( + APP_MAINTENANCE_LOCK_NAME, + lease_seconds=maintenance_settings.get('lease_seconds', 300), + ) + if lock_document: + maintenance_result = run_app_maintenance_once(triggered_by='background', settings=settings) + sleep_seconds = calculate_app_maintenance_sleep_seconds( + maintenance_result, + maintenance_settings, + ) + except Exception as exc: + log_event( + '[AppMaintenance] Error in maintenance scheduler loop.', + extra={'error': str(exc)}, + level=logging.ERROR, + exceptionTraceback=True, + ) + finally: + if lock_document: + release_distributed_task_lock(lock_document) + + time.sleep(max(int(sleep_seconds or 3600), 15)) + + def start_background_task_threads(): """Start all background task loops for the current process.""" task_specs = [ @@ -708,6 +748,7 @@ def start_background_task_threads(): ('File Sync scheduler background task started.', run_file_sync_scheduler_loop), ('Tabular generated-output scheduler background task started.', run_tabular_generated_output_scheduler_loop), ('Data Management scheduler background task started.', run_data_management_scheduler_loop), + ('App maintenance background task started.', run_app_maintenance_loop), ] started_threads = [] diff --git a/application/single_app/config.py b/application/single_app/config.py index a22a9d0b..11f93c24 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -71,7 +71,8 @@ from io import BytesIO from typing import List -from azure.cosmos import CosmosClient, PartitionKey, exceptions +import azure.cosmos as azure_cosmos +from azure.cosmos import PartitionKey, exceptions from azure.cosmos.exceptions import CosmosResourceNotFoundError from azure.core.credentials import AzureKeyCredential from azure.ai.documentintelligence import DocumentIntelligenceClient @@ -95,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.015" +VERSION = "0.250.047" SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') SESSION_COOKIE_HTTPONLY = os.getenv('SESSION_COOKIE_HTTPONLY', 'true').lower() != 'false' @@ -419,9 +420,9 @@ def get_redis_cache_infrastructure_endpoint(redis_hostname: str) -> str: cosmos_authentication_type = os.getenv("AZURE_COSMOS_AUTHENTICATION_TYPE", "key") #key or managed_identity if cosmos_authentication_type == "managed_identity": - cosmos_client = CosmosClient(cosmos_endpoint, credential=DefaultAzureCredential(), consistency_level="Session") + cosmos_client = azure_cosmos.CosmosClient(cosmos_endpoint, credential=DefaultAzureCredential(), consistency_level="Session") else: - cosmos_client = CosmosClient(cosmos_endpoint, cosmos_key, consistency_level="Session") + cosmos_client = azure_cosmos.CosmosClient(cosmos_endpoint, cosmos_key, consistency_level="Session") cosmos_database_name = "SimpleChat" cosmos_database = cosmos_client.create_database_if_not_exists(cosmos_database_name) @@ -564,6 +565,12 @@ def get_redis_cache_infrastructure_endpoint(redis_hostname: str) -> str: partition_key=PartitionKey(path="/id") ) +cosmos_document_access_index_container_name = "document_access_index" +cosmos_document_access_index_container = cosmos_database.create_container_if_not_exists( + id=cosmos_document_access_index_container_name, + partition_key=PartitionKey(path="/scope_key") +) + cosmos_personal_file_sync_sources_container_name = "personal_file_sync_sources" cosmos_personal_file_sync_sources_container = cosmos_database.create_container_if_not_exists( id=cosmos_personal_file_sync_sources_container_name, diff --git a/application/single_app/functions_app_maintenance.py b/application/single_app/functions_app_maintenance.py new file mode 100644 index 00000000..fa1cb398 --- /dev/null +++ b/application/single_app/functions_app_maintenance.py @@ -0,0 +1,486 @@ +# functions_app_maintenance.py +"""Application maintenance framework for safe Cosmos-backed foundation tasks.""" + +import logging +import time +import uuid +from datetime import datetime, timezone + +from config import VERSION, cosmos_governance_policies_container, cosmos_settings_container +from functions_appinsights import log_event +from functions_cosmos_indexing import ( + COSMOS_INDEXING_POLICY_APPLY_SETTING, + get_cosmos_indexing_policy_status, + run_cosmos_indexing_policy_maintenance, +) +from functions_conversation_cache import get_conversation_cache_metrics, get_conversation_cache_settings +from functions_document_access_index import ( + DOCUMENT_ACCESS_ACTIVE_MAINTENANCE_INTERVAL_SETTING, + DOCUMENT_ACCESS_BACKFILL_BATCH_SIZE_SETTING, + DOCUMENT_ACCESS_BACKFILL_ENABLED_SETTING, + DOCUMENT_ACCESS_REPAIR_BATCH_SIZE_SETTING, + get_document_access_index_settings, + get_document_access_index_backfill_status, + is_document_access_index_maintenance_pending, + refresh_document_access_cache_version_marker_ttls, + run_document_access_index_backfill_maintenance, +) +from functions_cosmos_stale_cleanup import ( + COSMOS_STALE_CACHE_CLEANUP_APPLY_SETTING, + COSMOS_STALE_CACHE_CLEANUP_BATCH_SIZE_SETTING, + COSMOS_STALE_CACHE_CLEANUP_MAX_BATCH_SIZE, + COSMOS_STALE_CACHE_CLEANUP_RUN_SETTING, + build_skipped_stale_cache_cleanup_result, + build_stale_cache_cleanup_status, + run_stale_cache_document_cleanup, +) +from functions_shared_cache import ensure_shared_cache_version_doc, get_shared_cache_metrics, get_shared_cache_version + + +APP_MAINTENANCE_STATE_DOC_ID = 'app_maintenance_state' +APP_MAINTENANCE_DOC_TYPE = 'app_maintenance_state' +APP_MAINTENANCE_LOCK_NAME = 'app_maintenance' +APP_MAINTENANCE_DEFAULT_INTERVAL_SECONDS = 3600 +APP_MAINTENANCE_DEFAULT_LEASE_SECONDS = 300 +APP_MAINTENANCE_DEFAULT_ACTIVE_INTERVAL_SECONDS = 30 + +CACHE_VERSION_DOCUMENTS = [ + { + 'id': 'app_settings_cache_version', + 'container': 'settings', + 'description': 'App settings cache invalidation version.', + }, + { + 'id': 'governance_cache_version', + 'container': 'governance_policies', + 'description': 'Governance and policy cache invalidation version.', + }, + { + 'id': 'custom_pages_cache_version', + 'container': 'settings', + 'description': 'Custom pages cache invalidation version.', + }, + { + 'id': 'chat_bootstrap_global_cache_version', + 'container': 'settings', + 'description': 'Chat bootstrap global cache invalidation version.', + }, + { + 'id': 'document_access_index_projection_version', + 'container': 'settings', + 'description': 'Document access index projection version for future read models.', + }, +] + + +def _utc_now(): + return datetime.now(timezone.utc) + + +def _utc_now_iso(): + return _utc_now().isoformat() + + +def _is_not_found_error(exc): + return getattr(exc, 'status_code', None) == 404 + + +def _get_cache_version_container(container_name): + if container_name == 'governance_policies': + return cosmos_governance_policies_container + return cosmos_settings_container + + +def _read_maintenance_state(): + try: + return cosmos_settings_container.read_item( + item=APP_MAINTENANCE_STATE_DOC_ID, + partition_key=APP_MAINTENANCE_STATE_DOC_ID, + ) + except Exception as ex: + if _is_not_found_error(ex): + return None + log_event( + '[AppMaintenance] Failed to read maintenance state.', + extra={'error': str(ex)}, + level=logging.ERROR, + exceptionTraceback=True, + ) + raise + + +def _write_maintenance_state(state): + state_body = dict(state or {}) + state_body.update({ + 'id': APP_MAINTENANCE_STATE_DOC_ID, + 'type': APP_MAINTENANCE_DOC_TYPE, + 'updated_at': _utc_now_iso(), + }) + cosmos_settings_container.upsert_item(state_body) + return state_body + + +def _record_started(run_id, triggered_by, requested_by): + existing = _read_maintenance_state() or {} + state = dict(existing) + state.update({ + 'current_run_id': run_id, + 'last_status': 'running', + 'last_triggered_by': triggered_by, + 'last_requested_by': requested_by, + 'last_started_at': _utc_now_iso(), + 'last_completed_at': None, + 'last_duration_ms': None, + 'last_error': None, + 'app_version': VERSION, + }) + return _write_maintenance_state(state) + + +def _record_completed(run_id, started_at, triggered_by, requested_by, steps): + duration_ms = int((time.perf_counter() - started_at) * 1000) + existing = _read_maintenance_state() or {} + last_status = 'succeeded' + if any(step.get('status') == 'failed' for step in list(steps or [])): + last_status = 'succeeded_with_warnings' + state = dict(existing) + state.update({ + 'current_run_id': None, + 'last_run_id': run_id, + 'last_status': last_status, + 'last_triggered_by': triggered_by, + 'last_requested_by': requested_by, + 'last_completed_at': _utc_now_iso(), + 'last_duration_ms': duration_ms, + 'last_error': None, + 'last_steps': steps, + 'app_version': VERSION, + }) + return _write_maintenance_state(state) + + +def _record_failed(run_id, started_at, triggered_by, requested_by, error): + duration_ms = int((time.perf_counter() - started_at) * 1000) + existing = _read_maintenance_state() or {} + state = dict(existing) + state.update({ + 'current_run_id': None, + 'last_run_id': run_id, + 'last_status': 'failed', + 'last_triggered_by': triggered_by, + 'last_requested_by': requested_by, + 'last_completed_at': _utc_now_iso(), + 'last_duration_ms': duration_ms, + 'last_error': str(error), + 'app_version': VERSION, + }) + return _write_maintenance_state(state) + + +def get_app_maintenance_settings(settings): + """Normalize maintenance settings from app settings.""" + settings = settings or {} + document_access_settings = get_document_access_index_settings(settings) + enabled = bool(settings.get('enable_app_maintenance', True)) + run_on_startup = bool(settings.get('enable_startup_app_maintenance', True)) + document_access_container_enabled = bool(document_access_settings.get('container_enabled')) + return { + 'enabled': enabled, + 'run_on_startup': run_on_startup, + 'check_interval_seconds': max( + int(settings.get( + 'app_maintenance_check_interval_seconds', + APP_MAINTENANCE_DEFAULT_INTERVAL_SECONDS, + ) or APP_MAINTENANCE_DEFAULT_INTERVAL_SECONDS), + 60, + ), + 'lease_seconds': max( + int(settings.get( + 'app_maintenance_job_lease_seconds', + APP_MAINTENANCE_DEFAULT_LEASE_SECONDS, + ) or APP_MAINTENANCE_DEFAULT_LEASE_SECONDS), + 60, + ), + 'apply_cosmos_indexing_policies': bool(settings.get(COSMOS_INDEXING_POLICY_APPLY_SETTING, False)), + 'run_document_access_index_backfill': document_access_container_enabled, + 'document_access_index_auto_maintenance': ( + enabled and run_on_startup and document_access_container_enabled + ), + 'document_access_index_active_interval_seconds': max( + int(settings.get( + DOCUMENT_ACCESS_ACTIVE_MAINTENANCE_INTERVAL_SETTING, + APP_MAINTENANCE_DEFAULT_ACTIVE_INTERVAL_SECONDS, + ) or APP_MAINTENANCE_DEFAULT_ACTIVE_INTERVAL_SECONDS), + 15, + ), + 'document_access_index_backfill_batch_size': max( + int(settings.get(DOCUMENT_ACCESS_BACKFILL_BATCH_SIZE_SETTING, 200) or 200), + 1, + ), + 'document_access_index_repair_batch_size': max( + int(settings.get(DOCUMENT_ACCESS_REPAIR_BATCH_SIZE_SETTING, 100) or 100), + 1, + ), + 'run_stale_cache_cleanup': bool(settings.get(COSMOS_STALE_CACHE_CLEANUP_RUN_SETTING, False)), + 'apply_stale_cache_cleanup': bool(settings.get(COSMOS_STALE_CACHE_CLEANUP_APPLY_SETTING, False)), + 'stale_cache_cleanup_batch_size': min( + max( + int(settings.get(COSMOS_STALE_CACHE_CLEANUP_BATCH_SIZE_SETTING, 100) or 100), + 1, + ), + COSMOS_STALE_CACHE_CLEANUP_MAX_BATCH_SIZE, + ), + } + + +def calculate_app_maintenance_sleep_seconds(maintenance_result, maintenance_settings): + """Return the next scheduler sleep interval, using a short loop while DAI work remains.""" + maintenance_settings = maintenance_settings or {} + default_sleep_seconds = max( + int(maintenance_settings.get('check_interval_seconds') or APP_MAINTENANCE_DEFAULT_INTERVAL_SECONDS), + 60, + ) + active_sleep_seconds = max( + int( + maintenance_settings.get('document_access_index_active_interval_seconds') + or APP_MAINTENANCE_DEFAULT_ACTIVE_INTERVAL_SECONDS + ), + 15, + ) + if is_document_access_index_maintenance_pending(_extract_document_access_index_step_status(maintenance_result)): + return active_sleep_seconds + return default_sleep_seconds + + +def _extract_document_access_index_step_status(maintenance_result): + if not isinstance(maintenance_result, dict): + return None + for step in maintenance_result.get('steps') or []: + if isinstance(step, dict) and step.get('name') == 'document_access_index_backfill': + results = step.get('results') if isinstance(step.get('results'), dict) else {} + return results.get('current_status') or results + return maintenance_result.get('document_access_index_backfill') + + +def _extract_last_step_results(state, step_name): + for step in reversed(list((state or {}).get('last_steps') or [])): + if isinstance(step, dict) and step.get('name') == step_name: + results = step.get('results') + return results if isinstance(results, dict) else None + return None + + +def initialize_cache_version_documents(): + """Ensure shared cache version documents exist in the settings container.""" + results = [] + for version_doc in CACHE_VERSION_DOCUMENTS: + result = ensure_shared_cache_version_doc( + version_doc['id'], + initial_version=0, + description=version_doc.get('description', ''), + container=_get_cache_version_container(version_doc.get('container')), + ) + result['container'] = version_doc.get('container', 'settings') + results.append(result) + return results + + +def get_cache_version_document_status(): + """Return current cache version document status for admin and diagnostics.""" + statuses = [] + for version_doc in CACHE_VERSION_DOCUMENTS: + version = get_shared_cache_version( + version_doc['id'], + default_version=0, + container=_get_cache_version_container(version_doc.get('container')), + ) + statuses.append({ + 'id': version_doc['id'], + 'container': version_doc.get('container', 'settings'), + 'description': version_doc.get('description', ''), + 'version': version, + }) + return statuses + + +def get_app_maintenance_status(settings=None): + """Return the latest app maintenance state and foundation document status.""" + state = _read_maintenance_state() or { + 'id': APP_MAINTENANCE_STATE_DOC_ID, + 'type': APP_MAINTENANCE_DOC_TYPE, + 'last_status': 'not_run', + 'app_version': VERSION, + } + maintenance_settings = get_app_maintenance_settings(settings or {}) + document_access_index_status = get_document_access_index_backfill_status(settings=settings or {}) + if isinstance(document_access_index_status.get('maintenance'), dict): + document_access_index_status['maintenance'].update({ + 'auto_maintenance_enabled': maintenance_settings.get('document_access_index_auto_maintenance'), + 'active_interval_seconds': maintenance_settings.get('document_access_index_active_interval_seconds'), + 'check_interval_seconds': maintenance_settings.get('check_interval_seconds'), + }) + return { + 'success': True, + 'state': state, + 'cache_version_documents': get_cache_version_document_status(), + 'shared_cache_metrics': get_shared_cache_metrics(), + 'conversation_cache': { + 'settings': get_conversation_cache_settings(settings or {}), + 'metrics': get_conversation_cache_metrics(), + }, + 'cosmos_indexing_policies': get_cosmos_indexing_policy_status(), + 'stale_cache_cleanup': build_stale_cache_cleanup_status( + _extract_last_step_results(state, 'stale_cache_document_cleanup') + ), + 'document_access_cache_version_ttl_hygiene': ( + _extract_last_step_results(state, 'document_access_cache_version_ttl_hygiene') or {} + ), + 'document_access_index_backfill': document_access_index_status, + 'settings': { + 'apply_cosmos_indexing_policies': maintenance_settings.get('apply_cosmos_indexing_policies'), + 'cosmos_indexing_policy_apply_setting': COSMOS_INDEXING_POLICY_APPLY_SETTING, + 'run_stale_cache_cleanup': maintenance_settings.get('run_stale_cache_cleanup'), + 'apply_stale_cache_cleanup': maintenance_settings.get('apply_stale_cache_cleanup'), + 'stale_cache_cleanup_batch_size': maintenance_settings.get('stale_cache_cleanup_batch_size'), + 'stale_cache_cleanup_run_setting': COSMOS_STALE_CACHE_CLEANUP_RUN_SETTING, + 'stale_cache_cleanup_apply_setting': COSMOS_STALE_CACHE_CLEANUP_APPLY_SETTING, + 'run_document_access_index_backfill': maintenance_settings.get('run_document_access_index_backfill'), + 'document_access_index_auto_maintenance': maintenance_settings.get('document_access_index_auto_maintenance'), + 'document_access_index_active_interval_seconds': maintenance_settings.get('document_access_index_active_interval_seconds'), + 'document_access_index_backfill_setting': DOCUMENT_ACCESS_BACKFILL_ENABLED_SETTING, + }, + 'app_version': VERSION, + } + + +def run_app_maintenance_once( + triggered_by='manual', + requested_by=None, + settings=None, + apply_indexing_policies=None, + run_document_access_backfill=None, + reset_document_access_backfill=False, + run_stale_cache_cleanup=None, + apply_stale_cache_cleanup=None, +): + """Run idempotent app maintenance tasks once and persist the outcome.""" + run_id = str(uuid.uuid4()) + started_at = time.perf_counter() + try: + maintenance_settings = get_app_maintenance_settings(settings or {}) + if apply_indexing_policies is None: + apply_indexing_policies = maintenance_settings.get('apply_cosmos_indexing_policies', False) + if run_document_access_backfill is None: + run_document_access_backfill = maintenance_settings.get('run_document_access_index_backfill', False) + if run_stale_cache_cleanup is None: + run_stale_cache_cleanup = maintenance_settings.get('run_stale_cache_cleanup', False) + if apply_stale_cache_cleanup is None: + apply_stale_cache_cleanup = maintenance_settings.get('apply_stale_cache_cleanup', False) + _record_started(run_id, triggered_by, requested_by) + log_event( + '[AppMaintenance] Maintenance run started.', + extra={'run_id': run_id, 'triggered_by': triggered_by, 'requested_by': requested_by}, + level=logging.INFO, + ) + cache_version_results = initialize_cache_version_documents() + indexing_policy_results = run_cosmos_indexing_policy_maintenance( + apply_changes=bool(apply_indexing_policies), + ) + if run_stale_cache_cleanup: + stale_cache_cleanup_results = run_stale_cache_document_cleanup( + apply_changes=bool(apply_stale_cache_cleanup), + batch_size=maintenance_settings.get('stale_cache_cleanup_batch_size'), + ) + else: + stale_cache_cleanup_results = build_skipped_stale_cache_cleanup_result() + document_access_cache_version_ttl_results = refresh_document_access_cache_version_marker_ttls( + settings=settings, + ) + document_access_backfill_results = run_document_access_index_backfill_maintenance( + settings=settings, + run_backfill=bool(run_document_access_backfill), + reset=bool(reset_document_access_backfill), + batch_size=maintenance_settings.get('document_access_index_backfill_batch_size'), + repair_batch_size=maintenance_settings.get('document_access_index_repair_batch_size'), + ) + steps = [ + { + 'name': 'initialize_cache_version_documents', + 'status': 'succeeded', + 'results': cache_version_results, + }, + { + 'name': 'cosmos_indexing_policy_maintenance', + 'status': 'succeeded' if indexing_policy_results.get('success') else 'failed', + 'apply_requested': bool(apply_indexing_policies), + 'results': indexing_policy_results, + }, + { + 'name': 'stale_cache_document_cleanup', + 'status': 'succeeded' if stale_cache_cleanup_results.get('success') else 'failed', + 'run_requested': bool(run_stale_cache_cleanup), + 'apply_requested': bool(apply_stale_cache_cleanup), + 'results': stale_cache_cleanup_results, + }, + { + 'name': 'document_access_cache_version_ttl_hygiene', + 'status': 'succeeded' if document_access_cache_version_ttl_results.get('success') else 'failed', + 'results': document_access_cache_version_ttl_results, + }, + { + 'name': 'document_access_index_backfill', + 'status': 'succeeded' if document_access_backfill_results.get('success') else 'failed', + 'run_requested': bool(run_document_access_backfill), + 'reset_requested': bool(reset_document_access_backfill), + 'results': document_access_backfill_results, + }, + ] + overall_success = all(step.get('status') == 'succeeded' for step in steps) + state = _record_completed(run_id, started_at, triggered_by, requested_by, steps) + log_event( + '[AppMaintenance] Maintenance run completed.', + extra={ + 'run_id': run_id, + 'triggered_by': triggered_by, + 'duration_ms': state.get('last_duration_ms'), + 'success': overall_success, + }, + level=logging.INFO, + ) + return { + 'success': overall_success, + 'run_id': run_id, + 'state': state, + 'steps': steps, + 'error': None if overall_success else 'One or more maintenance steps failed.', + } + except Exception as ex: + try: + state = _record_failed(run_id, started_at, triggered_by, requested_by, ex) + except Exception as state_error: + state = { + 'last_status': 'failed', + 'last_error': 'Failed to record maintenance state.', + } + log_event( + '[AppMaintenance] Failed to record maintenance failure state.', + extra={ + 'run_id': run_id, + 'triggered_by': triggered_by, + 'error': str(state_error), + }, + level=logging.ERROR, + exceptionTraceback=True, + ) + log_event( + '[AppMaintenance] Maintenance run failed.', + extra={'run_id': run_id, 'triggered_by': triggered_by, 'error': str(ex)}, + level=logging.ERROR, + exceptionTraceback=True, + ) + return { + 'success': False, + 'run_id': run_id, + 'state': state, + 'error': str(ex), + } diff --git a/application/single_app/functions_chat_bootstrap_cache.py b/application/single_app/functions_chat_bootstrap_cache.py new file mode 100644 index 00000000..e852f0ba --- /dev/null +++ b/application/single_app/functions_chat_bootstrap_cache.py @@ -0,0 +1,221 @@ +# functions_chat_bootstrap_cache.py +"""Versioned chat bootstrap cache helpers for low-churn catalog payloads.""" + +import copy +import hashlib +import json +import logging +from typing import Any, Dict, Iterable, Optional + +import app_settings_cache +from functions_appinsights import log_event +from functions_shared_cache import ( + bump_shared_cache_version, + get_shared_cache_entry, + get_shared_cache_version, + set_shared_cache_entry, +) + + +CHAT_BOOTSTRAP_CACHE_NAMESPACE = "chat_bootstrap" +CHAT_BOOTSTRAP_GLOBAL_VERSION_DOC_ID = "chat_bootstrap_global_cache_version" +CHAT_BOOTSTRAP_USER_VERSION_DOC_PREFIX = "chat_bootstrap_user_cache_version:" +CHAT_BOOTSTRAP_DEFAULT_TTL_SECONDS = 300 + + +def _stable_hash(payload: Any) -> str: + serialized = json.dumps(payload, sort_keys=True, default=str, separators=(",", ":")) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() + + +def _normalize_id_list(items: Iterable[Dict[str, Any]]) -> list[str]: + normalized = [] + for item in items or []: + if not isinstance(item, dict): + continue + item_id = str(item.get("id") or "").strip() + if item_id: + normalized.append(item_id) + return sorted(set(normalized)) + + +def _get_redis_client(): + try: + return app_settings_cache.get_app_cache_redis_client() + except Exception as ex: + log_event( + "[ChatBootstrapCache] Redis client lookup failed; using Cosmos cache fallback.", + extra={"error": str(ex)}, + level=logging.WARNING, + ) + return None + + +def _get_app_settings_version() -> int: + getter = getattr(app_settings_cache, "get_app_settings_cache_version", None) + if not callable(getter): + return 0 + try: + return int(getter() or 0) + except Exception as ex: + log_event( + "[ChatBootstrapCache] App settings version lookup failed.", + extra={"error": str(ex)}, + level=logging.WARNING, + ) + return 0 + + +def _get_governance_version() -> int: + getter = getattr(app_settings_cache, "get_governance_cache_version", None) + if not callable(getter): + return 0 + try: + return int(getter() or 0) + except Exception as ex: + log_event( + "[ChatBootstrapCache] Governance cache version lookup failed.", + extra={"error": str(ex)}, + level=logging.WARNING, + ) + return 0 + + +def _get_user_version_doc_id(user_id: str) -> str: + normalized_user_id = str(user_id or "").strip() + if not normalized_user_id: + raise ValueError("user_id is required for chat bootstrap user cache versioning.") + return f"{CHAT_BOOTSTRAP_USER_VERSION_DOC_PREFIX}{normalized_user_id}" + + +def get_chat_bootstrap_versions(user_id: str) -> Dict[str, int]: + """Return shared versions that make chat bootstrap cache keys cross-worker safe.""" + return { + "global": get_shared_cache_version(CHAT_BOOTSTRAP_GLOBAL_VERSION_DOC_ID, default_version=0), + "user": get_shared_cache_version(_get_user_version_doc_id(user_id), default_version=0), + "settings": _get_app_settings_version(), + "governance": _get_governance_version(), + } + + +def build_chat_bootstrap_cache_key( + user_id: str, + *, + settings: Dict[str, Any], + user_settings_dict: Dict[str, Any], + user_groups_raw: Iterable[Dict[str, Any]], + user_visible_public_workspaces: Iterable[Dict[str, Any]], +) -> str: + """Build a stable user-scoped cache key for chat bootstrap catalog fragments.""" + versions = get_chat_bootstrap_versions(user_id) + relevant_settings = { + key: settings.get(key) + for key in ( + "allow_user_agents", + "enable_semantic_kernel", + "enable_group_workspaces", + "allow_group_agents", + "enable_multi_model_endpoints", + "allow_user_custom_endpoints", + "allow_group_custom_endpoints", + "enable_public_workspaces", + "enable_user_workspace", + "allow_user_plugins", + "allow_group_plugins", + ) + } + fingerprint = _stable_hash({ + "user_id": user_id, + "versions": versions, + "settings": relevant_settings, + "personal_model_endpoints": user_settings_dict.get("personal_model_endpoints", []), + "group_ids": _normalize_id_list(user_groups_raw), + "public_workspace_ids": _normalize_id_list(user_visible_public_workspaces), + }) + return f"user:{user_id}:{fingerprint}" + + +def get_cached_chat_bootstrap_payload(cache_key: str) -> Optional[Dict[str, Any]]: + """Return a cached chat bootstrap payload or None on miss/failure.""" + cached = get_shared_cache_entry( + CHAT_BOOTSTRAP_CACHE_NAMESPACE, + cache_key, + redis_client=_get_redis_client(), + allow_cosmos_fallback=False, + ) + if not isinstance(cached, dict): + log_event( + "[ChatBootstrapCache] Chat bootstrap cache miss.", + extra={"cache_key_hash": _stable_hash(cache_key)[:16]}, + level=logging.INFO, + debug_only=True, + ) + return None + + log_event( + "[ChatBootstrapCache] Chat bootstrap cache hit.", + extra={"cache_key_hash": _stable_hash(cache_key)[:16]}, + level=logging.INFO, + debug_only=True, + ) + return copy.deepcopy(cached) + + +def set_cached_chat_bootstrap_payload(cache_key: str, payload: Dict[str, Any], ttl_seconds: int = None) -> bool: + """Persist a chat bootstrap payload without failing the caller on cache errors.""" + return set_shared_cache_entry( + CHAT_BOOTSTRAP_CACHE_NAMESPACE, + cache_key, + copy.deepcopy(payload or {}), + ttl_seconds=ttl_seconds or CHAT_BOOTSTRAP_DEFAULT_TTL_SECONDS, + redis_client=_get_redis_client(), + allow_cosmos_fallback=False, + ) + + +def bump_chat_bootstrap_global_cache_version(reason: str = "global_bootstrap_change") -> Optional[int]: + """Invalidate all chat bootstrap payloads by bumping the global version.""" + try: + version = bump_shared_cache_version( + CHAT_BOOTSTRAP_GLOBAL_VERSION_DOC_ID, + description="Global chat bootstrap catalog cache version.", + ) + log_event( + "[ChatBootstrapCache] Global chat bootstrap cache invalidated.", + extra={"reason": reason, "version": version}, + level=logging.INFO, + debug_only=True, + ) + return version + except Exception as ex: + log_event( + "[ChatBootstrapCache] Failed to invalidate global chat bootstrap cache.", + extra={"reason": reason, "error": str(ex)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None + + +def bump_chat_bootstrap_user_cache_version(user_id: str, reason: str = "user_bootstrap_change") -> Optional[int]: + """Invalidate one user's chat bootstrap payloads by bumping the user version.""" + try: + version = bump_shared_cache_version( + _get_user_version_doc_id(user_id), + description="User chat bootstrap catalog cache version.", + ) + log_event( + "[ChatBootstrapCache] User chat bootstrap cache invalidated.", + extra={"reason": reason, "user_id": user_id, "version": version}, + level=logging.INFO, + debug_only=True, + ) + return version + except Exception as ex: + log_event( + "[ChatBootstrapCache] Failed to invalidate user chat bootstrap cache.", + extra={"reason": reason, "user_id": user_id, "error": str(ex)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None diff --git a/application/single_app/functions_collaboration.py b/application/single_app/functions_collaboration.py index 2e08ae11..a3fcd46f 100644 --- a/application/single_app/functions_collaboration.py +++ b/application/single_app/functions_collaboration.py @@ -36,6 +36,7 @@ ) from functions_activity_logging import log_chat_activity from functions_appinsights import log_event +from functions_conversation_cache import bump_conversation_cache_version, invalidate_conversation_cache_for_item from functions_documents import sync_chat_upload_workspace_document_sharing_for_collaboration from functions_group import ( assert_group_role, @@ -1214,6 +1215,7 @@ def create_personal_collaboration_conversation_record(title, creator_user, invit cosmos_collaboration_user_state_container.upsert_item(state_doc) user_states.append(state_doc) + invalidate_conversation_cache_for_item(conversation_doc, reason="collaboration_created") log_event( '[Collaboration] Created personal collaborative conversation', extra={ @@ -1255,6 +1257,7 @@ def create_group_collaboration_conversation_record(title, creator_user, group_do ) user_states.append(state_doc) + invalidate_conversation_cache_for_item(conversation_doc, reason="collaboration_created") log_event( '[Collaboration] Created group collaborative conversation', extra={ @@ -1539,6 +1542,7 @@ def record_personal_invite_response(conversation_id, user_id, action): cosmos_collaboration_conversations_container.upsert_item(conversation_doc) cosmos_collaboration_user_state_container.upsert_item(user_state) + invalidate_conversation_cache_for_item(conversation_doc, reason="collaboration_invite_response") if membership_status == MEMBERSHIP_STATUS_ACCEPTED and is_personal_collaboration_conversation(conversation_doc): sync_chat_upload_workspace_document_sharing_for_collaboration(conversation_doc) return conversation_doc, user_state, participant_record @@ -1607,6 +1611,7 @@ def invite_personal_collaboration_participants(conversation_id, owner_user_id, p cosmos_collaboration_user_state_container.upsert_item(state_doc) created_state_docs.append(state_doc) + invalidate_conversation_cache_for_item(conversation_doc, reason="collaboration_participants_invited") return conversation_doc, created_state_docs @@ -1667,6 +1672,8 @@ def remove_personal_collaboration_member(conversation_id, owner_user_id, member_ if is_personal_collaboration_conversation(conversation_doc): sync_chat_upload_workspace_document_sharing_for_collaboration(conversation_doc) + bump_conversation_cache_version(member_user_id, reason="collaboration_member_removed") + invalidate_conversation_cache_for_item(conversation_doc, reason="collaboration_member_removed") return conversation_doc, removed_participant @@ -1745,6 +1752,7 @@ def _save_collaboration_message_doc(conversation_doc, message_doc): refresh_personal_participant_indexes(conversation_doc) cosmos_collaboration_conversations_container.upsert_item(conversation_doc) + invalidate_conversation_cache_for_item(conversation_doc, reason="collaboration_message_saved") if sender_user_id and sender_user_id != 'assistant' and str(message_doc.get('role') or '').strip().lower() == 'user': try: @@ -1807,6 +1815,7 @@ def sync_collaboration_conversation_metadata_from_source(conversation_doc, sourc if updated: cosmos_collaboration_conversations_container.upsert_item(conversation_doc) + invalidate_conversation_cache_for_item(conversation_doc, reason="collaboration_metadata_synced") return conversation_doc, updated @@ -1875,6 +1884,7 @@ def ensure_collaboration_source_conversation(conversation_doc, current_user): if source_updated: source_conversation_doc['last_updated'] = timestamp cosmos_conversations_container.upsert_item(source_conversation_doc) + invalidate_conversation_cache_for_item(source_conversation_doc, reason="collaboration_source_synced") if str((conversation_doc or {}).get('source_conversation_id') or '').strip() != source_conversation_id: conversation_doc['source_conversation_id'] = source_conversation_id @@ -1950,6 +1960,7 @@ def _refresh_collaboration_conversation_message_summary(conversation_doc): conversation_doc['updated_at'] = utc_now_iso() cosmos_collaboration_conversations_container.upsert_item(conversation_doc) + invalidate_conversation_cache_for_item(conversation_doc, reason="collaboration_message_summary_refreshed") return conversation_doc @@ -2015,6 +2026,7 @@ def update_personal_collaboration_title(conversation_id, current_user_id, new_ti conversation_doc['title'] = normalized_title conversation_doc['updated_at'] = utc_now_iso() cosmos_collaboration_conversations_container.upsert_item(conversation_doc) + invalidate_conversation_cache_for_item(conversation_doc, reason="collaboration_title_updated") return conversation_doc @@ -2050,6 +2062,7 @@ def toggle_personal_collaboration_pin(conversation_id, current_user_id): user_state['is_pinned'] = not bool(user_state.get('is_pinned', False)) user_state['updated_at'] = utc_now_iso() cosmos_collaboration_user_state_container.upsert_item(user_state) + bump_conversation_cache_version(current_user_id, reason="collaboration_pin_toggled") return conversation_doc, user_state @@ -2085,6 +2098,7 @@ def toggle_personal_collaboration_hide(conversation_id, current_user_id): user_state['is_hidden'] = not bool(user_state.get('is_hidden', False)) user_state['updated_at'] = utc_now_iso() cosmos_collaboration_user_state_container.upsert_item(user_state) + bump_conversation_cache_version(current_user_id, reason="collaboration_hide_toggled") return conversation_doc, user_state @@ -2133,6 +2147,7 @@ def update_personal_collaboration_member_role(conversation_id, current_user_id, user_state['updated_at'] = timestamp cosmos_collaboration_user_state_container.upsert_item(user_state) + invalidate_conversation_cache_for_item(conversation_doc, reason="collaboration_member_role_updated") return conversation_doc, participant @@ -2216,6 +2231,8 @@ def leave_personal_collaboration_conversation(conversation_id, current_user_id, if is_personal_collaboration_conversation(conversation_doc): sync_chat_upload_workspace_document_sharing_for_collaboration(conversation_doc) + bump_conversation_cache_version(current_user_id, reason="collaboration_left") + invalidate_conversation_cache_for_item(conversation_doc, reason="collaboration_left") return conversation_doc, participant, promoted_participant @@ -2348,6 +2365,7 @@ def delete_personal_collaboration_conversation(conversation_id, current_user_id) item=conversation_id, partition_key=conversation_id, ) + invalidate_conversation_cache_for_item(conversation_doc, reason="collaboration_deleted") return conversation_doc diff --git a/application/single_app/functions_conversation_cache.py b/application/single_app/functions_conversation_cache.py new file mode 100644 index 00000000..dc539b43 --- /dev/null +++ b/application/single_app/functions_conversation_cache.py @@ -0,0 +1,579 @@ +# functions_conversation_cache.py +"""Versioned conversation list/search cache helpers.""" + +import copy +import hashlib +import json +import logging +import threading +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Iterable, Optional + +import app_settings_cache +from functions_appinsights import log_event +from functions_group import find_group_by_id +from functions_shared_cache import ( + get_shared_cache_entry, + set_shared_cache_entry, +) + + +CONVERSATION_CACHE_NAMESPACE = "conversation_cache" +CONVERSATION_CACHE_VERSION_DOC_PREFIX = "conversation_cache_version:" +CONVERSATION_CACHE_ENABLED_SETTING = "enable_conversation_cache" +CONVERSATION_CACHE_TTL_SECONDS_SETTING = "conversation_cache_ttl_seconds" +CONVERSATION_CACHE_DEFAULT_TTL_SECONDS = 120 +CONVERSATION_CACHE_METRIC_WINDOWS_MINUTES = (5, 15, 60) +CONVERSATION_CACHE_METRIC_RETENTION_MINUTES = 60 +CONVERSATION_CACHE_METRIC_MAX_SAMPLES = 2000 + +_conversation_cache_metric_samples = [] +_conversation_cache_metric_lock = threading.Lock() + + +def _normalize_bool(value: Any, default: bool = True) -> bool: + if isinstance(value, bool): + return value + if value is None: + return default + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + return bool(value) + + +def get_conversation_cache_ttl_seconds(settings: Optional[Dict[str, Any]] = None) -> int: + """Return a safe positive TTL for conversation cache entries.""" + raw_ttl_seconds = CONVERSATION_CACHE_DEFAULT_TTL_SECONDS + if isinstance(settings, dict): + raw_ttl_seconds = settings.get( + CONVERSATION_CACHE_TTL_SECONDS_SETTING, + CONVERSATION_CACHE_DEFAULT_TTL_SECONDS, + ) + try: + return max(int(raw_ttl_seconds), 0) + except (TypeError, ValueError): + log_event( + "[ConversationCache] Invalid conversation cache TTL; using default.", + extra={"ttl_seconds": raw_ttl_seconds}, + level=logging.WARNING, + ) + return CONVERSATION_CACHE_DEFAULT_TTL_SECONDS + + +def get_conversation_cache_settings(settings: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """Return normalized conversation cache settings with Redis remaining optional.""" + settings = settings if isinstance(settings, dict) else {} + return { + "enabled": _normalize_bool(settings.get(CONVERSATION_CACHE_ENABLED_SETTING), default=True), + "ttl_seconds": get_conversation_cache_ttl_seconds(settings), + } + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _parse_utc_datetime(value: Any) -> Optional[datetime]: + if isinstance(value, datetime): + parsed = value + else: + try: + parsed = datetime.fromisoformat(str(value or "")) + except (TypeError, ValueError): + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def _metric_percent(numerator: int, denominator: int) -> Optional[float]: + if denominator <= 0: + return None + return round((float(numerator) / float(denominator)) * 100, 2) + + +def _safe_int(value: Any) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _increment_count(counts: Dict[str, int], key: Any) -> None: + normalized_key = str(key or "unknown").strip().lower() or "unknown" + counts[normalized_key] = int(counts.get(normalized_key) or 0) + 1 + + +def _get_cache_operation_from_key(cache_key: Any) -> str: + normalized_key = str(cache_key or "").strip() + operation = normalized_key.split(":", 1)[0].strip().lower() + return operation or "conversation_cache" + + +def _empty_conversation_cache_metric_window(window_minutes: int) -> Dict[str, Any]: + return { + "window_minutes": window_minutes, + "sample_count": 0, + "hit_count": 0, + "miss_count": 0, + "bypass_count": 0, + "write_count": 0, + "invalidation_count": 0, + "error_count": 0, + "hit_rate_percent": None, + "operation_counts": {}, + "event_counts": {}, + "reason_counts": {}, + "first_checked_at": None, + "last_checked_at": None, + } + + +def _prune_conversation_cache_metric_samples(samples: Iterable[Dict[str, Any]], now: datetime) -> list: + cutoff = now - timedelta(minutes=CONVERSATION_CACHE_METRIC_RETENTION_MINUTES) + pruned_samples = [] + for sample in samples or []: + if not isinstance(sample, dict): + continue + checked_at = _parse_utc_datetime(sample.get("checked_at")) + if not checked_at or checked_at < cutoff: + continue + pruned_samples.append(copy.deepcopy(sample)) + + pruned_samples.sort(key=lambda sample: sample.get("checked_at") or "") + return pruned_samples[-CONVERSATION_CACHE_METRIC_MAX_SAMPLES:] + + +def _conversation_cache_metric_matches(sample: Dict[str, Any], event_types: set) -> bool: + return str(sample.get("event_type") or "").strip().lower() in event_types + + +def _build_conversation_cache_metric_window( + samples: Iterable[Dict[str, Any]], + now: datetime, + window_minutes: int, +) -> Dict[str, Any]: + cutoff = now - timedelta(minutes=window_minutes) + window_samples = [ + sample + for sample in samples or [] + if (_parse_utc_datetime(sample.get("checked_at")) or datetime.min.replace(tzinfo=timezone.utc)) >= cutoff + ] + if not window_samples: + return _empty_conversation_cache_metric_window(window_minutes) + + operation_counts = {} + event_counts = {} + reason_counts = {} + for sample in window_samples: + _increment_count(operation_counts, sample.get("operation")) + _increment_count(event_counts, sample.get("event_type")) + _increment_count(reason_counts, sample.get("reason")) + + hit_count = sum(1 for sample in window_samples if _conversation_cache_metric_matches(sample, {"hit"})) + miss_count = sum(1 for sample in window_samples if _conversation_cache_metric_matches(sample, {"miss"})) + bypass_count = sum(1 for sample in window_samples if _conversation_cache_metric_matches(sample, {"bypass"})) + write_count = sum(1 for sample in window_samples if _conversation_cache_metric_matches(sample, {"write"})) + invalidation_count = sum( + 1 for sample in window_samples if _conversation_cache_metric_matches(sample, {"invalidate"}) + ) + error_count = sum( + 1 + for sample in window_samples + if _conversation_cache_metric_matches(sample, {"read_failed", "write_failed", "invalidate_failed"}) + ) + return { + "window_minutes": window_minutes, + "sample_count": len(window_samples), + "hit_count": hit_count, + "miss_count": miss_count, + "bypass_count": bypass_count, + "write_count": write_count, + "invalidation_count": invalidation_count, + "error_count": error_count, + "hit_rate_percent": _metric_percent(hit_count, hit_count + miss_count), + "operation_counts": operation_counts, + "event_counts": event_counts, + "reason_counts": reason_counts, + "first_checked_at": window_samples[0].get("checked_at"), + "last_checked_at": window_samples[-1].get("checked_at"), + } + + +def _build_conversation_cache_metrics(samples: Iterable[Dict[str, Any]], now: datetime) -> Dict[str, Any]: + error_samples = [ + sample + for sample in samples or [] + if str(sample.get("event_type") or "").strip().lower().endswith("_failed") + ] + invalidation_samples = [ + sample + for sample in samples or [] + if str(sample.get("event_type") or "").strip().lower() == "invalidate" + ] + return { + "updated_at": now.isoformat(), + "retention_minutes": CONVERSATION_CACHE_METRIC_RETENTION_MINUTES, + "sample_limit": CONVERSATION_CACHE_METRIC_MAX_SAMPLES, + "sample_count": len(samples or []), + "last_event": copy.deepcopy(samples[-1]) if samples else None, + "last_error": copy.deepcopy(error_samples[-1]) if error_samples else None, + "last_invalidation": copy.deepcopy(invalidation_samples[-1]) if invalidation_samples else None, + "windows": { + f"{window_minutes}m": _build_conversation_cache_metric_window(samples, now, window_minutes) + for window_minutes in CONVERSATION_CACHE_METRIC_WINDOWS_MINUTES + }, + } + + +def _record_conversation_cache_metric( + event_type: str, + operation: str = None, + reason: str = None, + ttl_seconds: int = None, + error: Exception = None, +) -> Dict[str, Any]: + sample = { + "checked_at": _utc_now_iso(), + "event_type": str(event_type or "unknown").strip().lower() or "unknown", + "operation": str(operation or "conversation_cache").strip().lower() or "conversation_cache", + "reason": str(reason or "").strip().lower() or None, + "ttl_seconds": _safe_int(ttl_seconds), + } + if error: + sample["error_type"] = type(error).__name__ + + with _conversation_cache_metric_lock: + now = datetime.now(timezone.utc) + samples = _prune_conversation_cache_metric_samples(_conversation_cache_metric_samples, now) + samples.append(sample) + del samples[:-CONVERSATION_CACHE_METRIC_MAX_SAMPLES] + _conversation_cache_metric_samples[:] = samples + return sample + + +def get_conversation_cache_metrics() -> Dict[str, Any]: + """Return lightweight in-process rolling metrics for conversation cache reads/writes.""" + with _conversation_cache_metric_lock: + samples = copy.deepcopy(_conversation_cache_metric_samples) + now = datetime.now(timezone.utc) + samples = _prune_conversation_cache_metric_samples(samples, now) + return _build_conversation_cache_metrics(samples, now) + + +def reset_conversation_cache_metrics() -> None: + """Reset lightweight conversation cache counters for deterministic functional tests.""" + with _conversation_cache_metric_lock: + _conversation_cache_metric_samples[:] = [] + + +def _stable_hash(payload: Any) -> str: + serialized = json.dumps(payload, sort_keys=True, default=str, separators=(",", ":")) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() + + +def _get_redis_client(): + try: + return app_settings_cache.get_app_cache_redis_client() + except Exception as ex: + log_event( + "[ConversationCache] Redis client lookup failed; bypassing conversation cache.", + extra={"error": str(ex)}, + level=logging.WARNING, + ) + return None + + +def _get_version_doc_id(user_id: str) -> str: + normalized_user_id = str(user_id or "").strip() + if not normalized_user_id: + raise ValueError("user_id is required for conversation cache versioning.") + return f"{CONVERSATION_CACHE_VERSION_DOC_PREFIX}{normalized_user_id}" + + +def get_conversation_cache_version(user_id: str) -> Optional[int]: + """Return the current Redis-backed user-scoped conversation cache version.""" + redis_client = _get_redis_client() + if redis_client is None: + _record_conversation_cache_metric("bypass", operation="version", reason="redis_unavailable") + return None + try: + version = redis_client.get(_get_version_doc_id(user_id)) + if version is None: + return 0 + if isinstance(version, bytes): + version = version.decode("utf-8") + return int(version) + except Exception as ex: + _record_conversation_cache_metric("read_failed", operation="version", reason="version_read_failed", error=ex) + log_event( + "[ConversationCache] Failed to read conversation cache version.", + extra={"user_id": user_id, "error": str(ex)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None + + +def bump_conversation_cache_version(user_id: str, reason: str = "conversation_changed") -> Optional[int]: + """Invalidate a user's conversation list/search cache without failing callers.""" + if not user_id: + return None + redis_client = _get_redis_client() + if redis_client is None: + _record_conversation_cache_metric("bypass", operation="version", reason="redis_unavailable") + return None + try: + version = int(redis_client.incr(_get_version_doc_id(user_id)) or 0) + log_event( + "[ConversationCache] Conversation cache invalidated.", + extra={"reason": reason, "user_id": user_id, "version": version}, + level=logging.INFO, + debug_only=True, + ) + _record_conversation_cache_metric("invalidate", operation="version", reason=reason) + return version + except Exception as ex: + _record_conversation_cache_metric("invalidate_failed", operation="version", reason=reason, error=ex) + log_event( + "[ConversationCache] Failed to invalidate conversation cache.", + extra={"reason": reason, "user_id": user_id, "error": str(ex)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None + + +def _participant_user_ids_from_tags(tags: Iterable[Dict[str, Any]]) -> set[str]: + user_ids = set() + for tag in tags or []: + if not isinstance(tag, dict): + continue + if tag.get("category") != "participant": + continue + user_id = str(tag.get("user_id") or "").strip() + if user_id: + user_ids.add(user_id) + return user_ids + + +def _add_user_id(user_ids: set[str], user_id: Any) -> None: + normalized_user_id = str(user_id or "").strip() + if normalized_user_id: + user_ids.add(normalized_user_id) + + +def _add_user_ids(user_ids: set[str], candidate_user_ids: Iterable[Any]) -> None: + for user_id in candidate_user_ids or []: + _add_user_id(user_ids, user_id) + + +def _add_participant_user_ids(user_ids: set[str], participants: Iterable[Dict[str, Any]]) -> None: + for participant in participants or []: + if not isinstance(participant, dict): + continue + if str(participant.get("status") or "").strip().lower() == "removed": + continue + _add_user_id(user_ids, participant.get("user_id")) + + +def _add_group_member_user_ids(user_ids: set[str], group_id: Any) -> None: + normalized_group_id = str(group_id or "").strip() + if not normalized_group_id: + return + try: + group_doc = find_group_by_id(normalized_group_id) + except Exception as ex: + log_event( + "[ConversationCache] Failed to load group members for cache invalidation.", + extra={"group_id": normalized_group_id, "error": str(ex)}, + level=logging.WARNING, + exceptionTraceback=True, + debug_only=True, + ) + return + if not isinstance(group_doc, dict): + return + + owner_doc = group_doc.get("owner") if isinstance(group_doc.get("owner"), dict) else {} + _add_user_id(user_ids, owner_doc.get("id") or owner_doc.get("userId")) + for collection_name in ("admins", "documentManagers", "users"): + for member in group_doc.get(collection_name, []) or []: + if not isinstance(member, dict): + continue + _add_user_id(user_ids, member.get("userId") or member.get("id")) + + +def get_conversation_cache_user_ids(conversation_item: Dict[str, Any]) -> set[str]: + """Return users whose conversation list/search cache can include this item.""" + user_ids = set() + if not isinstance(conversation_item, dict): + return user_ids + + _add_user_id(user_ids, conversation_item.get("user_id")) + _add_user_id(user_ids, conversation_item.get("created_by_user_id")) + user_ids.update(_participant_user_ids_from_tags(conversation_item.get("tags") or [])) + _add_participant_user_ids(user_ids, conversation_item.get("participants") or []) + for field_name in ("accepted_participant_ids", "pending_participant_ids", "owner_user_ids", "admin_user_ids"): + _add_user_ids(user_ids, conversation_item.get(field_name) or []) + + scope = conversation_item.get("scope") if isinstance(conversation_item.get("scope"), dict) else {} + _add_group_member_user_ids(user_ids, scope.get("group_id") or conversation_item.get("group_id")) + return user_ids + + +def invalidate_conversation_cache_for_item(conversation_item: Dict[str, Any], reason: str = "conversation_changed") -> None: + """Invalidate cache versions for the owner and known participants on a conversation item.""" + for user_id in get_conversation_cache_user_ids(conversation_item): + bump_conversation_cache_version(user_id, reason=reason) + + +def build_conversation_cache_key(user_id: str, operation: str, parameters: Dict[str, Any] = None) -> Optional[str]: + """Build a stable user-scoped cache key for conversation list/search payloads.""" + version = get_conversation_cache_version(user_id) + if version is None: + return None + fingerprint = _stable_hash({ + "user_id": user_id, + "operation": operation, + "version": version, + "parameters": parameters or {}, + }) + return f"{operation}:{user_id}:{fingerprint}" + + +def get_cached_conversation_payload(cache_key: str, settings: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]: + """Return a cached conversation payload or None on cache miss/failure.""" + cache_settings = get_conversation_cache_settings(settings) + operation = _get_cache_operation_from_key(cache_key) + if not cache_settings.get("enabled"): + _record_conversation_cache_metric("bypass", operation=operation, reason="disabled") + log_event( + "[ConversationCache] Conversation cache read bypassed because cache is disabled.", + extra={"cache_key_hash": _stable_hash(cache_key)[:16] if cache_key else ""}, + level=logging.INFO, + debug_only=True, + ) + return None + + redis_client = _get_redis_client() + if redis_client is None: + _record_conversation_cache_metric("bypass", operation=operation, reason="redis_unavailable") + return None + + try: + cached = get_shared_cache_entry( + CONVERSATION_CACHE_NAMESPACE, + cache_key, + redis_client=redis_client, + allow_cosmos_fallback=False, + ) + except Exception as ex: + _record_conversation_cache_metric("read_failed", operation=operation, error=ex) + log_event( + "[ConversationCache] Failed to read conversation cache payload.", + extra={"cache_key_hash": _stable_hash(cache_key)[:16], "error": str(ex)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None + + if not isinstance(cached, dict): + _record_conversation_cache_metric("miss", operation=operation) + log_event( + "[ConversationCache] Conversation cache miss.", + extra={"cache_key_hash": _stable_hash(cache_key)[:16]}, + level=logging.INFO, + debug_only=True, + ) + return None + + _record_conversation_cache_metric("hit", operation=operation) + log_event( + "[ConversationCache] Conversation cache hit.", + extra={"cache_key_hash": _stable_hash(cache_key)[:16]}, + level=logging.INFO, + debug_only=True, + ) + return copy.deepcopy(cached) + + +def set_cached_conversation_payload( + cache_key: str, + payload: Dict[str, Any], + ttl_seconds: int = None, + settings: Optional[Dict[str, Any]] = None, +) -> bool: + """Persist a conversation payload without failing the caller on cache errors.""" + cache_settings = get_conversation_cache_settings(settings) + operation = _get_cache_operation_from_key(cache_key) + if not cache_settings.get("enabled"): + _record_conversation_cache_metric("bypass", operation=operation, reason="disabled") + log_event( + "[ConversationCache] Conversation cache write bypassed because cache is disabled.", + extra={"cache_key_hash": _stable_hash(cache_key)[:16] if cache_key else ""}, + level=logging.INFO, + debug_only=True, + ) + return False + + safe_ttl_seconds = ( + get_conversation_cache_ttl_seconds({CONVERSATION_CACHE_TTL_SECONDS_SETTING: ttl_seconds}) + if ttl_seconds is not None + else cache_settings.get("ttl_seconds", CONVERSATION_CACHE_DEFAULT_TTL_SECONDS) + ) + if safe_ttl_seconds <= 0: + _record_conversation_cache_metric( + "bypass", + operation=operation, + reason="ttl_disabled", + ttl_seconds=safe_ttl_seconds, + ) + log_event( + "[ConversationCache] Conversation cache write bypassed because TTL is disabled.", + extra={"cache_key_hash": _stable_hash(cache_key)[:16] if cache_key else ""}, + level=logging.INFO, + debug_only=True, + ) + return False + + redis_client = _get_redis_client() + if redis_client is None: + _record_conversation_cache_metric("bypass", operation=operation, reason="redis_unavailable") + return False + + try: + cache_written = set_shared_cache_entry( + CONVERSATION_CACHE_NAMESPACE, + cache_key, + copy.deepcopy(payload or {}), + ttl_seconds=safe_ttl_seconds, + redis_client=redis_client, + allow_cosmos_fallback=False, + ) + _record_conversation_cache_metric( + "write" if cache_written else "write_failed", + operation=operation, + ttl_seconds=safe_ttl_seconds, + ) + return cache_written + except Exception as ex: + _record_conversation_cache_metric( + "write_failed", + operation=operation, + ttl_seconds=safe_ttl_seconds, + error=ex, + ) + log_event( + "[ConversationCache] Failed to write conversation cache payload.", + extra={"cache_key_hash": _stable_hash(cache_key)[:16], "error": str(ex)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + return False diff --git a/application/single_app/functions_conversation_metadata.py b/application/single_app/functions_conversation_metadata.py index 3b79e771..efd850e2 100644 --- a/application/single_app/functions_conversation_metadata.py +++ b/application/single_app/functions_conversation_metadata.py @@ -8,6 +8,7 @@ from functions_public_workspaces import find_public_workspace_by_id from functions_documents import get_document_metadata from functions_collaboration import get_collaboration_conversation +from functions_conversation_cache import invalidate_conversation_cache_for_item from functions_debug import debug_print def get_user_info_by_id(user_id): @@ -861,6 +862,7 @@ def update_conversation_with_metadata(conversation_id, metadata_updates): else: conversation_item['last_updated'] = updated_at cosmos_conversations_container.upsert_item(conversation_item) + invalidate_conversation_cache_for_item(conversation_item, reason="conversation_metadata_updated") return True diff --git a/application/single_app/functions_cosmos_indexing.py b/application/single_app/functions_cosmos_indexing.py new file mode 100644 index 00000000..a55dc066 --- /dev/null +++ b/application/single_app/functions_cosmos_indexing.py @@ -0,0 +1,355 @@ +# functions_cosmos_indexing.py +"""Cosmos DB indexing policy comparison and maintenance helpers.""" + +import copy +import logging +from datetime import datetime, timezone + +from azure.core import MatchConditions +from azure.cosmos import PartitionKey + +from config import ( + cosmos_collaboration_messages_container, + cosmos_collaboration_messages_container_name, + cosmos_conversations_container, + cosmos_conversations_container_name, + cosmos_database, + cosmos_group_documents_container, + cosmos_group_documents_container_name, + cosmos_messages_container, + cosmos_messages_container_name, + cosmos_public_documents_container, + cosmos_public_documents_container_name, + cosmos_user_documents_container, + cosmos_user_documents_container_name, +) +from functions_appinsights import log_event + + +COSMOS_INDEXING_POLICY_DEFINITION_VERSION = 1 +COSMOS_INDEXING_POLICY_APPLY_SETTING = 'app_maintenance_apply_cosmos_indexing_policies' +COSMOS_INDEXING_POLICY_MAX_REPLACE_RETRIES = 3 + + +def _utc_now_iso(): + return datetime.now(timezone.utc).isoformat() + + +def _composite_index(*paths): + return [ + { + 'path': path, + 'order': order, + } + for path, order in paths + ] + + +COSMOS_INDEXING_POLICY_DEFINITIONS = [ + { + 'container_name': cosmos_conversations_container_name, + 'container': cosmos_conversations_container, + 'partition_key_path': '/id', + 'description': 'Personal conversation list, feed, and search ordering by user and last update.', + 'expected_policy': { + 'compositeIndexes': [ + _composite_index(('/user_id', 'ascending'), ('/last_updated', 'descending')), + _composite_index( + ('/user_id', 'ascending'), + ('/is_hidden', 'ascending'), + ('/last_updated', 'descending'), + ), + ], + }, + }, + { + 'container_name': cosmos_messages_container_name, + 'container': cosmos_messages_container, + 'partition_key_path': '/conversation_id', + 'description': 'Personal chat message ordering by timestamp and retry-thread attempt metadata.', + 'expected_policy': { + 'compositeIndexes': [ + _composite_index(('/conversation_id', 'ascending'), ('/timestamp', 'ascending')), + _composite_index(('/conversation_id', 'ascending'), ('/timestamp', 'descending')), + _composite_index( + ('/conversation_id', 'ascending'), + ('/metadata/thread_info/thread_id', 'ascending'), + ('/role', 'ascending'), + ('/metadata/thread_info/thread_attempt', 'ascending'), + ), + ], + }, + }, + { + 'container_name': cosmos_collaboration_messages_container_name, + 'container': cosmos_collaboration_messages_container, + 'partition_key_path': '/conversation_id', + 'description': 'Collaboration chat message ordering by timestamp.', + 'expected_policy': { + 'compositeIndexes': [ + _composite_index(('/conversation_id', 'ascending'), ('/timestamp', 'ascending')), + _composite_index(('/conversation_id', 'ascending'), ('/timestamp', 'descending')), + ], + }, + }, + { + 'container_name': cosmos_user_documents_container_name, + 'container': cosmos_user_documents_container, + 'partition_key_path': '/id', + 'description': 'Personal document version lookups before the document access index read model.', + 'expected_policy': { + 'compositeIndexes': [ + _composite_index(('/id', 'ascending'), ('/version', 'descending')), + _composite_index(('/id', 'ascending'), ('/user_id', 'ascending'), ('/version', 'descending')), + ], + }, + }, + { + 'container_name': cosmos_group_documents_container_name, + 'container': cosmos_group_documents_container, + 'partition_key_path': '/id', + 'description': 'Group document version lookups before the document access index read model.', + 'expected_policy': { + 'compositeIndexes': [ + _composite_index(('/id', 'ascending'), ('/version', 'descending')), + _composite_index(('/id', 'ascending'), ('/group_id', 'ascending'), ('/version', 'descending')), + ], + }, + }, + { + 'container_name': cosmos_public_documents_container_name, + 'container': cosmos_public_documents_container, + 'partition_key_path': '/id', + 'description': 'Public workspace document version lookups before the document access index read model.', + 'expected_policy': { + 'compositeIndexes': [ + _composite_index(('/id', 'ascending'), ('/version', 'descending')), + _composite_index( + ('/id', 'ascending'), + ('/public_workspace_id', 'ascending'), + ('/version', 'descending'), + ), + ], + }, + }, +] + + +def _normalize_composite_path(path_definition): + path = str((path_definition or {}).get('path') or '').strip() + if path and not path.startswith('/'): + path = f'/{path}' + order = str((path_definition or {}).get('order') or 'ascending').strip().lower() + if order not in ('ascending', 'descending'): + order = 'ascending' + return { + 'path': path, + 'order': order, + } + + +def _canonical_composite_index(composite_index): + return tuple( + ( + normalized_path['path'], + normalized_path['order'], + ) + for normalized_path in [_normalize_composite_path(path) for path in list(composite_index or [])] + if normalized_path['path'] + ) + + +def _normalize_composite_index(composite_index): + return [ + { + 'path': path, + 'order': order, + } + for path, order in _canonical_composite_index(composite_index) + ] + + +def _get_current_indexing_policy(container_properties): + current_policy = copy.deepcopy((container_properties or {}).get('indexingPolicy') or {}) + current_policy.setdefault('indexingMode', 'consistent') + current_policy.setdefault('automatic', True) + current_policy.setdefault('includedPaths', [{'path': '/*'}]) + current_policy.setdefault('excludedPaths', [{'path': '/"_etag"/?'}]) + current_policy['compositeIndexes'] = list(current_policy.get('compositeIndexes') or []) + return current_policy + + +def _merge_expected_indexing_policy(current_policy, expected_policy): + desired_policy = copy.deepcopy(current_policy or {}) + current_composite_indexes = list(desired_policy.get('compositeIndexes') or []) + current_canonical_indexes = { + _canonical_composite_index(index) + for index in current_composite_indexes + if _canonical_composite_index(index) + } + + missing_indexes = [] + for expected_index in list((expected_policy or {}).get('compositeIndexes') or []): + normalized_expected_index = _normalize_composite_index(expected_index) + canonical_expected_index = _canonical_composite_index(normalized_expected_index) + if not canonical_expected_index: + continue + if canonical_expected_index in current_canonical_indexes: + continue + missing_indexes.append(normalized_expected_index) + current_composite_indexes.append(normalized_expected_index) + current_canonical_indexes.add(canonical_expected_index) + + desired_policy['compositeIndexes'] = current_composite_indexes + return desired_policy, missing_indexes + + +def _is_precondition_failed_error(exc): + return getattr(exc, 'status_code', None) == 412 + + +def _replace_container_indexing_policy(definition, container_properties, desired_policy): + replace_kwargs = {} + etag = (container_properties or {}).get('_etag') + if etag: + replace_kwargs['etag'] = etag + replace_kwargs['match_condition'] = MatchConditions.IfNotModified + + if 'defaultTtl' in (container_properties or {}): + replace_kwargs['default_ttl'] = container_properties.get('defaultTtl') + if 'analyticalStorageTtl' in (container_properties or {}): + replace_kwargs['analytical_storage_ttl'] = container_properties.get('analyticalStorageTtl') + if 'conflictResolutionPolicy' in (container_properties or {}): + replace_kwargs['conflict_resolution_policy'] = container_properties.get('conflictResolutionPolicy') + if 'fullTextPolicy' in (container_properties or {}): + replace_kwargs['full_text_policy'] = container_properties.get('fullTextPolicy') + + cosmos_database.replace_container( + container=definition['container_name'], + partition_key=PartitionKey(path=definition['partition_key_path']), + indexing_policy=desired_policy, + **replace_kwargs, + ) + + +def _evaluate_single_indexing_policy(definition, apply_changes=False): + for attempt in range(COSMOS_INDEXING_POLICY_MAX_REPLACE_RETRIES): + container_properties = definition['container'].read() + current_policy = _get_current_indexing_policy(container_properties) + desired_policy, missing_indexes = _merge_expected_indexing_policy( + current_policy, + definition.get('expected_policy', {}), + ) + + result = { + 'container_name': definition['container_name'], + 'partition_key_path': definition['partition_key_path'], + 'description': definition.get('description', ''), + 'status': 'aligned' if not missing_indexes else 'missing_expected_indexes', + 'apply_requested': bool(apply_changes), + 'applied': False, + 'definition_version': COSMOS_INDEXING_POLICY_DEFINITION_VERSION, + 'current_composite_index_count': len(current_policy.get('compositeIndexes') or []), + 'expected_composite_index_count': len( + list((definition.get('expected_policy') or {}).get('compositeIndexes') or []) + ), + 'missing_composite_index_count': len(missing_indexes), + 'missing_composite_indexes': missing_indexes, + 'index_transformation_progress': container_properties.get('indexTransformationProgress'), + } + + if not missing_indexes or not apply_changes: + return result + + try: + _replace_container_indexing_policy(definition, container_properties, desired_policy) + result.update({ + 'status': 'updated', + 'applied': True, + 'submitted_at': _utc_now_iso(), + 'desired_composite_index_count': len(desired_policy.get('compositeIndexes') or []), + }) + return result + except Exception as exc: + if _is_precondition_failed_error(exc) and attempt < COSMOS_INDEXING_POLICY_MAX_REPLACE_RETRIES - 1: + log_event( + '[CosmosIndexing] Retrying indexing policy update after ETag conflict.', + extra={ + 'container_name': definition['container_name'], + 'attempt': attempt + 1, + }, + level=logging.WARNING, + debug_only=True, + ) + continue + raise + + raise RuntimeError(f"Failed to update indexing policy for {definition['container_name']} after retries.") + + +def list_expected_cosmos_indexing_policies(): + """Return JSON-safe expected Cosmos indexing policy definitions.""" + return [ + { + 'container_name': definition['container_name'], + 'partition_key_path': definition['partition_key_path'], + 'description': definition.get('description', ''), + 'definition_version': COSMOS_INDEXING_POLICY_DEFINITION_VERSION, + 'expected_composite_indexes': [ + _normalize_composite_index(index) + for index in list((definition.get('expected_policy') or {}).get('compositeIndexes') or []) + ], + } + for definition in COSMOS_INDEXING_POLICY_DEFINITIONS + ] + + +def run_cosmos_indexing_policy_maintenance(apply_changes=False): + """Compare or apply expected Cosmos indexing policies for configured hot containers.""" + results = [] + for definition in COSMOS_INDEXING_POLICY_DEFINITIONS: + try: + results.append(_evaluate_single_indexing_policy(definition, apply_changes=apply_changes)) + except Exception as exc: + log_event( + '[CosmosIndexing] Failed to evaluate Cosmos indexing policy.', + extra={ + 'container_name': definition.get('container_name'), + 'apply_changes': bool(apply_changes), + 'error': str(exc), + }, + level=logging.ERROR, + exceptionTraceback=True, + ) + results.append({ + 'container_name': definition.get('container_name'), + 'partition_key_path': definition.get('partition_key_path'), + 'description': definition.get('description', ''), + 'status': 'failed', + 'apply_requested': bool(apply_changes), + 'applied': False, + 'definition_version': COSMOS_INDEXING_POLICY_DEFINITION_VERSION, + 'error': str(exc), + }) + + failed_count = sum(1 for result in results if result.get('status') == 'failed') + missing_count = sum(1 for result in results if result.get('missing_composite_index_count', 0)) + updated_count = sum(1 for result in results if result.get('applied')) + + return { + 'success': failed_count == 0, + 'mode': 'apply' if apply_changes else 'report_only', + 'apply_setting': COSMOS_INDEXING_POLICY_APPLY_SETTING, + 'definition_version': COSMOS_INDEXING_POLICY_DEFINITION_VERSION, + 'container_count': len(results), + 'failed_container_count': failed_count, + 'containers_missing_expected_indexes': missing_count, + 'updated_container_count': updated_count, + 'evaluated_at': _utc_now_iso(), + 'containers': results, + } + + +def get_cosmos_indexing_policy_status(): + """Return report-only Cosmos indexing policy status for admin diagnostics.""" + return run_cosmos_indexing_policy_maintenance(apply_changes=False) diff --git a/application/single_app/functions_cosmos_stale_cleanup.py b/application/single_app/functions_cosmos_stale_cleanup.py new file mode 100644 index 00000000..ff93e6ea --- /dev/null +++ b/application/single_app/functions_cosmos_stale_cleanup.py @@ -0,0 +1,349 @@ +# functions_cosmos_stale_cleanup.py +"""Allowlisted Cosmos settings-container cleanup helpers for stale cache artifacts.""" + +import copy +import hashlib +import logging +from datetime import datetime, timezone + +from config import cosmos_settings_container +from functions_appinsights import log_event + + +COSMOS_STALE_CACHE_CLEANUP_DEFINITION_VERSION = 1 +COSMOS_STALE_CACHE_CLEANUP_RUN_SETTING = "app_maintenance_run_stale_cache_cleanup" +COSMOS_STALE_CACHE_CLEANUP_APPLY_SETTING = "app_maintenance_apply_stale_cache_cleanup" +COSMOS_STALE_CACHE_CLEANUP_BATCH_SIZE_SETTING = "app_maintenance_stale_cache_cleanup_batch_size" +COSMOS_STALE_CACHE_CLEANUP_DEFAULT_BATCH_SIZE = 100 +COSMOS_STALE_CACHE_CLEANUP_MAX_BATCH_SIZE = 500 + +SHARED_CACHE_ENTRY_DOC_TYPE = "shared_cache_entry" +SHARED_CACHE_ENTRY_PREFIX = "shared_cache_entry:" +SHARED_CACHE_VERSION_DOC_TYPE = "cache_version" +CONVERSATION_CACHE_NAMESPACE = "conversation_cache" +CHAT_BOOTSTRAP_NAMESPACE = "chat_bootstrap" +CONVERSATION_CACHE_VERSION_DOC_PREFIX = "conversation_cache_version:" +CONVERSATION_CACHE_ENTRY_PREFIX = f"{SHARED_CACHE_ENTRY_PREFIX}{CONVERSATION_CACHE_NAMESPACE}:" +CHAT_BOOTSTRAP_ENTRY_PREFIX = f"{SHARED_CACHE_ENTRY_PREFIX}{CHAT_BOOTSTRAP_NAMESPACE}:" + +CLEANUP_CATEGORIES = { + "legacy_conversation_cache_versions": { + "description": "Redis-only conversation cache version documents from the retired Cosmos fallback path.", + }, + "redis_only_shared_cache_entries": { + "description": "Volatile conversation/chat bootstrap cache payloads that no longer use Cosmos fallback.", + }, + "expired_shared_cache_entries": { + "description": "Expired Cosmos fallback cache payloads that can be recomputed on demand.", + }, +} + + +def _utc_now(): + return datetime.now(timezone.utc) + + +def _utc_now_iso(): + return _utc_now().isoformat() + + +def _is_not_found_error(exc): + return getattr(exc, "status_code", None) == 404 + + +def _hash_document_id(document_id): + normalized_id = str(document_id or "").strip() + if not normalized_id: + return "" + return hashlib.sha256(normalized_id.encode("utf-8")).hexdigest()[:16] + + +def _normalize_batch_size(batch_size): + try: + normalized_batch_size = int(batch_size or COSMOS_STALE_CACHE_CLEANUP_DEFAULT_BATCH_SIZE) + except (TypeError, ValueError): + normalized_batch_size = COSMOS_STALE_CACHE_CLEANUP_DEFAULT_BATCH_SIZE + return min(max(normalized_batch_size, 1), COSMOS_STALE_CACHE_CLEANUP_MAX_BATCH_SIZE) + + +def _parse_utc_datetime(value): + if isinstance(value, datetime): + parsed = value + else: + try: + parsed = datetime.fromisoformat(str(value or "")) + except (TypeError, ValueError): + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def _is_expired_shared_cache_entry(document, now): + if not isinstance(document, dict) or document.get("type") != SHARED_CACHE_ENTRY_DOC_TYPE: + return False + expires_at = document.get("expires_at") + if not expires_at: + return False + parsed_expires_at = _parse_utc_datetime(expires_at) + if parsed_expires_at is None: + return True + return parsed_expires_at <= now + + +def _classify_cleanup_candidate(document, now): + if not isinstance(document, dict): + return None + + document_id = str(document.get("id") or "") + document_type = str(document.get("type") or "") + namespace = str(document.get("namespace") or "") + if ( + document_type == SHARED_CACHE_VERSION_DOC_TYPE + and document_id.startswith(CONVERSATION_CACHE_VERSION_DOC_PREFIX) + ): + return "legacy_conversation_cache_versions" + + if document_type == SHARED_CACHE_ENTRY_DOC_TYPE and ( + namespace in {CONVERSATION_CACHE_NAMESPACE, CHAT_BOOTSTRAP_NAMESPACE} + or document_id.startswith(CONVERSATION_CACHE_ENTRY_PREFIX) + or document_id.startswith(CHAT_BOOTSTRAP_ENTRY_PREFIX) + ): + return "redis_only_shared_cache_entries" + + if _is_expired_shared_cache_entry(document, now): + return "expired_shared_cache_entries" + + return None + + +def _empty_category_counts(): + return { + category_id: { + "candidate_count": 0, + "deleted_count": 0, + "skipped_count": 0, + "failed_count": 0, + } + for category_id in CLEANUP_CATEGORIES + } + + +def _increment_category_count(category_counts, category_id, count_name): + if category_id not in category_counts: + return + category_counts[category_id][count_name] = int(category_counts[category_id].get(count_name) or 0) + 1 + + +def _build_category_summary(category_counts): + return [ + { + "category": category_id, + "description": CLEANUP_CATEGORIES[category_id]["description"], + **counts, + } + for category_id, counts in category_counts.items() + ] + + +def _query_stale_cache_cleanup_candidates(container, batch_size, now_iso): + query_limit = _normalize_batch_size(batch_size) + query = f""" + SELECT TOP {query_limit} + c.id, + c.type, + c.namespace, + c.key, + c.expires_at, + c.updated_at + FROM c + WHERE + ( + c.type = @cacheVersionType + AND STARTSWITH(c.id, @conversationVersionPrefix) + ) + OR + ( + c.type = @entryType + AND ( + c.namespace = @conversationNamespace + OR c.namespace = @chatBootstrapNamespace + OR STARTSWITH(c.id, @conversationEntryPrefix) + OR STARTSWITH(c.id, @chatBootstrapEntryPrefix) + OR ( + IS_DEFINED(c.expires_at) + AND c.expires_at != null + AND c.expires_at <= @nowIso + ) + ) + ) + """ + parameters = [ + {"name": "@cacheVersionType", "value": SHARED_CACHE_VERSION_DOC_TYPE}, + {"name": "@entryType", "value": SHARED_CACHE_ENTRY_DOC_TYPE}, + {"name": "@conversationVersionPrefix", "value": CONVERSATION_CACHE_VERSION_DOC_PREFIX}, + {"name": "@conversationNamespace", "value": CONVERSATION_CACHE_NAMESPACE}, + {"name": "@chatBootstrapNamespace", "value": CHAT_BOOTSTRAP_NAMESPACE}, + {"name": "@conversationEntryPrefix", "value": CONVERSATION_CACHE_ENTRY_PREFIX}, + {"name": "@chatBootstrapEntryPrefix", "value": CHAT_BOOTSTRAP_ENTRY_PREFIX}, + {"name": "@nowIso", "value": now_iso}, + ] + return list(container.query_items( + query=query, + parameters=parameters, + enable_cross_partition_query=True, + )) + + +def build_stale_cache_cleanup_status(last_result=None): + """Return a JSON-safe stale cleanup status without querying Cosmos.""" + if isinstance(last_result, dict): + result = copy.deepcopy(last_result) + result.setdefault("success", result.get("failed_count", 0) == 0) + result.setdefault("status", "completed" if result.get("success") else "completed_with_errors") + result.setdefault("mode", "apply" if result.get("apply_requested") else "dry_run") + result.setdefault("categories", _build_category_summary(_empty_category_counts())) + return result + + return { + "success": True, + "status": "not_run", + "mode": "not_run", + "apply_requested": False, + "definition_version": COSMOS_STALE_CACHE_CLEANUP_DEFINITION_VERSION, + "candidate_count": 0, + "deleted_count": 0, + "skipped_count": 0, + "failed_count": 0, + "has_more_candidates": False, + "categories": _build_category_summary(_empty_category_counts()), + "evaluated_at": None, + } + + +def build_skipped_stale_cache_cleanup_result(reason="not_requested"): + """Return a skipped cleanup step result for maintenance runs that should not scan Cosmos.""" + result = build_stale_cache_cleanup_status() + result.update({ + "status": "skipped_disabled", + "mode": "skipped", + "skip_reason": reason, + "evaluated_at": _utc_now_iso(), + }) + return result + + +def run_stale_cache_document_cleanup(apply_changes=False, batch_size=None, container=None): + """Report or delete allowlisted stale operational cache artifacts from settings.""" + cleanup_container = container or cosmos_settings_container + now = _utc_now() + now_iso = now.isoformat() + safe_batch_size = _normalize_batch_size(batch_size) + category_counts = _empty_category_counts() + candidate_count = 0 + deleted_count = 0 + skipped_count = 0 + failed_count = 0 + + try: + candidate_documents = _query_stale_cache_cleanup_candidates( + cleanup_container, + safe_batch_size, + now_iso, + ) + except Exception as exc: + log_event( + "[CosmosStaleCleanup] Failed to query stale cache cleanup candidates.", + extra={"error": str(exc), "batch_size": safe_batch_size}, + level=logging.ERROR, + exceptionTraceback=True, + ) + return { + "success": False, + "status": "failed", + "mode": "apply" if apply_changes else "dry_run", + "apply_requested": bool(apply_changes), + "definition_version": COSMOS_STALE_CACHE_CLEANUP_DEFINITION_VERSION, + "batch_size": safe_batch_size, + "candidate_count": 0, + "deleted_count": 0, + "skipped_count": 0, + "failed_count": 1, + "has_more_candidates": False, + "categories": _build_category_summary(category_counts), + "evaluated_at": now_iso, + "error": str(exc), + } + + for document in candidate_documents: + category_id = _classify_cleanup_candidate(document, now) + if not category_id: + skipped_count += 1 + continue + + candidate_count += 1 + _increment_category_count(category_counts, category_id, "candidate_count") + document_id = str(document.get("id") or "").strip() + if not document_id: + skipped_count += 1 + _increment_category_count(category_counts, category_id, "skipped_count") + continue + + if not apply_changes: + continue + + try: + cleanup_container.delete_item(item=document_id, partition_key=document_id) + deleted_count += 1 + _increment_category_count(category_counts, category_id, "deleted_count") + except Exception as exc: + if _is_not_found_error(exc): + skipped_count += 1 + _increment_category_count(category_counts, category_id, "skipped_count") + continue + failed_count += 1 + _increment_category_count(category_counts, category_id, "failed_count") + log_event( + "[CosmosStaleCleanup] Failed to delete stale cache document.", + extra={ + "category": category_id, + "document_id_hash": _hash_document_id(document_id), + "error": str(exc), + }, + level=logging.ERROR, + exceptionTraceback=True, + ) + + status = "completed" + if failed_count: + status = "completed_with_errors" + elif not apply_changes: + status = "dry_run_completed" + + result = { + "success": failed_count == 0, + "status": status, + "mode": "apply" if apply_changes else "dry_run", + "apply_requested": bool(apply_changes), + "definition_version": COSMOS_STALE_CACHE_CLEANUP_DEFINITION_VERSION, + "batch_size": safe_batch_size, + "candidate_count": candidate_count, + "deleted_count": deleted_count, + "skipped_count": skipped_count, + "failed_count": failed_count, + "has_more_candidates": len(candidate_documents) >= safe_batch_size, + "categories": _build_category_summary(category_counts), + "evaluated_at": now_iso, + } + log_event( + "[CosmosStaleCleanup] Stale cache cleanup completed.", + extra={ + "mode": result["mode"], + "candidate_count": candidate_count, + "deleted_count": deleted_count, + "failed_count": failed_count, + "has_more_candidates": result["has_more_candidates"], + }, + level=logging.INFO if result["success"] else logging.WARNING, + ) + return result diff --git a/application/single_app/functions_cosmos_throughput.py b/application/single_app/functions_cosmos_throughput.py index d6bd7151..cd1f146e 100644 --- a/application/single_app/functions_cosmos_throughput.py +++ b/application/single_app/functions_cosmos_throughput.py @@ -2280,11 +2280,15 @@ def evaluate_and_apply_cosmos_throughput_scaling(settings, current_time=None, re scale_result['direction'] = decision.get('direction') scale_result['reason'] = decision.get('reason') + settings_update = None + if scale_result: + settings_update = build_runtime_update(status, decision, scale_result, settings=settings) + return { 'status': status, 'decision': decision, 'scale_result': scale_result, - 'settings_update': build_runtime_update(status, decision, scale_result, settings=settings), + 'settings_update': settings_update, } except Exception as exc: log_event( diff --git a/application/single_app/functions_custom_pages.py b/application/single_app/functions_custom_pages.py index 7dc94af2..07a6fcbc 100644 --- a/application/single_app/functions_custom_pages.py +++ b/application/single_app/functions_custom_pages.py @@ -3,6 +3,9 @@ import importlib.util import inspect +import copy +import hashlib +import json import logging import os import re @@ -14,7 +17,15 @@ from config import cosmos_custom_pages_container from custom_page_extension import CustomPageExtension +import app_settings_cache from functions_appinsights import log_event +from functions_shared_cache import ( + bump_shared_cache_version, + delete_shared_cache_entry, + get_shared_cache_entry, + get_shared_cache_version, + set_shared_cache_entry, +) CUSTOM_PAGES_DIR = os.path.join(os.path.dirname(__file__), "custom_pages") @@ -35,6 +46,63 @@ } LIST_FIELDS = ("roles", "css_files", "js_files", "asset_files", "json_files") PYTHON_EXTENSION_CACHE: Optional[Dict[str, Dict[str, Any]]] = None +CUSTOM_PAGES_CACHE_NAMESPACE = "custom_pages" +CUSTOM_PAGES_CACHE_VERSION_DOC_ID = "custom_pages_cache_version" +CUSTOM_PAGES_CATALOG_CACHE_KEY = "catalog" +CUSTOM_PAGES_NAV_CACHE_KEY_PREFIX = "nav" +CUSTOM_PAGES_CATALOG_CACHE_TTL_SECONDS = 300 +CUSTOM_PAGES_NAV_CACHE_TTL_SECONDS = 60 + + +def _get_shared_cache_redis_client(): + try: + return app_settings_cache.get_app_cache_redis_client() + except Exception as ex: + log_event( + "[CustomPages] Redis client lookup failed; using Cosmos cache fallback.", + extra={"error": str(ex)}, + level=logging.WARNING, + ) + return None + + +def _get_custom_pages_cache_version() -> int: + return get_shared_cache_version(CUSTOM_PAGES_CACHE_VERSION_DOC_ID, default_version=0) + + +def _stable_hash(payload: Any) -> str: + serialized = json.dumps(payload, sort_keys=True, default=str, separators=(",", ":")) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest()[:16] + + +def invalidate_custom_pages_cache(reason: str = "custom_pages_changed") -> Optional[int]: + """Invalidate custom page catalog and navigation caches without failing callers.""" + try: + version = bump_shared_cache_version( + CUSTOM_PAGES_CACHE_VERSION_DOC_ID, + description="Custom pages catalog and navigation cache version.", + ) + except Exception as ex: + version = None + log_event( + "[CustomPages] Failed to bump custom pages cache version.", + extra={"reason": reason, "error": str(ex)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + + delete_shared_cache_entry( + CUSTOM_PAGES_CACHE_NAMESPACE, + CUSTOM_PAGES_CATALOG_CACHE_KEY, + redis_client=_get_shared_cache_redis_client(), + ) + log_event( + "[CustomPages] Custom pages cache invalidated.", + extra={"reason": reason, "version": version}, + level=logging.INFO, + debug_only=True, + ) + return version def is_custom_pages_enabled(settings: Optional[Dict[str, Any]] = None) -> bool: @@ -147,7 +215,7 @@ def list_custom_pages(include_python: bool = True) -> List[Dict[str, Any]]: return sorted(pages.values(), key=lambda item: (item.get("nav_order", 100), item.get("nav_label", "").lower())) -def list_cosmos_custom_pages() -> List[Dict[str, Any]]: +def _list_cosmos_custom_pages_source() -> List[Dict[str, Any]]: """Return static custom page metadata stored in Cosmos DB.""" try: items = list(cosmos_custom_pages_container.query_items( @@ -167,7 +235,45 @@ def list_cosmos_custom_pages() -> List[Dict[str, Any]]: return [] -def get_custom_page(slug: str, include_python: bool = True) -> Optional[Dict[str, Any]]: +def list_cosmos_custom_pages() -> List[Dict[str, Any]]: + """Return cached static custom page metadata stored in Cosmos DB.""" + version = _get_custom_pages_cache_version() + redis_client = _get_shared_cache_redis_client() + cached = get_shared_cache_entry( + CUSTOM_PAGES_CACHE_NAMESPACE, + CUSTOM_PAGES_CATALOG_CACHE_KEY, + redis_client=redis_client, + ) + if isinstance(cached, dict) and cached.get("version") == version and isinstance(cached.get("pages"), list): + log_event( + "[CustomPages] Custom page catalog cache hit.", + extra={"version": version, "count": len(cached.get("pages") or [])}, + level=logging.INFO, + debug_only=True, + ) + return copy.deepcopy(cached.get("pages") or []) + + pages = _list_cosmos_custom_pages_source() + set_shared_cache_entry( + CUSTOM_PAGES_CACHE_NAMESPACE, + CUSTOM_PAGES_CATALOG_CACHE_KEY, + { + "version": version, + "pages": pages, + }, + ttl_seconds=CUSTOM_PAGES_CATALOG_CACHE_TTL_SECONDS, + redis_client=redis_client, + ) + log_event( + "[CustomPages] Custom page catalog cache miss.", + extra={"version": version, "count": len(pages)}, + level=logging.INFO, + debug_only=True, + ) + return pages + + +def get_custom_page(slug: str, include_python: bool = True, use_cache: bool = True) -> Optional[Dict[str, Any]]: """Return a custom page by slug from Cosmos or Python discovery.""" if not is_safe_slug(slug): return None @@ -178,6 +284,12 @@ def get_custom_page(slug: str, include_python: bool = True) -> Optional[Dict[str if normalized_slug in python_pages: return python_pages[normalized_slug] + if use_cache: + for page in list_cosmos_custom_pages(): + if page.get("slug") == normalized_slug: + return page + return None + try: item = cosmos_custom_pages_container.read_item(item=normalized_slug, partition_key=normalized_slug) return normalize_custom_page_metadata(_strip_cosmos_fields(item), source=CUSTOM_PAGE_SOURCE_COSMOS) @@ -202,12 +314,13 @@ def save_custom_page(metadata: Dict[str, Any], user_id: str = "system") -> Dict[ raise ValueError("; ".join(errors)) now = datetime.utcnow().isoformat() - existing = get_custom_page(page["slug"], include_python=False) + existing = get_custom_page(page["slug"], include_python=False, use_cache=False) page["created_by"] = existing.get("created_by", user_id) if existing else user_id page["created_at"] = existing.get("created_at", now) if existing else now page["modified_by"] = user_id page["modified_at"] = now result = cosmos_custom_pages_container.upsert_item(body=page) + invalidate_custom_pages_cache(reason="save_custom_page") return normalize_custom_page_metadata(_strip_cosmos_fields(result), source=CUSTOM_PAGE_SOURCE_COSMOS) @@ -217,6 +330,7 @@ def delete_custom_page(slug: str) -> bool: return False try: cosmos_custom_pages_container.delete_item(item=slug, partition_key=slug) + invalidate_custom_pages_cache(reason="delete_custom_page") return True except exceptions.CosmosResourceNotFoundError: return False @@ -280,8 +394,8 @@ def discover_python_custom_pages(force_refresh: bool = False) -> Dict[str, Dict[ return discovered -def get_custom_pages_nav(settings: Dict[str, Any]) -> List[Dict[str, Any]]: - """Return authorized custom page navigation items for the current request.""" +def _build_custom_pages_nav(settings: Dict[str, Any]) -> List[Dict[str, Any]]: + """Build authorized custom page navigation items for the current request.""" if not is_custom_pages_enabled(settings): return [] @@ -309,6 +423,53 @@ def get_custom_pages_nav(settings: Dict[str, Any]) -> List[Dict[str, Any]]: return sorted(nav_items, key=lambda item: (item.get("order", 100), item.get("label", "").lower())) +def get_custom_pages_nav(settings: Dict[str, Any]) -> List[Dict[str, Any]]: + """Return cached authorized custom page navigation items for the current request.""" + if not is_custom_pages_enabled(settings): + return [] + + roles = _current_user_roles() + version = _get_custom_pages_cache_version() + cache_key = ( + f"{CUSTOM_PAGES_NAV_CACHE_KEY_PREFIX}:" + f"{_stable_hash({'roles': sorted(roles), 'enabled': is_custom_pages_enabled(settings)})}" + ) + redis_client = _get_shared_cache_redis_client() + cached = get_shared_cache_entry( + CUSTOM_PAGES_CACHE_NAMESPACE, + cache_key, + redis_client=redis_client, + ) + if isinstance(cached, dict) and cached.get("version") == version and isinstance(cached.get("nav"), list): + log_event( + "[CustomPages] Custom page navigation cache hit.", + extra={"version": version, "nav_count": len(cached.get("nav") or [])}, + level=logging.INFO, + debug_only=True, + ) + return copy.deepcopy(cached.get("nav") or []) + + nav_items = _build_custom_pages_nav(settings) + ttl_seconds = int(settings.get("custom_pages_nav_cache_ttl_seconds") or CUSTOM_PAGES_NAV_CACHE_TTL_SECONDS) + set_shared_cache_entry( + CUSTOM_PAGES_CACHE_NAMESPACE, + cache_key, + { + "version": version, + "nav": nav_items, + }, + ttl_seconds=ttl_seconds, + redis_client=redis_client, + ) + log_event( + "[CustomPages] Custom page navigation cache miss.", + extra={"version": version, "nav_count": len(nav_items)}, + level=logging.INFO, + debug_only=True, + ) + return nav_items + + def is_custom_page_authorized(page: Dict[str, Any], roles: Optional[List[str]] = None) -> bool: """Return True when the current user roles satisfy custom page metadata.""" role_set = set(roles if roles is not None else _current_user_roles()) diff --git a/application/single_app/functions_data_management.py b/application/single_app/functions_data_management.py index 83fcb8b0..4938f2c2 100644 --- a/application/single_app/functions_data_management.py +++ b/application/single_app/functions_data_management.py @@ -12,7 +12,8 @@ from azure.core import MatchConditions from azure.core.credentials import AzureKeyCredential -from azure.cosmos import CosmosClient, PartitionKey +import azure.cosmos as azure_cosmos +from azure.cosmos import PartitionKey from azure.cosmos.exceptions import CosmosResourceNotFoundError from azure.identity import DefaultAzureCredential from azure.search.documents import SearchClient @@ -713,9 +714,9 @@ def _get_target_cosmos_database(settings): key = _safe_text((settings or {}).get("target_cosmos_key")) if not key: raise ValueError("Target Cosmos account key is required when account key authentication is selected.") - client = CosmosClient(endpoint, credential=key, consistency_level="Session") + client = azure_cosmos.CosmosClient(endpoint, credential=key, consistency_level="Session") else: - client = CosmosClient(endpoint, credential=DefaultAzureCredential(), consistency_level="Session") + client = azure_cosmos.CosmosClient(endpoint, credential=DefaultAzureCredential(), consistency_level="Session") return client.create_database_if_not_exists(DATA_MANAGEMENT_TARGET_COSMOS_DATABASE_NAME) diff --git a/application/single_app/functions_document_access_index.py b/application/single_app/functions_document_access_index.py new file mode 100644 index 00000000..a34250e1 --- /dev/null +++ b/application/single_app/functions_document_access_index.py @@ -0,0 +1,4309 @@ +# functions_document_access_index.py +"""Document access index projection helpers for Cosmos-backed document reads.""" + +import copy +import hashlib +import json +import logging +import re +import threading +import uuid +from datetime import datetime, timedelta, timezone +from time import perf_counter + +import app_settings_cache +from azure.core import MatchConditions +from config import ( + cosmos_document_access_index_container, + cosmos_groups_container, + cosmos_group_documents_container, + cosmos_group_documents_container_name, + cosmos_public_workspaces_container, + cosmos_public_documents_container, + cosmos_public_documents_container_name, + cosmos_settings_container, + cosmos_user_settings_container, + cosmos_user_documents_container, + cosmos_user_documents_container_name, +) +from functions_appinsights import log_event +from functions_settings import get_settings + + +DOCUMENT_ACCESS_INDEX_TYPE = 'document_access_index' +DOCUMENT_ACCESS_REPAIR_TYPE = 'document_access_index_repair' +DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_TYPE = 'document_access_index_repair_backlog_state' +DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID = 'document_access_index_repair_backlog_state' +DOCUMENT_ACCESS_BACKFILL_STATE_TYPE = 'document_access_index_backfill_state' +DOCUMENT_ACCESS_BACKFILL_STATE_DOC_ID = 'document_access_index_backfill_state' +DOCUMENT_ACCESS_SHADOW_STATE_TYPE = 'document_access_index_shadow_validation_state' +DOCUMENT_ACCESS_SHADOW_STATE_DOC_ID = 'document_access_index_shadow_validation_state' +DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION = 2 + +DOCUMENT_ACCESS_SCOPE_PERSONAL = 'personal' +DOCUMENT_ACCESS_SCOPE_GROUP = 'group' +DOCUMENT_ACCESS_SCOPE_PUBLIC = 'public' +DOCUMENT_ACCESS_PRINCIPAL_USER = 'user' + +DOCUMENT_ACCESS_OPERATION_UPSERT = 'upsert' +DOCUMENT_ACCESS_OPERATION_DELETE = 'delete' + +DOCUMENT_ACCESS_APPROVAL_APPROVED = 'approved' +DOCUMENT_ACCESS_APPROVAL_NOT_APPROVED = 'not_approved' + +DOCUMENT_ACCESS_BACKFILL_ENABLED_SETTING = 'enable_startup_document_access_index_backfill' +DOCUMENT_ACCESS_BACKFILL_BATCH_SIZE_SETTING = 'document_access_index_backfill_batch_size' +DOCUMENT_ACCESS_REPAIR_BATCH_SIZE_SETTING = 'document_access_index_repair_batch_size' +DOCUMENT_ACCESS_ACTIVE_MAINTENANCE_INTERVAL_SETTING = 'document_access_index_active_maintenance_interval_seconds' +DOCUMENT_ACCESS_CACHE_ENABLED_SETTING = 'enable_document_access_index_cache' +DOCUMENT_ACCESS_CACHE_TTL_SECONDS_SETTING = 'document_access_index_cache_ttl_seconds' +DOCUMENT_ACCESS_DEFAULT_BACKFILL_BATCH_SIZE = 200 +DOCUMENT_ACCESS_MAX_BACKFILL_BATCH_SIZE = 1000 +DOCUMENT_ACCESS_DEFAULT_REPAIR_BATCH_SIZE = 100 +DOCUMENT_ACCESS_MAX_REPAIR_BATCH_SIZE = 500 +DOCUMENT_ACCESS_DEFAULT_CACHE_TTL_SECONDS = 900 +DOCUMENT_ACCESS_MIN_CACHE_TTL_SECONDS = 60 +DOCUMENT_ACCESS_MAX_CACHE_TTL_SECONDS = 900 +DOCUMENT_ACCESS_CACHE_VERSION_MIN_TTL_SECONDS = 3600 +DOCUMENT_ACCESS_CACHE_VERSION_MAX_TTL_SECONDS = 86400 +DOCUMENT_ACCESS_CACHE_VERSION_TTL_MULTIPLIER = 4 +DOCUMENT_ACCESS_CACHE_VERSION_HYGIENE_BATCH_SIZE = 100 +DOCUMENT_ACCESS_CACHE_VERSION_HYGIENE_MAX_SCAN_ITERATIONS = 5 +DOCUMENT_ACCESS_CACHE_KEY_PREFIX = 'DAI_LIST_CACHE' +DOCUMENT_ACCESS_CACHE_VERSION_KEY_PREFIX = 'DAI_LIST_CACHE_VERSION' +DOCUMENT_ACCESS_ARCHIVED_REVISION_BLOB_PATH_MODE = 'archived_revision' +DOCUMENT_ACCESS_SHADOW_MAX_SAMPLE_IDS = 20 +DOCUMENT_ACCESS_SHADOW_METRIC_WINDOWS_MINUTES = (5, 15) +DOCUMENT_ACCESS_SHADOW_METRIC_RETENTION_MINUTES = 60 +DOCUMENT_ACCESS_SHADOW_METRIC_MAX_SAMPLES = 1000 +DOCUMENT_ACCESS_SHADOW_STATE_WRITE_MAX_RETRIES = 5 +DOCUMENT_ACCESS_READ_METRIC_WINDOWS_MINUTES = (5, 15, 60) +DOCUMENT_ACCESS_READ_METRIC_RETENTION_MINUTES = 60 +DOCUMENT_ACCESS_READ_METRIC_MAX_SAMPLES = 2000 +DOCUMENT_ACCESS_CACHE_METRIC_WINDOWS_MINUTES = (5, 15, 60) +DOCUMENT_ACCESS_CACHE_METRIC_RETENTION_MINUTES = 60 +DOCUMENT_ACCESS_CACHE_METRIC_MAX_SAMPLES = 2000 +DOCUMENT_ACCESS_STATE_READ_TTL_SECONDS = 30 +DOCUMENT_ACCESS_BACKFILL_READY_STATUSES = {'succeeded', 'succeeded_with_errors'} +DOCUMENT_ACCESS_BACKFILL_COMPLETE_STATUSES = DOCUMENT_ACCESS_BACKFILL_READY_STATUSES | {'skipped_completed'} + +DOCUMENT_ACCESS_SOURCE_SCOPES = ( + DOCUMENT_ACCESS_SCOPE_PERSONAL, + DOCUMENT_ACCESS_SCOPE_GROUP, + DOCUMENT_ACCESS_SCOPE_PUBLIC, +) + +_document_access_read_metric_lock = threading.Lock() +_document_access_read_metric_samples = [] +_document_access_cache_metric_lock = threading.Lock() +_document_access_state_cache = {} +_document_access_state_cache_lock = threading.Lock() +_document_access_cache_metric_samples = [] +_document_access_cache_epoch_lock = threading.Lock() +_document_access_cache_process_epoch = uuid.uuid4().hex +_document_access_cache_local_generation = 0 + + +class DocumentAccessIndexCacheInvalidationError(RuntimeError): + """Raised when DAI cache scope invalidation fails after projection mutation.""" + + def __init__(self, operation, status, scope_keys=None): + super().__init__( + 'Document access index cache invalidation failed ' + f'for {operation}: {status or "unknown"}' + ) + self.operation = operation + self.status = status + self.scope_keys = list(scope_keys or []) + + +class DocumentAccessIndexProjectionMutationError(RuntimeError): + """Raised when projection mutation fails after affected cache scopes are known.""" + + def __init__(self, operation, error, scope_keys=None): + super().__init__(str(error)) + self.operation = operation + self.original_error = error + self.scope_keys = list(scope_keys or []) + + +def _utc_now_iso(): + return datetime.now(timezone.utc).isoformat() + + +def _parse_utc_datetime(value): + if not value: + return None + try: + parsed_value = datetime.fromisoformat(str(value).replace('Z', '+00:00')) + except ValueError: + return None + if parsed_value.tzinfo is None: + parsed_value = parsed_value.replace(tzinfo=timezone.utc) + return parsed_value.astimezone(timezone.utc) + + +def _get_cached_document_access_state(doc_id): + now = perf_counter() + with _document_access_state_cache_lock: + cached_state = _document_access_state_cache.get(doc_id) + if not cached_state: + return False, None + if cached_state.get('expires_at', 0) <= now: + _document_access_state_cache.pop(doc_id, None) + return False, None + return True, copy.deepcopy(cached_state.get('value')) + + +def _set_cached_document_access_state(doc_id, value): + with _document_access_state_cache_lock: + _document_access_state_cache[doc_id] = { + 'value': copy.deepcopy(value), + 'expires_at': perf_counter() + DOCUMENT_ACCESS_STATE_READ_TTL_SECONDS, + } + + +def _normalize_positive_int(value, default_value, min_value=1, max_value=1000): + try: + normalized_value = int(value) + except (TypeError, ValueError): + normalized_value = default_value + return min(max(normalized_value, min_value), max_value) + + +def _calculate_document_access_cache_version_ttl_seconds(cache_ttl_seconds): + normalized_cache_ttl = _normalize_positive_int( + cache_ttl_seconds, + DOCUMENT_ACCESS_DEFAULT_CACHE_TTL_SECONDS, + min_value=DOCUMENT_ACCESS_MIN_CACHE_TTL_SECONDS, + max_value=DOCUMENT_ACCESS_MAX_CACHE_TTL_SECONDS, + ) + return min( + max( + normalized_cache_ttl * DOCUMENT_ACCESS_CACHE_VERSION_TTL_MULTIPLIER, + DOCUMENT_ACCESS_CACHE_VERSION_MIN_TTL_SECONDS, + ), + DOCUMENT_ACCESS_CACHE_VERSION_MAX_TTL_SECONDS, + ) + + +def _safe_float(value): + if value in (None, ''): + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _round_metric(value, digits=3): + normalized_value = _safe_float(value) + if normalized_value is None: + return None + return round(normalized_value, digits) + + +def _sum_metric_values(items, key): + values = [ + _safe_float(item.get(key)) + for item in items or [] + if _safe_float(item.get(key)) is not None + ] + if not values: + return None + return _round_metric(sum(values)) + + +def _average_metric_values(items, key): + values = [ + _safe_float(item.get(key)) + for item in items or [] + if _safe_float(item.get(key)) is not None + ] + if not values: + return None + return _round_metric(sum(values) / len(values)) + + +def _sum_int_values(items, key): + total = 0 + for item in items or []: + try: + total += int(item.get(key) or 0) + except (TypeError, ValueError): + continue + return total + + +def _safe_int(value, default_value=0): + try: + return int(value or default_value) + except (TypeError, ValueError): + return default_value + + +def _get_header_value(headers, name): + if not hasattr(headers, 'get'): + return None + return headers.get(name) or headers.get(name.lower()) or headers.get(name.upper()) + + +def _metric_delta(source_value, projection_value): + source_metric = _safe_float(source_value) + projection_metric = _safe_float(projection_value) + if source_metric is None or projection_metric is None: + return None + return _round_metric(source_metric - projection_metric) + + +def _metric_percent(numerator, denominator): + try: + denominator_value = float(denominator or 0) + if denominator_value <= 0: + return None + return _round_metric((float(numerator or 0) / denominator_value) * 100) + except (TypeError, ValueError): + return None + + +def _percentile_metric_value(items, key, percentile): + values = sorted( + _safe_float(item.get(key)) + for item in items or [] + if _safe_float(item.get(key)) is not None + ) + if not values: + return None + index = int(round((len(values) - 1) * percentile)) + return _round_metric(values[min(max(index, 0), len(values) - 1)]) + + +def _empty_read_metric_window(window_minutes): + return { + 'window_minutes': window_minutes, + 'sample_count': 0, + 'served_from_index_count': 0, + 'served_from_cache_count': 0, + 'source_fallback_count': 0, + 'query_failed_count': 0, + 'fallback_rate_percent': None, + 'cache_hit_rate_percent': None, + 'request_charge': None, + 'elapsed_ms_avg': None, + 'elapsed_ms_p95': None, + 'item_count': 0, + 'page_count': 0, + 'operation_counts': {}, + 'status_counts': {}, + 'first_checked_at': None, + 'last_checked_at': None, + } + + +def _prune_read_metric_samples(samples, now): + cutoff = now - timedelta(minutes=DOCUMENT_ACCESS_READ_METRIC_RETENTION_MINUTES) + pruned_samples = [] + for sample in samples or []: + if not isinstance(sample, dict): + continue + checked_at = _parse_utc_datetime(sample.get('checked_at')) + if not checked_at or checked_at < cutoff: + continue + pruned_samples.append(copy.deepcopy(sample)) + + pruned_samples.sort(key=lambda sample: sample.get('checked_at') or '') + return pruned_samples[-DOCUMENT_ACCESS_READ_METRIC_MAX_SAMPLES:] + + +def _increment_count(counts, key): + normalized_key = str(key or 'unknown').strip().lower() or 'unknown' + counts[normalized_key] = int(counts.get(normalized_key) or 0) + 1 + + +def _build_read_metric_window(samples, now, window_minutes): + cutoff = now - timedelta(minutes=window_minutes) + window_samples = [ + sample + for sample in samples or [] + if (_parse_utc_datetime(sample.get('checked_at')) or datetime.min.replace(tzinfo=timezone.utc)) >= cutoff + ] + if not window_samples: + return _empty_read_metric_window(window_minutes) + + served_samples = [ + sample + for sample in window_samples + if sample.get('served_from_index') + ] + operation_counts = {} + status_counts = {} + for sample in window_samples: + _increment_count(operation_counts, sample.get('operation')) + _increment_count(status_counts, sample.get('status')) + + source_fallback_count = sum(1 for sample in window_samples if sample.get('source_fallback')) + served_from_cache_count = sum(1 for sample in served_samples if sample.get('served_from_cache')) + return { + 'window_minutes': window_minutes, + 'sample_count': len(window_samples), + 'served_from_index_count': len(served_samples), + 'served_from_cache_count': served_from_cache_count, + 'source_fallback_count': source_fallback_count, + 'query_failed_count': sum(1 for sample in window_samples if sample.get('status') == 'query_failed'), + 'fallback_rate_percent': _metric_percent(source_fallback_count, len(window_samples)), + 'cache_hit_rate_percent': _metric_percent(served_from_cache_count, len(served_samples)), + 'request_charge': _sum_metric_values(served_samples, 'request_charge'), + 'elapsed_ms_avg': _average_metric_values(served_samples, 'elapsed_ms'), + 'elapsed_ms_p95': _percentile_metric_value(served_samples, 'elapsed_ms', 0.95), + 'item_count': _sum_int_values(served_samples, 'item_count'), + 'page_count': _sum_int_values(served_samples, 'page_count'), + 'operation_counts': operation_counts, + 'status_counts': status_counts, + 'first_checked_at': window_samples[0].get('checked_at'), + 'last_checked_at': window_samples[-1].get('checked_at'), + } + + +def _build_read_metrics(samples, now): + fallback_samples = [ + sample + for sample in samples or [] + if sample.get('source_fallback') + ] + return { + 'updated_at': now.isoformat(), + 'retention_minutes': DOCUMENT_ACCESS_READ_METRIC_RETENTION_MINUTES, + 'sample_limit': DOCUMENT_ACCESS_READ_METRIC_MAX_SAMPLES, + 'sample_count': len(samples or []), + 'last_sample': copy.deepcopy(samples[-1]) if samples else None, + 'last_fallback_sample': copy.deepcopy(fallback_samples[-1]) if fallback_samples else None, + 'windows': { + f'{window_minutes}m': _build_read_metric_window(samples, now, window_minutes) + for window_minutes in DOCUMENT_ACCESS_READ_METRIC_WINDOWS_MINUTES + }, + } + + +def _record_document_access_read_metric( + operation, + source_scope, + status, + served_from_index, + diagnostics=None, + scope_key_count=0, + served_from_cache=False, + cache_status=None, + error=None, +): + diagnostics = diagnostics if isinstance(diagnostics, dict) else {} + sample = { + 'checked_at': _utc_now_iso(), + 'operation': str(operation or 'document_list').strip().lower(), + 'source_scope': str(source_scope or '').strip().lower(), + 'status': str(status or 'unknown').strip().lower() or 'unknown', + 'served_from_index': bool(served_from_index), + 'served_from_cache': bool(served_from_cache), + 'source_fallback': not bool(served_from_index), + 'cache_status': str(cache_status or '').strip().lower() or None, + 'request_charge': _round_metric(diagnostics.get('request_charge')), + 'elapsed_ms': _round_metric(diagnostics.get('elapsed_ms')), + 'item_count': _safe_int(diagnostics.get('item_count')), + 'page_count': _safe_int(diagnostics.get('page_count')), + 'scope_key_count': _safe_int(scope_key_count), + } + if error: + sample['error'] = str(error) + + with _document_access_read_metric_lock: + now = datetime.now(timezone.utc) + samples = _prune_read_metric_samples(_document_access_read_metric_samples, now) + samples.append(sample) + del samples[:-DOCUMENT_ACCESS_READ_METRIC_MAX_SAMPLES] + _document_access_read_metric_samples[:] = samples + return sample + + +def get_document_access_index_read_metrics(): + """Return lightweight in-process rolling metrics for production DAI reads.""" + with _document_access_read_metric_lock: + samples = copy.deepcopy(_document_access_read_metric_samples) + now = datetime.now(timezone.utc) + samples = _prune_read_metric_samples(samples, now) + return _build_read_metrics(samples, now) + + +def _empty_cache_metric_window(window_minutes): + return { + 'window_minutes': window_minutes, + 'sample_count': 0, + 'hit_count': 0, + 'miss_count': 0, + 'bypass_count': 0, + 'write_count': 0, + 'invalidation_count': 0, + 'error_count': 0, + 'hit_rate_percent': None, + 'operation_counts': {}, + 'event_counts': {}, + 'reason_counts': {}, + 'first_checked_at': None, + 'last_checked_at': None, + } + + +def _prune_cache_metric_samples(samples, now): + cutoff = now - timedelta(minutes=DOCUMENT_ACCESS_CACHE_METRIC_RETENTION_MINUTES) + pruned_samples = [] + for sample in samples or []: + if not isinstance(sample, dict): + continue + checked_at = _parse_utc_datetime(sample.get('checked_at')) + if not checked_at or checked_at < cutoff: + continue + pruned_samples.append(copy.deepcopy(sample)) + + pruned_samples.sort(key=lambda sample: sample.get('checked_at') or '') + return pruned_samples[-DOCUMENT_ACCESS_CACHE_METRIC_MAX_SAMPLES:] + + +def _cache_metric_matches(sample, event_types): + return str(sample.get('event_type') or '').strip().lower() in event_types + + +def _build_cache_metric_window(samples, now, window_minutes): + cutoff = now - timedelta(minutes=window_minutes) + window_samples = [ + sample + for sample in samples or [] + if (_parse_utc_datetime(sample.get('checked_at')) or datetime.min.replace(tzinfo=timezone.utc)) >= cutoff + ] + if not window_samples: + return _empty_cache_metric_window(window_minutes) + + operation_counts = {} + event_counts = {} + reason_counts = {} + for sample in window_samples: + _increment_count(operation_counts, sample.get('operation')) + _increment_count(event_counts, sample.get('event_type')) + _increment_count(reason_counts, sample.get('reason')) + + hit_count = sum(1 for sample in window_samples if _cache_metric_matches(sample, {'hit'})) + miss_count = sum(1 for sample in window_samples if _cache_metric_matches(sample, {'miss'})) + bypass_count = sum(1 for sample in window_samples if _cache_metric_matches(sample, {'bypass'})) + write_count = sum(1 for sample in window_samples if _cache_metric_matches(sample, {'write'})) + invalidation_count = sum(1 for sample in window_samples if _cache_metric_matches(sample, {'invalidate'})) + error_count = sum( + 1 + for sample in window_samples + if _cache_metric_matches(sample, {'read_failed', 'write_failed', 'version_read_failed', 'invalidate_failed'}) + ) + return { + 'window_minutes': window_minutes, + 'sample_count': len(window_samples), + 'hit_count': hit_count, + 'miss_count': miss_count, + 'bypass_count': bypass_count, + 'write_count': write_count, + 'invalidation_count': invalidation_count, + 'error_count': error_count, + 'hit_rate_percent': _metric_percent(hit_count, hit_count + miss_count), + 'operation_counts': operation_counts, + 'event_counts': event_counts, + 'reason_counts': reason_counts, + 'first_checked_at': window_samples[0].get('checked_at'), + 'last_checked_at': window_samples[-1].get('checked_at'), + } + + +def _build_cache_metrics(samples, now): + error_samples = [ + sample + for sample in samples or [] + if str(sample.get('event_type') or '').strip().lower().endswith('_failed') + ] + invalidation_samples = [ + sample + for sample in samples or [] + if str(sample.get('event_type') or '').strip().lower() == 'invalidate' + ] + return { + 'updated_at': now.isoformat(), + 'retention_minutes': DOCUMENT_ACCESS_CACHE_METRIC_RETENTION_MINUTES, + 'sample_limit': DOCUMENT_ACCESS_CACHE_METRIC_MAX_SAMPLES, + 'sample_count': len(samples or []), + 'last_event': copy.deepcopy(samples[-1]) if samples else None, + 'last_error': copy.deepcopy(error_samples[-1]) if error_samples else None, + 'last_invalidation': copy.deepcopy(invalidation_samples[-1]) if invalidation_samples else None, + 'windows': { + f'{window_minutes}m': _build_cache_metric_window(samples, now, window_minutes) + for window_minutes in DOCUMENT_ACCESS_CACHE_METRIC_WINDOWS_MINUTES + }, + } + + +def _record_document_access_cache_metric( + event_type, + operation=None, + source_scope=None, + scope_key_count=0, + reason=None, + elapsed_ms=None, + ttl_seconds=None, + error=None, +): + sample = { + 'checked_at': _utc_now_iso(), + 'event_type': str(event_type or 'unknown').strip().lower() or 'unknown', + 'operation': str(operation or 'document_list').strip().lower() or 'document_list', + 'source_scope': str(source_scope or '').strip().lower(), + 'scope_key_count': _safe_int(scope_key_count), + 'reason': str(reason or '').strip().lower() or None, + 'elapsed_ms': _round_metric(elapsed_ms), + 'ttl_seconds': _safe_int(ttl_seconds), + } + if error: + sample['error_type'] = type(error).__name__ + + with _document_access_cache_metric_lock: + now = datetime.now(timezone.utc) + samples = _prune_cache_metric_samples(_document_access_cache_metric_samples, now) + samples.append(sample) + del samples[:-DOCUMENT_ACCESS_CACHE_METRIC_MAX_SAMPLES] + _document_access_cache_metric_samples[:] = samples + return sample + + +def get_document_access_index_cache_metrics(): + """Return lightweight in-process rolling metrics for Redis DAI list caching.""" + with _document_access_cache_metric_lock: + samples = copy.deepcopy(_document_access_cache_metric_samples) + now = datetime.now(timezone.utc) + samples = _prune_cache_metric_samples(samples, now) + return _build_cache_metrics(samples, now) + + +def get_document_access_index_settings(settings=None): + """Normalize document access index feature flags.""" + if settings is None: + try: + settings = get_settings() + except Exception as exc: + log_event( + '[DocumentAccessIndex] Settings could not be loaded; DAI reads will use source fallback until settings are available.', + extra={'error': str(exc)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + settings = {} + if not isinstance(settings, dict): + log_event( + '[DocumentAccessIndex] Settings payload was invalid; DAI reads will use source fallback until settings are available.', + extra={'settings_type': type(settings).__name__}, + level=logging.WARNING, + ) + settings = {} + cache_ttl_seconds = _normalize_positive_int( + settings.get(DOCUMENT_ACCESS_CACHE_TTL_SECONDS_SETTING), + DOCUMENT_ACCESS_DEFAULT_CACHE_TTL_SECONDS, + min_value=DOCUMENT_ACCESS_MIN_CACHE_TTL_SECONDS, + max_value=DOCUMENT_ACCESS_MAX_CACHE_TTL_SECONDS, + ) + return { + 'container_enabled': True, + 'write_through_enabled': True, + 'reads_enabled': True, + 'shadow_validation_enabled': bool(settings.get('enable_document_access_index_shadow_validation', False)), + 'startup_backfill_enabled': True, + 'backfill_batch_size': _normalize_positive_int( + settings.get(DOCUMENT_ACCESS_BACKFILL_BATCH_SIZE_SETTING), + DOCUMENT_ACCESS_DEFAULT_BACKFILL_BATCH_SIZE, + max_value=DOCUMENT_ACCESS_MAX_BACKFILL_BATCH_SIZE, + ), + 'repair_batch_size': _normalize_positive_int( + settings.get(DOCUMENT_ACCESS_REPAIR_BATCH_SIZE_SETTING), + DOCUMENT_ACCESS_DEFAULT_REPAIR_BATCH_SIZE, + max_value=DOCUMENT_ACCESS_MAX_REPAIR_BATCH_SIZE, + ), + 'cache_enabled': bool(settings.get(DOCUMENT_ACCESS_CACHE_ENABLED_SETTING, True)), + 'redis_cache_configured': bool(settings.get('enable_redis_cache', False)), + 'cache_ttl_seconds': cache_ttl_seconds, + 'cache_version_ttl_seconds': _calculate_document_access_cache_version_ttl_seconds(cache_ttl_seconds), + } + + +def is_document_access_shadow_validation_enabled(settings=None): + """Return True when shadow validation should collect comparison diagnostics.""" + normalized_settings = get_document_access_index_settings(settings) + return ( + bool(normalized_settings.get('container_enabled')) + and bool(normalized_settings.get('shadow_validation_enabled')) + ) + + +def is_document_access_index_reads_enabled(settings=None): + """Return True when the default DAI read path is enabled.""" + normalized_settings = get_document_access_index_settings(settings) + return ( + bool(normalized_settings.get('container_enabled')) + and bool(normalized_settings.get('write_through_enabled')) + and bool(normalized_settings.get('reads_enabled')) + ) + + +def query_items_with_cosmos_diagnostics(container, diagnostics_label, collect_diagnostics=True, **query_kwargs): + """Run a Cosmos query and optionally capture elapsed time and request charge.""" + if not collect_diagnostics: + return list(container.query_items(**query_kwargs)), None + + query_kwargs = copy.copy(query_kwargs) + existing_response_hook = query_kwargs.get('response_hook') + request_charge_total = 0.0 + has_request_charge = False + page_count = 0 + activity_ids = [] + query_metrics = [] + + def capture_response_headers(headers, response): + nonlocal request_charge_total, has_request_charge, page_count + if callable(existing_response_hook): + existing_response_hook(headers, response) + + if hasattr(response, 'by_page'): + return + + page_count += 1 + request_charge = _safe_float(_get_header_value(headers, 'x-ms-request-charge')) + if request_charge is not None: + request_charge_total += request_charge + has_request_charge = True + + activity_id = _get_header_value(headers, 'x-ms-activity-id') + if activity_id: + activity_ids.append(str(activity_id)) + + query_metric = _get_header_value(headers, 'x-ms-documentdb-query-metrics') + if query_metric: + query_metrics.append(str(query_metric)) + + query_kwargs['populate_query_metrics'] = True + query_kwargs['response_hook'] = capture_response_headers + started_at = perf_counter() + items = list(container.query_items(**query_kwargs)) + elapsed_ms = _round_metric((perf_counter() - started_at) * 1000) + diagnostics = { + 'label': diagnostics_label, + 'request_charge': _round_metric(request_charge_total) if has_request_charge else None, + 'elapsed_ms': elapsed_ms, + 'item_count': len(items), + 'page_count': page_count, + } + if activity_ids: + diagnostics['activity_ids'] = activity_ids[:5] + if query_metrics: + diagnostics['query_metrics'] = query_metrics[:5] + return items, diagnostics + + +def _combine_query_diagnostics(label, diagnostics_list): + diagnostics_list = [diagnostics for diagnostics in diagnostics_list or [] if isinstance(diagnostics, dict)] + request_charges = [ + _safe_float(diagnostics.get('request_charge')) + for diagnostics in diagnostics_list + if _safe_float(diagnostics.get('request_charge')) is not None + ] + elapsed_values = [ + _safe_float(diagnostics.get('elapsed_ms')) + for diagnostics in diagnostics_list + if _safe_float(diagnostics.get('elapsed_ms')) is not None + ] + return { + 'label': label, + 'request_charge': _round_metric(sum(request_charges)) if request_charges else None, + 'elapsed_ms': _round_metric(sum(elapsed_values)) if elapsed_values else None, + 'item_count': sum(int(diagnostics.get('item_count') or 0) for diagnostics in diagnostics_list), + 'page_count': sum(int(diagnostics.get('page_count') or 0) for diagnostics in diagnostics_list), + } + + +def _build_shadow_metric_fields(source_query_metrics, projection_query_metrics, candidate_read_metrics=None): + source_ru = _round_metric((source_query_metrics or {}).get('request_charge')) + validation_index_ru = _round_metric((projection_query_metrics or {}).get('request_charge')) + source_ms = _round_metric((source_query_metrics or {}).get('elapsed_ms')) + validation_index_ms = _round_metric((projection_query_metrics or {}).get('elapsed_ms')) + candidate_metrics_available = ( + isinstance(candidate_read_metrics, dict) + and not candidate_read_metrics.get('partial_failure') + and not candidate_read_metrics.get('errors') + ) + candidate_read_ru = ( + _round_metric(candidate_read_metrics.get('request_charge')) + if candidate_metrics_available + else None + ) + candidate_read_ms = ( + _round_metric(candidate_read_metrics.get('elapsed_ms')) + if candidate_metrics_available + else None + ) + return { + 'source_query_ru': source_ru, + 'projection_query_ru': validation_index_ru, + 'validation_index_ru': validation_index_ru, + 'candidate_read_ru': candidate_read_ru, + 'estimated_ru_savings': _metric_delta(source_ru, validation_index_ru), + 'estimated_wave5_ru_savings': _metric_delta(source_ru, candidate_read_ru), + 'shadow_overhead_ru': validation_index_ru, + 'source_query_ms': source_ms, + 'projection_query_ms': validation_index_ms, + 'validation_index_ms': validation_index_ms, + 'candidate_read_ms': candidate_read_ms, + 'estimated_ms_savings': _metric_delta(source_ms, validation_index_ms), + 'estimated_wave5_ms_savings': _metric_delta(source_ms, candidate_read_ms), + 'shadow_overhead_ms': validation_index_ms, + 'source_query_item_count': int((source_query_metrics or {}).get('item_count') or 0), + 'projection_query_item_count': int((projection_query_metrics or {}).get('item_count') or 0), + 'candidate_read_item_count': int((candidate_read_metrics or {}).get('item_count') or 0), + 'source_query_page_count': int((source_query_metrics or {}).get('page_count') or 0), + 'projection_query_page_count': int((projection_query_metrics or {}).get('page_count') or 0), + 'candidate_read_page_count': int((candidate_read_metrics or {}).get('page_count') or 0), + } + + +def _is_container_enabled(settings=None): + normalized_settings = get_document_access_index_settings(settings) + return normalized_settings.get('container_enabled') + + +def _is_write_through_enabled(settings=None, force=False): + normalized_settings = get_document_access_index_settings(settings) + if force: + return normalized_settings.get('container_enabled') + return normalized_settings.get('container_enabled') and normalized_settings.get('write_through_enabled') + + +def _is_not_found_error(exc): + return getattr(exc, 'status_code', None) == 404 + + +def _is_write_conflict_error(exc): + return getattr(exc, 'status_code', None) in (409, 412) + + +def _safe_id_part(value): + normalized = str(value or '').strip() + normalized = re.sub(r'[^A-Za-z0-9_.:-]+', '_', normalized) + return normalized or 'none' + + +def build_document_access_scope_key(scope_type, scope_id): + """Build the access-index partition key for a user, group, or public workspace scope.""" + normalized_scope_type = str(scope_type or '').strip().lower() + normalized_scope_id = str(scope_id or '').strip() + if not normalized_scope_type or not normalized_scope_id: + return None + return f'{normalized_scope_type}:{normalized_scope_id}' + + +def _document_access_cache_hash(value): + serialized_value = json.dumps(value, sort_keys=True, separators=(',', ':'), default=str) + return hashlib.sha256(serialized_value.encode('utf-8')).hexdigest() + + +def build_document_access_cache_scope_hash(scope_key): + """Return the stable Redis DAI cache hash for a user/group/public scope key.""" + return _document_access_cache_hash({'scope_key': scope_key}) + + +def _document_access_cache_version_key(scope_key): + scope_hash = build_document_access_cache_scope_hash(scope_key) + return f'{DOCUMENT_ACCESS_CACHE_VERSION_KEY_PREFIX}:{scope_hash}' + + +def _document_access_cache_entry_key(operation, source_scope, scope_versions, key_payload): + cache_hash = _document_access_cache_hash({ + 'operation': operation, + 'source_scope': source_scope, + 'schema_version': DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION, + 'local_epoch': _get_document_access_cache_local_epoch(), + 'scope_versions': scope_versions, + 'key_payload': key_payload or {}, + }) + return f'{DOCUMENT_ACCESS_CACHE_KEY_PREFIX}:{operation}:{source_scope}:{cache_hash}' + + +def _get_document_access_cache_local_epoch(): + with _document_access_cache_epoch_lock: + return f'{_document_access_cache_process_epoch}:{_document_access_cache_local_generation}' + + +def _bump_document_access_cache_local_epoch(): + global _document_access_cache_local_generation + with _document_access_cache_epoch_lock: + _document_access_cache_local_generation += 1 + return f'{_document_access_cache_process_epoch}:{_document_access_cache_local_generation}' + + +def _decode_cache_json(raw_value): + if raw_value is None: + return None + if isinstance(raw_value, bytes): + raw_value = raw_value.decode('utf-8') + return json.loads(raw_value) + + +def _get_document_access_cache_client( + normalized_settings, + operation, + source_scope, + scope_key_count, + require_cache_enabled=True, +): + if require_cache_enabled and not normalized_settings.get('cache_enabled'): + _record_document_access_cache_metric( + 'bypass', + operation=operation, + source_scope=source_scope, + scope_key_count=scope_key_count, + reason='disabled', + ) + return None + + redis_client = app_settings_cache.get_app_cache_redis_client() + if redis_client is None: + _record_document_access_cache_metric( + 'bypass', + operation=operation, + source_scope=source_scope, + scope_key_count=scope_key_count, + reason='redis_unavailable', + ) + return None + return redis_client + + +def _set_document_access_cache_version_key_if_missing(redis_client, version_key, ttl_seconds): + try: + return redis_client.set(version_key, 0, ex=ttl_seconds, nx=True) + except TypeError: + created = redis_client.setnx(version_key, 0) + if created: + redis_client.expire(version_key, ttl_seconds) + return created + + +def _refresh_document_access_cache_version_key_ttl(redis_client, version_key, ttl_seconds): + return redis_client.expire(version_key, ttl_seconds) + + +def _read_document_access_cache_scope_versions(redis_client, scope_keys, operation, source_scope, version_ttl_seconds): + scope_versions = [] + started_at = perf_counter() + try: + for scope_key in sorted(set(scope_keys or [])): + version_key = _document_access_cache_version_key(scope_key) + version_value = redis_client.get(version_key) + if version_value is None: + created = _set_document_access_cache_version_key_if_missing( + redis_client, + version_key, + version_ttl_seconds, + ) + if created: + version_value = 0 + else: + version_value = redis_client.get(version_key) + if version_value is None: + _set_document_access_cache_version_key_if_missing( + redis_client, + version_key, + version_ttl_seconds, + ) + version_value = 0 + else: + _refresh_document_access_cache_version_key_ttl( + redis_client, + version_key, + version_ttl_seconds, + ) + scope_versions.append({ + 'scope_hash': build_document_access_cache_scope_hash(scope_key), + 'version': _safe_int(version_value), + }) + except Exception as exc: + _record_document_access_cache_metric( + 'version_read_failed', + operation=operation, + source_scope=source_scope, + scope_key_count=len(scope_keys or []), + reason='redis_error', + elapsed_ms=(perf_counter() - started_at) * 1000, + error=exc, + ) + log_event( + '[DocumentAccessIndexCache] Failed to read Redis DAI cache scope versions; DAI query will run uncached.', + extra={ + 'operation': operation, + 'source_scope': source_scope, + 'scope_key_count': len(scope_keys or []), + 'error_type': type(exc).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None + return scope_versions + + +def _build_document_access_cache_context(operation, source_scope, scope_keys, key_payload, normalized_settings): + redis_client = _get_document_access_cache_client( + normalized_settings, + operation, + source_scope, + len(scope_keys or []), + ) + if redis_client is None: + return None + + scope_versions = _read_document_access_cache_scope_versions( + redis_client, + scope_keys, + operation, + source_scope, + normalized_settings.get('cache_version_ttl_seconds'), + ) + if scope_versions is None: + return None + + return { + 'redis_client': redis_client, + 'cache_key': _document_access_cache_entry_key(operation, source_scope, scope_versions, key_payload), + 'operation': operation, + 'source_scope': source_scope, + 'scope_key_count': len(scope_keys or []), + 'ttl_seconds': normalized_settings.get('cache_ttl_seconds'), + } + + +def _try_get_document_access_cache_entry(cache_context): + if not cache_context: + return None + + started_at = perf_counter() + operation = cache_context.get('operation') + source_scope = cache_context.get('source_scope') + scope_key_count = cache_context.get('scope_key_count') + try: + cached_value = cache_context['redis_client'].get(cache_context['cache_key']) + except Exception as exc: + _record_document_access_cache_metric( + 'read_failed', + operation=operation, + source_scope=source_scope, + scope_key_count=scope_key_count, + reason='redis_error', + elapsed_ms=(perf_counter() - started_at) * 1000, + error=exc, + ) + log_event( + '[DocumentAccessIndexCache] Redis DAI cache read failed; DAI query will run uncached.', + extra={ + 'operation': operation, + 'source_scope': source_scope, + 'scope_key_count': scope_key_count, + 'error_type': type(exc).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None + + elapsed_ms = (perf_counter() - started_at) * 1000 + if cached_value is None: + _record_document_access_cache_metric( + 'miss', + operation=operation, + source_scope=source_scope, + scope_key_count=scope_key_count, + elapsed_ms=elapsed_ms, + ) + return None + + try: + payload = _decode_cache_json(cached_value) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + _record_document_access_cache_metric( + 'read_failed', + operation=operation, + source_scope=source_scope, + scope_key_count=scope_key_count, + reason='decode_failed', + elapsed_ms=elapsed_ms, + error=exc, + ) + return None + + _record_document_access_cache_metric( + 'hit', + operation=operation, + source_scope=source_scope, + scope_key_count=scope_key_count, + elapsed_ms=elapsed_ms, + ttl_seconds=cache_context.get('ttl_seconds'), + ) + return payload if isinstance(payload, dict) else None + + +def _try_set_document_access_cache_entry(cache_context, payload): + if not cache_context: + return False + + started_at = perf_counter() + operation = cache_context.get('operation') + source_scope = cache_context.get('source_scope') + scope_key_count = cache_context.get('scope_key_count') + ttl_seconds = _safe_int(cache_context.get('ttl_seconds'), DOCUMENT_ACCESS_DEFAULT_CACHE_TTL_SECONDS) + try: + cache_context['redis_client'].setex( + cache_context['cache_key'], + ttl_seconds, + json.dumps(payload or {}, default=str), + ) + except Exception as exc: + _record_document_access_cache_metric( + 'write_failed', + operation=operation, + source_scope=source_scope, + scope_key_count=scope_key_count, + reason='redis_error', + elapsed_ms=(perf_counter() - started_at) * 1000, + ttl_seconds=ttl_seconds, + error=exc, + ) + log_event( + '[DocumentAccessIndexCache] Redis DAI cache write failed; DAI read result was still returned.', + extra={ + 'operation': operation, + 'source_scope': source_scope, + 'scope_key_count': scope_key_count, + 'error_type': type(exc).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return False + + _record_document_access_cache_metric( + 'write', + operation=operation, + source_scope=source_scope, + scope_key_count=scope_key_count, + elapsed_ms=(perf_counter() - started_at) * 1000, + ttl_seconds=ttl_seconds, + ) + return True + + +def _document_access_scope_keys_from_document_item(document_item): + try: + rows = build_document_access_index_rows(document_item) + except Exception: + rows = [] + return sorted({ + row.get('scope_key') + for row in rows + if row.get('scope_key') + }) + + +def invalidate_document_access_index_cache_scope_keys(scope_keys, reason=None, settings=None): + """Bump Redis-only DAI cache versions for affected access scopes.""" + normalized_scope_keys = sorted({ + str(scope_key or '').strip() + for scope_key in scope_keys or [] + if str(scope_key or '').strip() + }) + if not normalized_scope_keys: + return { + 'success': True, + 'status': 'skipped_no_scope_keys', + 'invalidated_count': 0, + } + + operation = 'scope_invalidation' + source_scope = 'mixed' + normalized_settings = get_document_access_index_settings(settings) + redis_client = _get_document_access_cache_client( + normalized_settings, + operation, + source_scope, + len(normalized_scope_keys), + require_cache_enabled=False, + ) + if redis_client is None: + if normalized_settings.get('cache_enabled') and normalized_settings.get('redis_cache_configured'): + _record_document_access_cache_metric( + 'invalidate_failed', + operation=operation, + source_scope=source_scope, + scope_key_count=len(normalized_scope_keys), + reason='redis_unavailable', + ) + return { + 'success': False, + 'status': 'redis_unavailable', + 'invalidated_count': 0, + } + _bump_document_access_cache_local_epoch() + return { + 'success': True, + 'status': 'skipped_cache_unavailable', + 'invalidated_count': 0, + } + + invalidated_count = 0 + started_at = perf_counter() + try: + version_ttl_seconds = normalized_settings.get('cache_version_ttl_seconds') + for scope_key in normalized_scope_keys: + version_key = _document_access_cache_version_key(scope_key) + redis_client.incr(version_key) + _refresh_document_access_cache_version_key_ttl( + redis_client, + version_key, + version_ttl_seconds, + ) + invalidated_count += 1 + except Exception as exc: + _record_document_access_cache_metric( + 'invalidate_failed', + operation=operation, + source_scope=source_scope, + scope_key_count=len(normalized_scope_keys), + reason=reason or 'redis_error', + elapsed_ms=(perf_counter() - started_at) * 1000, + error=exc, + ) + log_event( + '[DocumentAccessIndexCache] Failed to invalidate Redis DAI cache scope versions.', + extra={ + 'scope_key_count': len(normalized_scope_keys), + 'invalidated_count': invalidated_count, + 'reason': reason, + 'error_type': type(exc).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return { + 'success': False, + 'status': 'invalidate_failed', + 'invalidated_count': invalidated_count, + } + + _record_document_access_cache_metric( + 'invalidate', + operation=operation, + source_scope=source_scope, + scope_key_count=invalidated_count, + reason=reason, + elapsed_ms=(perf_counter() - started_at) * 1000, + ) + return { + 'success': True, + 'status': 'invalidated', + 'invalidated_count': invalidated_count, + } + + +def _normalize_document_access_cache_scope_keys(scope_keys): + return sorted({ + str(scope_key or '').strip() + for scope_key in scope_keys or [] + if str(scope_key or '').strip() + }) + + +def _cache_scope_keys_from_exception(error): + return _normalize_document_access_cache_scope_keys(getattr(error, 'scope_keys', None)) + + +def _raise_for_failed_cache_invalidation(invalidation_result, operation, scope_keys=None): + if not isinstance(invalidation_result, dict) or invalidation_result.get('success'): + return + raise DocumentAccessIndexCacheInvalidationError( + operation, + invalidation_result.get('status') or 'unknown', + scope_keys=scope_keys, + ) + + +def _normalize_document_access_cache_version_hashes(scope_hashes): + return sorted({ + str(scope_hash or '').strip().lower() + for scope_hash in scope_hashes or [] + if re.fullmatch(r'[a-f0-9]{64}', str(scope_hash or '').strip().lower()) + }) + + +def _empty_document_access_cache_version_resolution(scope_hash): + return { + 'kind': 'document_access_index_scope_version', + 'label': 'DAI scope version marker', + 'resolved': False, + 'resolution_status': 'unresolved', + 'scope_hash': scope_hash, + 'scope_key': None, + 'entity_type': None, + 'entity_id': None, + 'entity_name': None, + 'entity_status': None, + 'row_count': None, + 'granted_row_count': None, + 'source_scopes': {}, + 'access_roles': {}, + 'note': 'No matching SimpleChat user, group, public workspace, or DAI projection scope was found.', + } + + +def _document_access_resolution_entity_type(scope_type): + if scope_type == DOCUMENT_ACCESS_SCOPE_GROUP: + return 'group_workspace' + if scope_type == DOCUMENT_ACCESS_SCOPE_PUBLIC: + return 'public_workspace' + if scope_type == DOCUMENT_ACCESS_PRINCIPAL_USER: + return 'user' + return scope_type + + +def _build_document_access_cache_version_resolution( + scope_hash, + scope_key, + entity_type, + entity_id, + entity_name=None, + entity_status=None, + row_count=None, + granted_row_count=None, + source_scopes=None, + access_roles=None, + source='document_access_index', +): + return { + 'kind': 'document_access_index_scope_version', + 'label': 'DAI scope version marker', + 'resolved': True, + 'resolution_status': 'resolved', + 'scope_hash': scope_hash, + 'scope_key': scope_key, + 'entity_type': _document_access_resolution_entity_type(entity_type), + 'entity_id': entity_id, + 'entity_name': entity_name, + 'entity_status': entity_status, + 'row_count': row_count, + 'granted_row_count': granted_row_count, + 'source_scopes': source_scopes or {}, + 'access_roles': access_roles or {}, + 'source': source, + 'note': 'Resolved by re-hashing known SimpleChat access scope keys.', + } + + +def _increment_counter(counter, key): + normalized_key = str(key or '').strip() or 'unknown' + counter[normalized_key] = counter.get(normalized_key, 0) + 1 + + +def _resolve_document_access_hashes_from_projection(scope_hashes, resolutions): + if not scope_hashes: + return + query = ( + 'SELECT c.scope_key, c.scope_type, c.scope_id, c.source_scope, c.access_role, c.access_granted ' + 'FROM c WHERE c.type = @type' + ) + projection_rows = cosmos_document_access_index_container.query_items( + query=query, + parameters=[{'name': '@type', 'value': DOCUMENT_ACCESS_INDEX_TYPE}], + enable_cross_partition_query=True, + ) + scope_summaries = {} + for row in projection_rows: + scope_key = str(row.get('scope_key') or '').strip() + if not scope_key: + continue + scope_hash = build_document_access_cache_scope_hash(scope_key) + if scope_hash not in scope_hashes: + continue + summary = scope_summaries.setdefault(scope_hash, { + 'scope_key': scope_key, + 'entity_type': row.get('scope_type'), + 'entity_id': row.get('scope_id'), + 'row_count': 0, + 'granted_row_count': 0, + 'source_scopes': {}, + 'access_roles': {}, + }) + summary['row_count'] += 1 + if row.get('access_granted'): + summary['granted_row_count'] += 1 + _increment_counter(summary['source_scopes'], row.get('source_scope')) + _increment_counter(summary['access_roles'], row.get('access_role')) + + for scope_hash, summary in scope_summaries.items(): + resolutions[scope_hash] = _build_document_access_cache_version_resolution( + scope_hash, + summary.get('scope_key'), + summary.get('entity_type'), + summary.get('entity_id'), + row_count=summary.get('row_count'), + granted_row_count=summary.get('granted_row_count'), + source_scopes=summary.get('source_scopes'), + access_roles=summary.get('access_roles'), + ) + + +def _resolve_document_access_hashes_from_workspace_container( + scope_hashes, + resolutions, + container, + scope_type, + entity_status_default='active', +): + if not scope_hashes: + return + query = 'SELECT c.id, c.name, c.status FROM c' + for item in container.query_items(query=query, enable_cross_partition_query=True): + entity_id = str(item.get('id') or '').strip() + if not entity_id: + continue + scope_key = build_document_access_scope_key(scope_type, entity_id) + scope_hash = build_document_access_cache_scope_hash(scope_key) + if scope_hash not in scope_hashes: + continue + if resolutions.get(scope_hash, {}).get('resolved'): + resolutions[scope_hash]['entity_name'] = item.get('name') + resolutions[scope_hash]['entity_status'] = item.get('status') or entity_status_default + continue + resolutions[scope_hash] = _build_document_access_cache_version_resolution( + scope_hash, + scope_key, + scope_type, + entity_id, + entity_name=item.get('name'), + entity_status=item.get('status') or entity_status_default, + row_count=0, + granted_row_count=0, + source='workspace_container', + ) + + +def _resolve_document_access_hashes_from_user_settings(scope_hashes, resolutions): + if not scope_hashes: + return + query = 'SELECT c.id, c.displayName, c.name FROM c' + for item in cosmos_user_settings_container.query_items(query=query, enable_cross_partition_query=True): + entity_id = str(item.get('id') or '').strip() + if not entity_id: + continue + scope_key = build_document_access_scope_key(DOCUMENT_ACCESS_PRINCIPAL_USER, entity_id) + scope_hash = build_document_access_cache_scope_hash(scope_key) + if scope_hash not in scope_hashes: + continue + if resolutions.get(scope_hash, {}).get('resolved'): + resolutions[scope_hash]['entity_name'] = item.get('displayName') or item.get('name') + continue + resolutions[scope_hash] = _build_document_access_cache_version_resolution( + scope_hash, + scope_key, + DOCUMENT_ACCESS_PRINCIPAL_USER, + entity_id, + entity_name=item.get('displayName') or item.get('name'), + row_count=0, + granted_row_count=0, + source='user_settings', + ) + + +def resolve_document_access_cache_version_hashes(scope_hashes): + """Resolve Redis DAI version marker hashes to safe SimpleChat scope metadata.""" + normalized_hashes = _normalize_document_access_cache_version_hashes(scope_hashes) + resolutions = { + scope_hash: _empty_document_access_cache_version_resolution(scope_hash) + for scope_hash in normalized_hashes + } + if not normalized_hashes: + return resolutions + + try: + _resolve_document_access_hashes_from_projection(set(normalized_hashes), resolutions) + except Exception as exc: + log_event( + '[DocumentAccessIndexCache] Failed to resolve DAI Redis hashes from projection rows.', + extra={'hash_count': len(normalized_hashes), 'error_type': type(exc).__name__}, + level=logging.WARNING, + exceptionTraceback=True, + ) + + try: + _resolve_document_access_hashes_from_workspace_container( + set(normalized_hashes), + resolutions, + cosmos_groups_container, + DOCUMENT_ACCESS_SCOPE_GROUP, + ) + except Exception as exc: + log_event( + '[DocumentAccessIndexCache] Failed to resolve DAI Redis hashes from group workspaces.', + extra={'hash_count': len(normalized_hashes), 'error_type': type(exc).__name__}, + level=logging.WARNING, + exceptionTraceback=True, + ) + + try: + _resolve_document_access_hashes_from_workspace_container( + set(normalized_hashes), + resolutions, + cosmos_public_workspaces_container, + DOCUMENT_ACCESS_SCOPE_PUBLIC, + ) + except Exception as exc: + log_event( + '[DocumentAccessIndexCache] Failed to resolve DAI Redis hashes from public workspaces.', + extra={'hash_count': len(normalized_hashes), 'error_type': type(exc).__name__}, + level=logging.WARNING, + exceptionTraceback=True, + ) + + try: + _resolve_document_access_hashes_from_user_settings(set(normalized_hashes), resolutions) + except Exception as exc: + log_event( + '[DocumentAccessIndexCache] Failed to resolve DAI Redis hashes from user settings.', + extra={'hash_count': len(normalized_hashes), 'error_type': type(exc).__name__}, + level=logging.WARNING, + exceptionTraceback=True, + ) + + return resolutions + + +def refresh_document_access_cache_version_marker_ttls(settings=None, redis_client=None, batch_size=None): + """Apply the derived TTL policy to legacy DAI Redis version markers.""" + normalized_settings = get_document_access_index_settings(settings) + resolved_client = redis_client or _get_document_access_cache_client( + normalized_settings, + 'version_marker_hygiene', + 'mixed', + 0, + require_cache_enabled=False, + ) + version_ttl_seconds = normalized_settings.get('cache_version_ttl_seconds') + payload_ttl_seconds = normalized_settings.get('cache_ttl_seconds') + result = { + 'success': True, + 'status': 'skipped_redis_unavailable', + 'version_marker_ttl_seconds': version_ttl_seconds, + 'payload_ttl_seconds': payload_ttl_seconds, + 'scanned_count': 0, + 'refreshed_count': 0, + 'no_expiry_count': 0, + 'unsafe_ttl_count': 0, + 'skipped_count': 0, + 'scan_iterations': 0, + 'has_more': False, + } + if resolved_client is None: + return result + + normalized_batch_size = _normalize_positive_int( + batch_size, + DOCUMENT_ACCESS_CACHE_VERSION_HYGIENE_BATCH_SIZE, + min_value=1, + max_value=1000, + ) + cursor = 0 + try: + while result['scan_iterations'] < DOCUMENT_ACCESS_CACHE_VERSION_HYGIENE_MAX_SCAN_ITERATIONS: + cursor, keys = resolved_client.scan( + cursor=cursor, + match=f'{DOCUMENT_ACCESS_CACHE_VERSION_KEY_PREFIX}:*', + count=normalized_batch_size, + ) + cursor = _safe_int(cursor) + result['scan_iterations'] += 1 + for key in list(keys or []): + result['scanned_count'] += 1 + try: + ttl_seconds = int(resolved_client.ttl(key)) + except (TypeError, ValueError): + ttl_seconds = -2 + if ttl_seconds == -1: + result['no_expiry_count'] += 1 + if ttl_seconds == -1 or (ttl_seconds >= 0 and ttl_seconds < payload_ttl_seconds): + if ttl_seconds >= 0 and ttl_seconds < payload_ttl_seconds: + result['unsafe_ttl_count'] += 1 + _refresh_document_access_cache_version_key_ttl( + resolved_client, + key, + version_ttl_seconds, + ) + result['refreshed_count'] += 1 + else: + result['skipped_count'] += 1 + if cursor == 0: + break + result['has_more'] = cursor != 0 + result['status'] = 'completed' + return result + except Exception as exc: + log_event( + '[DocumentAccessIndexCache] Failed to refresh DAI Redis version marker TTLs.', + extra={ + 'scanned_count': result.get('scanned_count'), + 'refreshed_count': result.get('refreshed_count'), + 'error_type': type(exc).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + result['success'] = False + result['status'] = 'failed' + result['error'] = str(exc) + return result + + +def build_document_access_index_row_id(scope_key, source_scope, document_id, version): + """Build a deterministic Cosmos id for one scope/document/version projection row.""" + raw_id = ':'.join([ + 'dai', + _safe_id_part(scope_key), + _safe_id_part(source_scope), + _safe_id_part(document_id), + _safe_id_part(version), + ]) + if len(raw_id) <= 900: + return raw_id + return f'dai:{hashlib.sha256(raw_id.encode("utf-8")).hexdigest()}' + + +def _normalize_string_list(value): + if not isinstance(value, list): + return [] + return [ + str(item).strip() + for item in value + if str(item or '').strip() + ] + + +def _normalize_filter_text(value): + return str(value or '').strip().lower() + + +def _has_persisted_blob_reference(document_item): + if not isinstance(document_item, dict): + return False + if document_item.get('blob_path'): + return True + return ( + document_item.get('blob_path_mode') == DOCUMENT_ACCESS_ARCHIVED_REVISION_BLOB_PATH_MODE + and bool(document_item.get('archived_blob_path')) + ) + + +def _document_identity(document_item): + if not isinstance(document_item, dict): + return None + document_id = str( + document_item.get('document_id') + or document_item.get('source_document_id') + or document_item.get('id') + or '' + ).strip() + try: + version = int(document_item.get('version') or 0) + except (TypeError, ValueError): + version = 0 + if not document_id: + return None + return f'{document_id}:{version}' + + +def _document_family_identity(document_item, source_scope): + if not isinstance(document_item, dict): + return None + revision_family_id = str(document_item.get('revision_family_id') or '').strip() + if revision_family_id: + return revision_family_id + + if source_scope == DOCUMENT_ACCESS_SCOPE_PUBLIC: + scope_value = document_item.get('owner_public_workspace_id') or document_item.get('public_workspace_id') + elif source_scope == DOCUMENT_ACCESS_SCOPE_GROUP: + scope_value = document_item.get('owner_group_id') or document_item.get('group_id') + else: + scope_value = document_item.get('owner_user_id') or document_item.get('user_id') + return f'legacy::{scope_value or "unknown"}::{document_item.get("file_name") or ""}' + + +def _normalize_share_entry(entry): + normalized_entry = str(entry or '').strip() + if not normalized_entry: + return None + if ',' in normalized_entry: + scope_id, approval_status = normalized_entry.split(',', 1) + approval_status = str(approval_status or '').strip().lower() or 'unknown' + else: + scope_id = normalized_entry + approval_status = DOCUMENT_ACCESS_APPROVAL_NOT_APPROVED + + scope_id = str(scope_id or '').strip() + if not scope_id: + return None + return { + 'scope_id': scope_id, + 'approval_status': approval_status, + 'raw_entry': normalized_entry, + } + + +def _resolve_source_scope(document_item): + if document_item.get('public_workspace_id'): + return DOCUMENT_ACCESS_SCOPE_PUBLIC + if document_item.get('group_id'): + return DOCUMENT_ACCESS_SCOPE_GROUP + return DOCUMENT_ACCESS_SCOPE_PERSONAL + + +def _source_scope_container_name(source_scope): + if source_scope == DOCUMENT_ACCESS_SCOPE_PUBLIC: + return 'public_documents' + if source_scope == DOCUMENT_ACCESS_SCOPE_GROUP: + return 'group_documents' + return 'documents' + + +def _get_document_version(document_item): + try: + return int(document_item.get('version') or 0) + except (TypeError, ValueError): + return 0 + + +def _build_base_row(document_item, source_scope, scope_type, scope_id, access_role, approval_status, projected_at): + document_id = str(document_item.get('id') or document_item.get('document_id') or '').strip() + version = _get_document_version(document_item) + scope_key = build_document_access_scope_key(scope_type, scope_id) + if not document_id or not scope_key: + return None + + approval_status = str(approval_status or '').strip().lower() or 'unknown' + access_granted = approval_status == DOCUMENT_ACCESS_APPROVAL_APPROVED + source_updated_at = document_item.get('last_updated') or document_item.get('upload_date') + + return { + 'id': build_document_access_index_row_id(scope_key, source_scope, document_id, version), + 'type': DOCUMENT_ACCESS_INDEX_TYPE, + 'schema_version': DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION, + 'projection_status': 'ready', + 'projection_version': DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION, + 'repair_required': False, + 'projected_at': projected_at, + 'scope_key': scope_key, + 'scope_type': scope_type, + 'scope_id': str(scope_id or '').strip(), + 'source_scope': source_scope, + 'source_container': _source_scope_container_name(source_scope), + 'source_document_id': document_id, + 'source_partition_key': document_id, + 'document_id': document_id, + 'revision_family_id': document_item.get('revision_family_id'), + 'version': version, + 'is_current_version': bool(document_item.get('is_current_version', True)), + 'search_visibility_state': str(document_item.get('search_visibility_state') or 'active').strip().lower(), + 'access_role': access_role, + 'approval_status': approval_status, + 'access_granted': access_granted, + 'shared_user_ids': _normalize_string_list(document_item.get('shared_user_ids')), + 'shared_group_ids': _normalize_string_list(document_item.get('shared_group_ids')), + 'file_name': document_item.get('file_name'), + 'title': document_item.get('title'), + 'document_classification': document_item.get('document_classification'), + 'tags': _normalize_string_list(document_item.get('tags')), + 'authors': _normalize_string_list(document_item.get('authors')), + 'keywords': _normalize_string_list(document_item.get('keywords')), + 'abstract': document_item.get('abstract'), + 'status': document_item.get('status'), + 'percentage_complete': document_item.get('percentage_complete'), + 'number_of_pages': document_item.get('number_of_pages'), + 'publication_date': document_item.get('publication_date'), + 'enhanced_citations': _has_persisted_blob_reference(document_item), + 'document_intelligence_extraction_mode': document_item.get('document_intelligence_extraction_mode'), + 'generated_artifact_promotion_status': document_item.get('generated_artifact_promotion_status'), + 'generated_artifact_requested_by_user_id': document_item.get('generated_artifact_requested_by_user_id'), + 'file_sync': document_item.get('file_sync'), + 'created_from_chat_upload': document_item.get('created_from_chat_upload'), + 'conversation_id': document_item.get('conversation_id'), + 'conversation_title_at_upload': document_item.get('conversation_title_at_upload'), + 'upload_date': document_item.get('upload_date'), + 'last_updated': document_item.get('last_updated'), + 'source_ts': document_item.get('_ts'), + 'source_updated_at': source_updated_at, + 'user_id': document_item.get('user_id'), + 'owner_user_id': document_item.get('user_id'), + 'owner_group_id': document_item.get('group_id'), + 'owner_public_workspace_id': document_item.get('public_workspace_id'), + } + + +def _prefer_projection_row(existing_row, candidate_row): + if not existing_row: + return candidate_row + existing_sort_key = ( + _safe_int(existing_row.get('version')), + str(existing_row.get('upload_date') or ''), + _safe_int(existing_row.get('source_ts')), + ) + candidate_sort_key = ( + _safe_int(candidate_row.get('version')), + str(candidate_row.get('upload_date') or ''), + _safe_int(candidate_row.get('source_ts')), + ) + if candidate_sort_key > existing_sort_key: + return candidate_row + if candidate_sort_key < existing_sort_key: + return existing_row + if existing_row.get('access_role') == 'owner': + return existing_row + if candidate_row.get('access_role') == 'owner': + return candidate_row + if existing_row.get('access_granted') and not candidate_row.get('access_granted'): + return candidate_row + return existing_row + + +def build_document_access_index_rows(document_item): + """Build deterministic per-scope projection rows for one source document.""" + if not isinstance(document_item, dict): + return [] + + source_scope = _resolve_source_scope(document_item) + projected_at = _utc_now_iso() + rows_by_key = {} + + def add_row(scope_type, scope_id, access_role, approval_status): + row = _build_base_row( + document_item, + source_scope, + scope_type, + scope_id, + access_role, + approval_status, + projected_at, + ) + if not row: + return + row_key = (row['scope_key'], row['id']) + rows_by_key[row_key] = _prefer_projection_row(rows_by_key.get(row_key), row) + + if source_scope == DOCUMENT_ACCESS_SCOPE_PUBLIC: + add_row( + DOCUMENT_ACCESS_SCOPE_PUBLIC, + document_item.get('public_workspace_id'), + 'public_workspace', + DOCUMENT_ACCESS_APPROVAL_APPROVED, + ) + elif source_scope == DOCUMENT_ACCESS_SCOPE_GROUP: + add_row( + DOCUMENT_ACCESS_SCOPE_GROUP, + document_item.get('group_id'), + 'owner', + DOCUMENT_ACCESS_APPROVAL_APPROVED, + ) + for share_entry in _normalize_string_list(document_item.get('shared_group_ids')): + normalized_share = _normalize_share_entry(share_entry) + if normalized_share: + add_row( + DOCUMENT_ACCESS_SCOPE_GROUP, + normalized_share['scope_id'], + 'shared_group', + normalized_share['approval_status'], + ) + else: + add_row( + DOCUMENT_ACCESS_PRINCIPAL_USER, + document_item.get('user_id'), + 'owner', + DOCUMENT_ACCESS_APPROVAL_APPROVED, + ) + for share_entry in _normalize_string_list(document_item.get('shared_user_ids')): + normalized_share = _normalize_share_entry(share_entry) + if normalized_share: + add_row( + DOCUMENT_ACCESS_PRINCIPAL_USER, + normalized_share['scope_id'], + 'shared_user', + normalized_share['approval_status'], + ) + + return sorted(rows_by_key.values(), key=lambda row: (row['scope_key'], row['id'])) + + +def _query_existing_projection_rows(source_scope, document_id): + query = ( + 'SELECT c.id, c.scope_key FROM c ' + 'WHERE c.type = @type AND c.source_scope = @source_scope AND c.source_document_id = @document_id' + ) + return list(cosmos_document_access_index_container.query_items( + query=query, + parameters=[ + {'name': '@type', 'value': DOCUMENT_ACCESS_INDEX_TYPE}, + {'name': '@source_scope', 'value': source_scope}, + {'name': '@document_id', 'value': document_id}, + ], + enable_cross_partition_query=True, + )) + + +def _read_shadow_validation_state(use_cache=True): + if use_cache: + cache_hit, cached_state = _get_cached_document_access_state(DOCUMENT_ACCESS_SHADOW_STATE_DOC_ID) + if cache_hit: + return cached_state + try: + state = cosmos_settings_container.read_item( + item=DOCUMENT_ACCESS_SHADOW_STATE_DOC_ID, + partition_key=DOCUMENT_ACCESS_SHADOW_STATE_DOC_ID, + ) + except Exception as exc: + if _is_not_found_error(exc): + _set_cached_document_access_state(DOCUMENT_ACCESS_SHADOW_STATE_DOC_ID, None) + return None + log_event( + '[DocumentAccessIndex] Failed to read document access shadow validation state.', + extra={'error': str(exc)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None + _set_cached_document_access_state(DOCUMENT_ACCESS_SHADOW_STATE_DOC_ID, state) + return state + + +def _empty_shadow_metric_window(window_minutes): + return { + 'window_minutes': window_minutes, + 'sample_count': 0, + 'comparable_sample_count': 0, + 'matched_count': 0, + 'mismatch_count': 0, + 'error_count': 0, + 'source_query_ru': None, + 'validation_index_ru': None, + 'candidate_read_ru': None, + 'estimated_wave5_ru_savings': None, + 'shadow_overhead_ru': None, + 'source_query_ms_avg': None, + 'candidate_read_ms_avg': None, + 'estimated_wave5_ms_savings_avg': None, + 'source_query_item_count': 0, + 'candidate_read_item_count': 0, + 'source_query_page_count': 0, + 'candidate_read_page_count': 0, + 'first_checked_at': None, + 'last_checked_at': None, + } + + +def _empty_shadow_rolling_metrics(): + return { + 'updated_at': None, + 'retention_minutes': DOCUMENT_ACCESS_SHADOW_METRIC_RETENTION_MINUTES, + 'sample_limit': DOCUMENT_ACCESS_SHADOW_METRIC_MAX_SAMPLES, + 'sample_count': 0, + 'windows': { + f'{window_minutes}m': _empty_shadow_metric_window(window_minutes) + for window_minutes in DOCUMENT_ACCESS_SHADOW_METRIC_WINDOWS_MINUTES + }, + } + + +def _build_shadow_metric_sample(state_body): + if not isinstance(state_body, dict): + return None + + metric_keys = ( + 'source_query_ru', + 'validation_index_ru', + 'candidate_read_ru', + 'source_query_ms', + 'candidate_read_ms', + ) + if not any(_safe_float(state_body.get(metric_key)) is not None for metric_key in metric_keys): + return None + + scope_keys = state_body.get('scope_keys') + return { + 'checked_at': state_body.get('checked_at') or _utc_now_iso(), + 'status': str(state_body.get('status') or 'unknown').strip().lower() or 'unknown', + 'context': str(state_body.get('context') or '').strip(), + 'source_scope': str(state_body.get('source_scope') or '').strip().lower(), + 'scope_key_count': len(scope_keys) if isinstance(scope_keys, list) else 0, + 'authoritative_count': _safe_int(state_body.get('authoritative_count')), + 'projection_count': _safe_int(state_body.get('projection_count')), + 'missing_count': _safe_int(state_body.get('missing_count')), + 'extra_count': _safe_int(state_body.get('extra_count')), + 'source_query_ru': _round_metric(state_body.get('source_query_ru')), + 'validation_index_ru': _round_metric(state_body.get('validation_index_ru')), + 'candidate_read_ru': _round_metric(state_body.get('candidate_read_ru')), + 'estimated_wave5_ru_savings': _round_metric(state_body.get('estimated_wave5_ru_savings')), + 'shadow_overhead_ru': _round_metric(state_body.get('shadow_overhead_ru')), + 'source_query_ms': _round_metric(state_body.get('source_query_ms')), + 'candidate_read_ms': _round_metric(state_body.get('candidate_read_ms')), + 'estimated_wave5_ms_savings': _round_metric(state_body.get('estimated_wave5_ms_savings')), + 'source_query_item_count': _safe_int(state_body.get('source_query_item_count')), + 'candidate_read_item_count': _safe_int(state_body.get('candidate_read_item_count')), + 'source_query_page_count': _safe_int(state_body.get('source_query_page_count')), + 'candidate_read_page_count': _safe_int(state_body.get('candidate_read_page_count')), + } + + +def _prune_shadow_metric_samples(samples, now): + cutoff = now - timedelta(minutes=DOCUMENT_ACCESS_SHADOW_METRIC_RETENTION_MINUTES) + pruned_samples = [] + for sample in samples or []: + if not isinstance(sample, dict): + continue + checked_at = _parse_utc_datetime(sample.get('checked_at')) + if not checked_at or checked_at < cutoff: + continue + pruned_samples.append(copy.deepcopy(sample)) + + pruned_samples.sort(key=lambda sample: sample.get('checked_at') or '') + return pruned_samples[-DOCUMENT_ACCESS_SHADOW_METRIC_MAX_SAMPLES:] + + +def _build_shadow_metric_window(samples, now, window_minutes): + cutoff = now - timedelta(minutes=window_minutes) + window_samples = [ + sample + for sample in samples or [] + if (_parse_utc_datetime(sample.get('checked_at')) or datetime.min.replace(tzinfo=timezone.utc)) >= cutoff + ] + if not window_samples: + return _empty_shadow_metric_window(window_minutes) + + paired_ru_samples = [ + sample + for sample in window_samples + if _safe_float(sample.get('source_query_ru')) is not None + and _safe_float(sample.get('candidate_read_ru')) is not None + ] + paired_ms_samples = [ + sample + for sample in window_samples + if _safe_float(sample.get('source_query_ms')) is not None + and _safe_float(sample.get('candidate_read_ms')) is not None + ] + paired_source_ru = _sum_metric_values(paired_ru_samples, 'source_query_ru') + paired_candidate_ru = _sum_metric_values(paired_ru_samples, 'candidate_read_ru') + source_ms_avg = _average_metric_values(paired_ms_samples, 'source_query_ms') + candidate_ms_avg = _average_metric_values(paired_ms_samples, 'candidate_read_ms') + + return { + 'window_minutes': window_minutes, + 'sample_count': len(window_samples), + 'comparable_sample_count': len(paired_ru_samples), + 'matched_count': sum(1 for sample in window_samples if sample.get('status') == 'matched'), + 'mismatch_count': sum(1 for sample in window_samples if sample.get('status') == 'mismatch'), + 'error_count': sum(1 for sample in window_samples if sample.get('status') == 'error'), + 'source_query_ru': paired_source_ru, + 'validation_index_ru': _sum_metric_values(window_samples, 'validation_index_ru'), + 'candidate_read_ru': paired_candidate_ru, + 'estimated_wave5_ru_savings': _metric_delta(paired_source_ru, paired_candidate_ru), + 'shadow_overhead_ru': _sum_metric_values(window_samples, 'shadow_overhead_ru'), + 'source_query_ms_avg': source_ms_avg, + 'candidate_read_ms_avg': candidate_ms_avg, + 'estimated_wave5_ms_savings_avg': _metric_delta(source_ms_avg, candidate_ms_avg), + 'source_query_item_count': _sum_int_values(paired_ru_samples, 'source_query_item_count'), + 'candidate_read_item_count': _sum_int_values(paired_ru_samples, 'candidate_read_item_count'), + 'source_query_page_count': _sum_int_values(paired_ru_samples, 'source_query_page_count'), + 'candidate_read_page_count': _sum_int_values(paired_ru_samples, 'candidate_read_page_count'), + 'first_checked_at': window_samples[0].get('checked_at'), + 'last_checked_at': window_samples[-1].get('checked_at'), + } + + +def _build_shadow_rolling_metrics(samples, now): + return { + 'updated_at': now.isoformat(), + 'retention_minutes': DOCUMENT_ACCESS_SHADOW_METRIC_RETENTION_MINUTES, + 'sample_limit': DOCUMENT_ACCESS_SHADOW_METRIC_MAX_SAMPLES, + 'sample_count': len(samples or []), + 'windows': { + f'{window_minutes}m': _build_shadow_metric_window(samples, now, window_minutes) + for window_minutes in DOCUMENT_ACCESS_SHADOW_METRIC_WINDOWS_MINUTES + }, + } + + +def _merge_shadow_rolling_metrics(state_body, previous_state=None): + previous_state = previous_state or {} + samples = _prune_shadow_metric_samples( + previous_state.get('recent_metric_samples') if isinstance(previous_state, dict) else [], + datetime.now(timezone.utc), + ) + metric_sample = _build_shadow_metric_sample(state_body) + if metric_sample: + samples.append(metric_sample) + now = datetime.now(timezone.utc) + samples = _prune_shadow_metric_samples(samples, now) + state_body['recent_metric_samples'] = samples + state_body['rolling_metrics'] = _build_shadow_rolling_metrics(samples, now) + return state_body + + +def _write_shadow_validation_state(state): + state_body = None + last_conflict = None + + for attempt in range(DOCUMENT_ACCESS_SHADOW_STATE_WRITE_MAX_RETRIES): + previous_state = _read_shadow_validation_state(use_cache=False) + state_body = copy.deepcopy(state or {}) + state_body.update({ + 'id': DOCUMENT_ACCESS_SHADOW_STATE_DOC_ID, + 'type': DOCUMENT_ACCESS_SHADOW_STATE_TYPE, + 'updated_at': _utc_now_iso(), + 'schema_version': DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION, + }) + state_body = _merge_shadow_rolling_metrics(state_body, previous_state=previous_state) + + try: + if previous_state: + replace_kwargs = {} + etag = previous_state.get('_etag') if isinstance(previous_state, dict) else None + if etag: + replace_kwargs['etag'] = etag + replace_kwargs['match_condition'] = MatchConditions.IfNotModified + cosmos_settings_container.replace_item( + item=DOCUMENT_ACCESS_SHADOW_STATE_DOC_ID, + body=state_body, + **replace_kwargs, + ) + else: + cosmos_settings_container.create_item(body=state_body) + _set_cached_document_access_state(DOCUMENT_ACCESS_SHADOW_STATE_DOC_ID, state_body) + return state_body + except Exception as exc: + if _is_write_conflict_error(exc): + last_conflict = exc + log_event( + '[DocumentAccessIndex] Retrying shadow validation state write after ETag conflict.', + extra={'attempt': attempt + 1, 'error': str(exc)}, + level=logging.WARNING, + debug_only=True, + ) + continue + raise + + log_event( + '[DocumentAccessIndex] Failed to persist shadow validation state after ETag conflicts.', + extra={ + 'attempt_count': DOCUMENT_ACCESS_SHADOW_STATE_WRITE_MAX_RETRIES, + 'error': str(last_conflict) if last_conflict else None, + }, + level=logging.WARNING, + ) + return state_body + + +def _query_projection_rows_for_scope(scope_key, source_scope): + rows, _diagnostics = _query_projection_rows_for_scope_with_diagnostics(scope_key, source_scope) + return rows + + +def _query_projection_rows_for_scope_with_diagnostics(scope_key, source_scope): + query = ( + 'SELECT * FROM c ' + 'WHERE c.type = @type ' + 'AND c.source_scope = @source_scope ' + 'AND c.scope_key = @scope_key ' + 'AND (c.access_granted = true OR c.approval_status = @approval_not_approved) ' + 'AND c.is_current_version = true ' + 'AND c.projection_version = @projection_version' + ) + return query_items_with_cosmos_diagnostics( + cosmos_document_access_index_container, + diagnostics_label=f'projection:{scope_key}', + query=query, + parameters=[ + {'name': '@type', 'value': DOCUMENT_ACCESS_INDEX_TYPE}, + {'name': '@source_scope', 'value': source_scope}, + {'name': '@scope_key', 'value': scope_key}, + {'name': '@approval_not_approved', 'value': DOCUMENT_ACCESS_APPROVAL_NOT_APPROVED}, + {'name': '@projection_version', 'value': DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION}, + ], + partition_key=scope_key, + ) + + +def _query_candidate_projection_rows_for_scope(scope_key, source_scope): + query = ( + 'SELECT c.id, c.document_id, c.source_document_id, c.version, c.scope_key, ' + 'c.scope_id, c.access_role, c.approval_status, c.access_granted, ' + 'c.shared_user_ids, c.shared_group_ids, c.file_name, c.title, c.document_classification, c.tags, c.authors, c.keywords, ' + 'c.abstract, c.status, c.percentage_complete, c.number_of_pages, c.publication_date, ' + 'c.enhanced_citations, c.document_intelligence_extraction_mode, ' + 'c.generated_artifact_promotion_status, c.generated_artifact_requested_by_user_id, ' + 'c.file_sync, c.created_from_chat_upload, ' + 'c.conversation_id, c.conversation_title_at_upload, c.upload_date, c.last_updated, ' + 'c.revision_family_id, c.search_visibility_state, c.source_ts, ' + 'c.user_id, c.owner_user_id, c.owner_group_id, c.owner_public_workspace_id ' + 'FROM c ' + 'WHERE c.type = @type ' + 'AND c.source_scope = @source_scope ' + 'AND c.scope_key = @scope_key ' + 'AND (c.access_granted = true OR c.approval_status = @approval_not_approved) ' + 'AND c.is_current_version = true ' + 'AND c.projection_version = @projection_version' + ) + return query_items_with_cosmos_diagnostics( + cosmos_document_access_index_container, + diagnostics_label=f'candidate_read:{scope_key}', + query=query, + parameters=[ + {'name': '@type', 'value': DOCUMENT_ACCESS_INDEX_TYPE}, + {'name': '@source_scope', 'value': source_scope}, + {'name': '@scope_key', 'value': scope_key}, + {'name': '@approval_not_approved', 'value': DOCUMENT_ACCESS_APPROVAL_NOT_APPROVED}, + {'name': '@projection_version', 'value': DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION}, + ], + partition_key=scope_key, + ) + + +def _query_tag_projection_rows_for_scope(scope_key, source_scope): + query = ( + 'SELECT c.document_id, c.source_document_id, c.version, c.scope_key, ' + 'c.scope_id, c.access_role, c.access_granted, c.file_name, c.tags, c.revision_family_id, ' + 'c.source_ts, c.owner_user_id, c.owner_group_id, c.owner_public_workspace_id ' + 'FROM c ' + 'WHERE c.type = @type ' + 'AND c.source_scope = @source_scope ' + 'AND c.scope_key = @scope_key ' + 'AND c.access_granted = true ' + 'AND c.is_current_version = true ' + 'AND c.projection_version = @projection_version ' + 'AND IS_DEFINED(c.tags) ' + 'AND ARRAY_LENGTH(c.tags) > 0' + ) + return query_items_with_cosmos_diagnostics( + cosmos_document_access_index_container, + diagnostics_label=f'tag_read:{scope_key}', + query=query, + parameters=[ + {'name': '@type', 'value': DOCUMENT_ACCESS_INDEX_TYPE}, + {'name': '@source_scope', 'value': source_scope}, + {'name': '@scope_key', 'value': scope_key}, + {'name': '@projection_version', 'value': DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION}, + ], + partition_key=scope_key, + ) + + +def _query_legacy_projection_rows_for_scope(scope_key, source_scope, access_role): + query = ( + 'SELECT c.document_id, c.source_document_id, c.version, c.revision_family_id, ' + 'c.file_name, c.owner_user_id, c.owner_group_id, c.owner_public_workspace_id ' + 'FROM c ' + 'WHERE c.type = @type ' + 'AND c.source_scope = @source_scope ' + 'AND c.scope_key = @scope_key ' + 'AND c.access_role = @access_role ' + 'AND c.access_granted = true ' + 'AND c.projection_version = @projection_version ' + 'AND (NOT IS_DEFINED(c.percentage_complete) OR IS_NULL(c.percentage_complete))' + ) + return list(cosmos_document_access_index_container.query_items( + query=query, + parameters=[ + {'name': '@type', 'value': DOCUMENT_ACCESS_INDEX_TYPE}, + {'name': '@source_scope', 'value': source_scope}, + {'name': '@scope_key', 'value': scope_key}, + {'name': '@access_role', 'value': access_role}, + {'name': '@projection_version', 'value': DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION}, + ], + partition_key=scope_key, + )) + + +def _collect_candidate_read_metrics( + scope_keys, + source_scope, +): + diagnostics = [] + errors = [] + for scope_key in scope_keys: + try: + _rows, scope_diagnostics = _query_candidate_projection_rows_for_scope( + scope_key, + source_scope, + ) + diagnostics.append(scope_diagnostics) + except Exception as exc: + errors.append(str(exc)) + log_event( + '[DocumentAccessIndex] Candidate read metrics query failed.', + extra={ + 'scope_key': scope_key, + 'source_scope': source_scope, + 'error': str(exc), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + combined = _combine_query_diagnostics('candidate_read', diagnostics) + if errors: + combined['errors'] = errors[:DOCUMENT_ACCESS_SHADOW_MAX_SAMPLE_IDS] + combined['error_count'] = len(errors) + combined['partial_failure'] = True + return combined + + +def _projection_row_to_document(row, source_scope): + document_id = str(row.get('source_document_id') or row.get('document_id') or '').strip() + approval_status = str(row.get('approval_status') or DOCUMENT_ACCESS_APPROVAL_APPROVED).strip().lower() + scope_id = str(row.get('scope_id') or '').strip() + access_role = str(row.get('access_role') or '').strip().lower() + document_item = { + 'id': document_id, + 'document_id': document_id, + 'file_name': row.get('file_name'), + 'title': row.get('title'), + 'document_classification': row.get('document_classification'), + 'tags': _normalize_string_list(row.get('tags')), + 'authors': _normalize_string_list(row.get('authors')), + 'keywords': _normalize_string_list(row.get('keywords')), + 'abstract': row.get('abstract'), + 'status': row.get('status'), + 'percentage_complete': row.get('percentage_complete'), + 'number_of_pages': row.get('number_of_pages'), + 'publication_date': row.get('publication_date'), + 'enhanced_citations': row.get('enhanced_citations'), + 'document_intelligence_extraction_mode': row.get('document_intelligence_extraction_mode'), + 'generated_artifact_promotion_status': row.get('generated_artifact_promotion_status'), + 'generated_artifact_requested_by_user_id': row.get('generated_artifact_requested_by_user_id'), + 'file_sync': row.get('file_sync'), + 'created_from_chat_upload': row.get('created_from_chat_upload'), + 'conversation_id': row.get('conversation_id'), + 'conversation_title_at_upload': row.get('conversation_title_at_upload'), + 'upload_date': row.get('upload_date'), + 'last_updated': row.get('last_updated'), + 'revision_family_id': row.get('revision_family_id') or document_id, + 'version': _get_document_version(row), + 'is_current_version': True, + 'search_visibility_state': str(row.get('search_visibility_state') or 'active').strip().lower(), + '_ts': _safe_int(row.get('source_ts')), + } + + if source_scope == DOCUMENT_ACCESS_SCOPE_GROUP: + owner_group_id = row.get('owner_group_id') + document_item['group_id'] = owner_group_id + document_item['owner_group_id'] = owner_group_id + document_item['shared_group_ids'] = _normalize_string_list(row.get('shared_group_ids')) + if access_role == 'shared_group' and scope_id: + shared_entry = f'{scope_id},{approval_status}' + if shared_entry not in document_item['shared_group_ids']: + document_item['shared_group_ids'].append(shared_entry) + elif source_scope == DOCUMENT_ACCESS_SCOPE_PUBLIC: + public_workspace_id = row.get('owner_public_workspace_id') + document_item['public_workspace_id'] = public_workspace_id + document_item['owner_public_workspace_id'] = public_workspace_id + document_item['user_id'] = row.get('user_id') + else: + owner_user_id = row.get('owner_user_id') + document_item['user_id'] = owner_user_id + document_item['owner_user_id'] = owner_user_id + document_item['shared_user_ids'] = _normalize_string_list(row.get('shared_user_ids')) + if access_role == 'shared_user' and scope_id: + shared_entry = f'{scope_id},{approval_status}' + if shared_entry not in document_item['shared_user_ids']: + document_item['shared_user_ids'].append(shared_entry) + + return document_item + + +def _is_backfill_state_ready_for_scope(state, source_scope): + if not isinstance(state, dict): + return False + if _safe_int(state.get('schema_version')) != DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION: + return False + if str(state.get('status') or '').strip().lower() not in DOCUMENT_ACCESS_BACKFILL_READY_STATUSES: + return False + completed_scopes = { + str(scope or '').strip().lower() + for scope in state.get('completed_source_scopes') or [] + } + return source_scope in completed_scopes + + +def _get_document_access_index_readiness(source_scope, settings=None): + normalized_settings = get_document_access_index_settings(settings) + if not normalized_settings.get('container_enabled'): + return { + 'ready': False, + 'status': 'container_disabled', + 'settings': normalized_settings, + } + if not normalized_settings.get('reads_enabled'): + return { + 'ready': False, + 'status': 'reads_disabled', + 'settings': normalized_settings, + } + try: + state = _read_backfill_state() + except Exception as exc: + log_event( + '[DocumentAccessIndex] DAI read path readiness check failed; source document read should be used.', + extra={'source_scope': source_scope, 'error': str(exc)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + return { + 'ready': False, + 'status': 'readiness_check_failed', + 'settings': normalized_settings, + } + if not _is_backfill_state_ready_for_scope(state, source_scope): + return { + 'ready': False, + 'status': 'backfill_not_ready', + 'settings': normalized_settings, + 'backfill_status': (state or {}).get('status'), + } + + has_repair_backlog = has_document_access_index_repair_backlog() + if has_repair_backlog is None: + return { + 'ready': False, + 'status': 'repair_backlog_unknown', + 'settings': normalized_settings, + 'backfill_status': state.get('status'), + } + if has_repair_backlog: + return { + 'ready': False, + 'status': 'repair_backlog_present', + 'settings': normalized_settings, + 'backfill_status': state.get('status'), + } + + return { + 'ready': True, + 'status': 'ready', + 'settings': normalized_settings, + 'backfill_status': state.get('status'), + } + + +def query_document_access_index_documents( + source_scope, + user_id=None, + group_ids=None, + public_workspace_id=None, + public_workspace_ids=None, + filters=None, + settings=None, +): + """Return list-ready documents from DAI when the default read path is ready.""" + source_scope = str(source_scope or '').strip().lower() + if source_scope not in DOCUMENT_ACCESS_SOURCE_SCOPES: + return { + 'success': False, + 'status': 'invalid_source_scope', + 'documents': [], + } + + readiness = _get_document_access_index_readiness(source_scope, settings=settings) + if not readiness.get('ready'): + _record_document_access_read_metric( + 'document_list', + source_scope, + readiness.get('status'), + served_from_index=False, + ) + return { + 'success': False, + 'status': readiness.get('status'), + 'documents': [], + 'readiness': readiness, + } + + scope_keys = [ + scope_key + for scope_key in _build_shadow_scope( + source_scope, + user_id=user_id, + group_ids=group_ids, + public_workspace_id=public_workspace_id, + public_workspace_ids=public_workspace_ids, + ) + if scope_key + ] + if not scope_keys: + _record_document_access_read_metric( + 'document_list', + source_scope, + 'missing_scope_keys', + served_from_index=False, + ) + return { + 'success': False, + 'status': 'missing_scope_keys', + 'documents': [], + 'readiness': readiness, + } + + normalized_settings = readiness.get('settings') if isinstance(readiness.get('settings'), dict) else get_document_access_index_settings(settings) + cache_context = _build_document_access_cache_context( + 'document_list', + source_scope, + scope_keys, + {'filters': filters or {}}, + normalized_settings, + ) + cached_payload = _try_get_document_access_cache_entry(cache_context) + if cached_payload is not None: + cached_documents = copy.deepcopy(cached_payload.get('documents') or []) + cache_diagnostics = { + 'label': 'document_access_index_cache_hit', + 'request_charge': 0, + 'elapsed_ms': None, + 'item_count': len(cached_documents), + 'page_count': 0, + 'cache_hit': True, + } + _record_document_access_read_metric( + 'document_list', + source_scope, + 'served_from_cache', + served_from_index=True, + diagnostics=cache_diagnostics, + scope_key_count=len(scope_keys), + served_from_cache=True, + cache_status='hit', + ) + return { + 'success': True, + 'status': 'served_from_cache', + 'documents': cached_documents, + 'scope_keys': copy.deepcopy(cached_payload.get('scope_keys') or scope_keys), + 'diagnostics': cache_diagnostics, + 'readiness': readiness, + 'cache': { + 'hit': True, + 'ttl_seconds': normalized_settings.get('cache_ttl_seconds'), + }, + } + + projection_rows = [] + diagnostics = [] + try: + for scope_key in scope_keys: + scope_rows, scope_diagnostics = _query_candidate_projection_rows_for_scope( + scope_key, + source_scope, + ) + projection_rows.extend(scope_rows) + diagnostics.append(scope_diagnostics) + except Exception as exc: + log_event( + '[DocumentAccessIndex] DAI read path failed; source document read should be used.', + extra={ + 'source_scope': source_scope, + 'scope_key_count': len(scope_keys), + 'error': str(exc), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + _record_document_access_read_metric( + 'document_list', + source_scope, + 'query_failed', + served_from_index=False, + scope_key_count=len(scope_keys), + error=exc, + ) + return { + 'success': False, + 'status': 'query_failed', + 'documents': [], + 'scope_keys': scope_keys, + 'error': str(exc), + 'readiness': readiness, + } + + rows_by_identity = {} + for row in projection_rows: + if not _matches_shadow_filters(row, filters): + continue + identity = _document_family_identity(row, source_scope) + if not identity: + continue + rows_by_identity[identity] = _prefer_projection_row(rows_by_identity.get(identity), row) + + documents = [ + _projection_row_to_document(row, source_scope) + for row in rows_by_identity.values() + ] + combined_diagnostics = _combine_query_diagnostics('document_access_index_read', diagnostics) + _record_document_access_read_metric( + 'document_list', + source_scope, + 'served_from_index', + served_from_index=True, + diagnostics=combined_diagnostics, + scope_key_count=len(scope_keys), + ) + _try_set_document_access_cache_entry( + cache_context, + { + 'documents': documents, + 'scope_keys': scope_keys, + }, + ) + return { + 'success': True, + 'status': 'served_from_index', + 'documents': documents, + 'scope_keys': scope_keys, + 'diagnostics': combined_diagnostics, + 'readiness': readiness, + 'cache': { + 'hit': False, + 'ttl_seconds': normalized_settings.get('cache_ttl_seconds'), + }, + } + + +def _projection_rows_to_tag_counts(rows, source_scope): + owner_access_roles = {'owner'} + if source_scope == DOCUMENT_ACCESS_SCOPE_PUBLIC: + owner_access_roles.add('public_workspace') + + rows_by_identity = {} + for row in rows or []: + access_role = str(row.get('access_role') or '').strip().lower() + if access_role not in owner_access_roles: + continue + identity = _document_family_identity(row, source_scope) + if not identity: + continue + rows_by_identity[identity] = _prefer_projection_row(rows_by_identity.get(identity), row) + + tag_counts = {} + for row in rows_by_identity.values(): + for tag in _normalize_string_list(row.get('tags')): + normalized_tag = _normalize_filter_text(tag) + if normalized_tag: + tag_counts[normalized_tag] = tag_counts.get(normalized_tag, 0) + 1 + + return tag_counts + + +def _merge_tag_counts(tag_count_items): + merged_counts = {} + for tag_counts in tag_count_items or []: + for tag_name, count in (tag_counts or {}).items(): + normalized_tag = _normalize_filter_text(tag_name) + if not normalized_tag: + continue + merged_counts[normalized_tag] = merged_counts.get(normalized_tag, 0) + _safe_int(count) + return merged_counts + + +def query_document_access_index_tag_counts( + source_scope, + user_id=None, + group_ids=None, + public_workspace_id=None, + public_workspace_ids=None, + settings=None, +): + """Return document tag counts from DAI when the default read path is ready.""" + source_scope = str(source_scope or '').strip().lower() + if source_scope not in DOCUMENT_ACCESS_SOURCE_SCOPES: + return { + 'success': False, + 'status': 'invalid_source_scope', + 'tag_counts': {}, + 'tag_counts_by_scope_key': {}, + } + + readiness = _get_document_access_index_readiness(source_scope, settings=settings) + if not readiness.get('ready'): + _record_document_access_read_metric( + 'tag_list', + source_scope, + readiness.get('status'), + served_from_index=False, + ) + return { + 'success': False, + 'status': readiness.get('status'), + 'tag_counts': {}, + 'tag_counts_by_scope_key': {}, + 'readiness': readiness, + } + + scope_keys = [ + scope_key + for scope_key in _build_shadow_scope( + source_scope, + user_id=user_id, + group_ids=group_ids, + public_workspace_id=public_workspace_id, + public_workspace_ids=public_workspace_ids, + ) + if scope_key + ] + if not scope_keys: + _record_document_access_read_metric( + 'tag_list', + source_scope, + 'missing_scope_keys', + served_from_index=False, + ) + return { + 'success': False, + 'status': 'missing_scope_keys', + 'tag_counts': {}, + 'tag_counts_by_scope_key': {}, + 'readiness': readiness, + } + + normalized_settings = readiness.get('settings') if isinstance(readiness.get('settings'), dict) else get_document_access_index_settings(settings) + cache_context = _build_document_access_cache_context( + 'tag_list', + source_scope, + scope_keys, + {}, + normalized_settings, + ) + cached_payload = _try_get_document_access_cache_entry(cache_context) + if cached_payload is not None: + cached_tag_counts = copy.deepcopy(cached_payload.get('tag_counts') or {}) + cached_tag_counts_by_scope_key = copy.deepcopy(cached_payload.get('tag_counts_by_scope_key') or {}) + cache_diagnostics = { + 'label': 'document_access_index_tag_cache_hit', + 'request_charge': 0, + 'elapsed_ms': None, + 'item_count': len(cached_tag_counts), + 'page_count': 0, + 'cache_hit': True, + } + _record_document_access_read_metric( + 'tag_list', + source_scope, + 'served_from_cache', + served_from_index=True, + diagnostics=cache_diagnostics, + scope_key_count=len(scope_keys), + served_from_cache=True, + cache_status='hit', + ) + return { + 'success': True, + 'status': 'served_from_cache', + 'tag_counts': cached_tag_counts, + 'tag_counts_by_scope_key': cached_tag_counts_by_scope_key, + 'scope_keys': copy.deepcopy(cached_payload.get('scope_keys') or scope_keys), + 'readiness': readiness, + 'cache': { + 'hit': True, + 'ttl_seconds': normalized_settings.get('cache_ttl_seconds'), + }, + } + + diagnostics = [] + tag_counts_by_scope_key = {} + try: + for scope_key in scope_keys: + scope_rows, scope_diagnostics = _query_tag_projection_rows_for_scope( + scope_key, + source_scope, + ) + tag_counts_by_scope_key[scope_key] = _projection_rows_to_tag_counts( + scope_rows, + source_scope, + ) + diagnostics.append(scope_diagnostics) + except Exception as exc: + log_event( + '[DocumentAccessIndex] DAI tag read path failed; source document tag read should be used.', + extra={ + 'source_scope': source_scope, + 'scope_key_count': len(scope_keys), + 'error': str(exc), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + _record_document_access_read_metric( + 'tag_list', + source_scope, + 'query_failed', + served_from_index=False, + scope_key_count=len(scope_keys), + error=exc, + ) + return { + 'success': False, + 'status': 'query_failed', + 'tag_counts': {}, + 'tag_counts_by_scope_key': {}, + 'scope_keys': scope_keys, + 'error': str(exc), + 'readiness': readiness, + } + + combined_diagnostics = _combine_query_diagnostics('document_access_index_tag_read', diagnostics) + _record_document_access_read_metric( + 'tag_list', + source_scope, + 'served_from_index', + served_from_index=True, + diagnostics=combined_diagnostics, + scope_key_count=len(scope_keys), + ) + result = { + 'success': True, + 'status': 'served_from_index', + 'tag_counts': _merge_tag_counts(tag_counts_by_scope_key.values()), + 'tag_counts_by_scope_key': tag_counts_by_scope_key, + 'scope_keys': scope_keys, + 'cache': { + 'hit': False, + 'ttl_seconds': normalized_settings.get('cache_ttl_seconds'), + }, + } + _try_set_document_access_cache_entry( + cache_context, + { + 'tag_counts': result['tag_counts'], + 'tag_counts_by_scope_key': tag_counts_by_scope_key, + 'scope_keys': scope_keys, + }, + ) + return result + + +def query_document_access_index_legacy_count( + source_scope, + user_id=None, + group_ids=None, + public_workspace_id=None, + public_workspace_ids=None, + settings=None, +): + """Return the unfiltered owner-scope legacy document count from DAI.""" + source_scope = str(source_scope or '').strip().lower() + if source_scope not in DOCUMENT_ACCESS_SOURCE_SCOPES: + return { + 'success': False, + 'status': 'invalid_source_scope', + 'legacy_count': 0, + } + + readiness = _get_document_access_index_readiness(source_scope, settings=settings) + if not readiness.get('ready'): + return { + 'success': False, + 'status': readiness.get('status'), + 'legacy_count': 0, + 'readiness': readiness, + } + + scope_keys = [ + scope_key + for scope_key in _build_shadow_scope( + source_scope, + user_id=user_id, + group_ids=group_ids, + public_workspace_id=public_workspace_id, + public_workspace_ids=public_workspace_ids, + ) + if scope_key + ] + if not scope_keys: + return { + 'success': False, + 'status': 'missing_scope_keys', + 'legacy_count': 0, + 'readiness': readiness, + } + + normalized_settings = readiness.get('settings') if isinstance(readiness.get('settings'), dict) else get_document_access_index_settings(settings) + access_role = 'public_workspace' if source_scope == DOCUMENT_ACCESS_SCOPE_PUBLIC else 'owner' + cache_context = _build_document_access_cache_context( + 'legacy_count', + source_scope, + scope_keys, + {'access_role': access_role}, + normalized_settings, + ) + cached_payload = _try_get_document_access_cache_entry(cache_context) + if cached_payload is not None: + return { + 'success': True, + 'status': 'served_from_cache', + 'legacy_count': _safe_int(cached_payload.get('legacy_count')), + 'scope_keys': copy.deepcopy(cached_payload.get('scope_keys') or scope_keys), + 'readiness': readiness, + 'cache': { + 'hit': True, + 'ttl_seconds': normalized_settings.get('cache_ttl_seconds'), + }, + } + + legacy_identities = set() + try: + for scope_key in scope_keys: + for row in _query_legacy_projection_rows_for_scope(scope_key, source_scope, access_role): + identity = _document_identity(row) + if identity: + legacy_identities.add(identity) + except Exception as exc: + log_event( + '[DocumentAccessIndex] DAI legacy count query failed; legacy update prompt should use safe default.', + extra={ + 'source_scope': source_scope, + 'scope_key_count': len(scope_keys), + 'error': str(exc), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return { + 'success': False, + 'status': 'query_failed', + 'legacy_count': 0, + 'scope_keys': scope_keys, + 'error': str(exc), + 'readiness': readiness, + } + + result = { + 'success': True, + 'status': 'served_from_index', + 'legacy_count': len(legacy_identities), + 'scope_keys': scope_keys, + 'readiness': readiness, + 'cache': { + 'hit': False, + 'ttl_seconds': normalized_settings.get('cache_ttl_seconds'), + }, + } + _try_set_document_access_cache_entry( + cache_context, + { + 'legacy_count': result['legacy_count'], + 'scope_keys': scope_keys, + }, + ) + return result + + +def _matches_shadow_filters(row, filters): + filters = filters or {} + array_match_mode = str(filters.get('array_match_mode') or 'contains').strip().lower() + + def array_matches(items, filter_value): + if array_match_mode == 'exact': + raw_filter = str(filter_value or '').strip() + if not raw_filter: + return True + raw_items = [str(item or '').strip() for item in items or []] + return any(item == raw_filter for item in raw_items) + + normalized_filter = _normalize_filter_text(filter_value) + if not normalized_filter: + return True + normalized_items = [_normalize_filter_text(item) for item in items or []] + return any(normalized_filter in item for item in normalized_items) + + search_term = _normalize_filter_text(filters.get('search')) + if search_term: + searchable_text = ' '.join([ + _normalize_filter_text(row.get('file_name')), + _normalize_filter_text(row.get('title')), + ]) + if search_term not in searchable_text: + return False + + classification_filter_raw = str(filters.get('classification') or '').strip() + if classification_filter_raw: + classification_filter = _normalize_filter_text(classification_filter_raw) + classification = str(row.get('document_classification') or '').strip() + if classification_filter == 'none': + normalized_classification = _normalize_filter_text(classification) + none_values = {'', 'none'} if filters.get('classification_none_matches_literal') else {''} + if normalized_classification not in none_values: + return False + elif classification != classification_filter_raw: + return False + + if not array_matches(row.get('authors'), filters.get('author')): + return False + + if not array_matches(row.get('keywords'), filters.get('keywords')): + return False + + abstract_filter = _normalize_filter_text(filters.get('abstract')) + if abstract_filter and abstract_filter not in _normalize_filter_text(row.get('abstract')): + return False + + tag_filters = [ + _normalize_filter_text(tag) + for tag in filters.get('tags') or [] + if _normalize_filter_text(tag) + ] + if tag_filters: + row_tags = {_normalize_filter_text(tag) for tag in row.get('tags') or []} + if not all(tag in row_tags for tag in tag_filters): + return False + + return True + + +def _build_shadow_scope(source_scope, user_id=None, group_ids=None, public_workspace_id=None, public_workspace_ids=None): + if source_scope == DOCUMENT_ACCESS_SCOPE_PUBLIC: + workspace_ids = list(public_workspace_ids or []) + if not workspace_ids and public_workspace_id: + workspace_ids = [public_workspace_id] + return [ + build_document_access_scope_key(DOCUMENT_ACCESS_SCOPE_PUBLIC, workspace_id) + for workspace_id in workspace_ids + ] + if source_scope == DOCUMENT_ACCESS_SCOPE_GROUP: + return [ + build_document_access_scope_key(DOCUMENT_ACCESS_SCOPE_GROUP, group_id) + for group_id in list(group_ids or []) + ] + return [build_document_access_scope_key(DOCUMENT_ACCESS_PRINCIPAL_USER, user_id)] + + +def validate_document_access_index_shadow( + authoritative_documents, + source_scope, + user_id=None, + group_ids=None, + public_workspace_id=None, + public_workspace_ids=None, + filters=None, + source_query_metrics=None, + settings=None, + context='document_list', +): + """Compare source document-list results with projection rows without changing reads.""" + normalized_settings = get_document_access_index_settings(settings) + source_scope = str(source_scope or '').strip().lower() + if not normalized_settings.get('container_enabled') or not normalized_settings.get('shadow_validation_enabled'): + return { + 'success': True, + 'status': 'skipped_disabled', + 'context': context, + } + if source_scope not in DOCUMENT_ACCESS_SOURCE_SCOPES: + return { + 'success': False, + 'status': 'skipped_invalid_scope', + 'context': context, + } + + scope_keys = [ + scope_key + for scope_key in _build_shadow_scope( + source_scope, + user_id=user_id, + group_ids=group_ids, + public_workspace_id=public_workspace_id, + public_workspace_ids=public_workspace_ids, + ) + if scope_key + ] + if not scope_keys: + return { + 'success': False, + 'status': 'skipped_missing_scope', + 'context': context, + } + + projection_query_metrics = None + candidate_read_metrics = None + try: + authoritative_ids = { + identity + for identity in ( + _document_family_identity(document_item, source_scope) + for document_item in authoritative_documents or [] + ) + if identity + } + projection_rows = [] + projection_scope_diagnostics = [] + for scope_key in scope_keys: + scope_rows, scope_diagnostics = _query_projection_rows_for_scope_with_diagnostics(scope_key, source_scope) + projection_rows.extend(scope_rows) + projection_scope_diagnostics.append(scope_diagnostics) + projection_query_metrics = _combine_query_diagnostics( + 'projection', + projection_scope_diagnostics, + ) + candidate_read_metrics = _collect_candidate_read_metrics( + scope_keys, + source_scope, + ) + + projection_ids = { + identity + for identity in ( + _document_family_identity(row, source_scope) + for row in projection_rows + if _matches_shadow_filters(row, filters) + ) + if identity + } + missing_from_projection = sorted(authoritative_ids - projection_ids) + extra_in_projection = sorted(projection_ids - authoritative_ids) + status = 'matched' if not missing_from_projection and not extra_in_projection else 'mismatch' + result = { + 'success': status == 'matched', + 'status': status, + 'context': context, + 'source_scope': source_scope, + 'scope_keys': scope_keys, + 'authoritative_count': len(authoritative_ids), + 'projection_count': len(projection_ids), + 'missing_count': len(missing_from_projection), + 'extra_count': len(extra_in_projection), + 'missing_sample': missing_from_projection[:DOCUMENT_ACCESS_SHADOW_MAX_SAMPLE_IDS], + 'extra_sample': extra_in_projection[:DOCUMENT_ACCESS_SHADOW_MAX_SAMPLE_IDS], + 'checked_at': _utc_now_iso(), + } + result.update(_build_shadow_metric_fields( + source_query_metrics, + projection_query_metrics, + candidate_read_metrics, + )) + _write_shadow_validation_state(result) + log_level = logging.INFO if status == 'matched' else logging.WARNING + log_event( + '[DocumentAccessIndex] Shadow validation completed.', + extra=result, + level=log_level, + ) + return result + except Exception as exc: + result = { + 'success': False, + 'status': 'error', + 'context': context, + 'source_scope': source_scope, + 'scope_keys': scope_keys, + 'error': str(exc), + 'checked_at': _utc_now_iso(), + } + result.update(_build_shadow_metric_fields( + source_query_metrics, + projection_query_metrics, + candidate_read_metrics, + )) + try: + _write_shadow_validation_state(result) + except Exception as state_error: + log_event( + '[DocumentAccessIndex] Failed to persist shadow validation error state.', + extra={'error': str(state_error), 'shadow_error': str(exc)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + log_event( + '[DocumentAccessIndex] Shadow validation failed; source document results remain authoritative.', + extra=result, + level=logging.WARNING, + exceptionTraceback=True, + ) + return result + + +def _repair_document_id(source_scope, document_id): + raw_id = f'document_access_projection_repair:{source_scope}:{document_id}' + if len(raw_id) <= 900: + return raw_id + return f'document_access_projection_repair:{hashlib.sha256(raw_id.encode("utf-8")).hexdigest()}' + + +def _record_projection_repair_required(document_item, operation, error): + document_item = document_item if isinstance(document_item, dict) else {} + document_id = str(document_item.get('id') or document_item.get('document_id') or '').strip() + if not document_id: + return None + source_scope = _resolve_source_scope(document_item) + repair_doc_id = _repair_document_id(source_scope, document_id) + repair_doc = { + 'id': repair_doc_id, + 'type': DOCUMENT_ACCESS_REPAIR_TYPE, + 'status': 'repair_required', + 'operation': operation, + 'source_scope': source_scope, + 'source_document_id': document_id, + 'source_updated_at': document_item.get('last_updated') or document_item.get('upload_date'), + 'error': str(error), + 'updated_at': _utc_now_iso(), + 'schema_version': DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION, + } + cache_scope_keys = _cache_scope_keys_from_exception(error) + if cache_scope_keys: + repair_doc['cache_scope_keys'] = cache_scope_keys + try: + cosmos_settings_container.upsert_item(repair_doc) + except Exception: + _try_write_repair_backlog_state( + True, + reason=f'{operation}_repair_doc_write_failed', + repair_tracking_failed=True, + ) + raise + + if _try_write_repair_backlog_state(True, reason=operation) is None: + _try_delete_repair_backlog_state(reason=f'{operation}_repair_state_untrusted') + return repair_doc + + +def _clear_projection_repair_required(source_scope, document_id): + repair_doc_id = _repair_document_id(source_scope, document_id) + try: + cosmos_settings_container.delete_item(item=repair_doc_id, partition_key=repair_doc_id) + except Exception as exc: + if _is_not_found_error(exc): + pass + else: + log_event( + '[DocumentAccessIndex] Failed to clear projection repair state.', + extra={ + 'source_scope': source_scope, + 'document_id': document_id, + 'error': str(exc), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return + + try: + has_backlog = _query_repair_backlog_exists() + except Exception as exc: + log_event( + '[DocumentAccessIndex] Failed to refresh projection repair backlog state after repair clear.', + extra={ + 'source_scope': source_scope, + 'document_id': document_id, + 'error': str(exc), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return + + if not has_backlog: + try: + state = _read_repair_backlog_state() + except Exception as exc: + log_event( + '[DocumentAccessIndex] Failed to verify repair backlog state after repair clear.', + extra={ + 'source_scope': source_scope, + 'document_id': document_id, + 'error': str(exc), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return + if _repair_backlog_state_has_untracked_failure(state): + return + + updated_state = _try_write_repair_backlog_state(has_backlog, reason='repair_cleared') + if updated_state is None and not has_backlog: + _try_delete_repair_backlog_state(reason='repair_cleared_write_failed') + + +def sync_document_access_index_for_document( + document_item, + operation=DOCUMENT_ACCESS_OPERATION_UPSERT, + settings=None, + force=False, +): + """Synchronize access-index rows for one source document.""" + if not _is_write_through_enabled(settings, force=force): + return { + 'success': True, + 'status': 'skipped_disabled', + 'operation': operation, + 'upserted_count': 0, + 'deleted_count': 0, + } + + rows = build_document_access_index_rows(document_item) + document_id = str((document_item or {}).get('id') or (document_item or {}).get('document_id') or '').strip() + if not document_id: + raise ValueError('document_item must include id or document_id') + + source_scope = _resolve_source_scope(document_item) + expected_keys = {(row['scope_key'], row['id']) for row in rows} + existing_rows = _query_existing_projection_rows(source_scope, document_id) + affected_scope_keys = sorted({ + row.get('scope_key') + for row in rows + existing_rows + if row.get('scope_key') + }) + deleted_count = 0 + + try: + for existing_row in existing_rows: + existing_key = (existing_row.get('scope_key'), existing_row.get('id')) + if existing_key in expected_keys: + continue + cosmos_document_access_index_container.delete_item( + item=existing_row.get('id'), + partition_key=existing_row.get('scope_key'), + ) + deleted_count += 1 + + for row in rows: + cosmos_document_access_index_container.upsert_item(copy.deepcopy(row)) + + _clear_projection_repair_required(source_scope, document_id) + invalidation_result = invalidate_document_access_index_cache_scope_keys( + affected_scope_keys, + reason=operation, + settings=settings, + ) + _raise_for_failed_cache_invalidation(invalidation_result, operation, affected_scope_keys) + except DocumentAccessIndexCacheInvalidationError: + raise + except Exception as exc: + raise DocumentAccessIndexProjectionMutationError(operation, exc, affected_scope_keys) from exc + + return { + 'success': True, + 'status': 'synchronized', + 'operation': operation, + 'source_scope': source_scope, + 'document_id': document_id, + 'upserted_count': len(rows), + 'deleted_count': deleted_count, + } + + +def delete_document_access_index_for_document( + document_item, + operation=DOCUMENT_ACCESS_OPERATION_DELETE, + settings=None, + force=False, +): + """Delete all access-index rows for one source document.""" + if not _is_write_through_enabled(settings, force=force): + return { + 'success': True, + 'status': 'skipped_disabled', + 'operation': operation, + 'deleted_count': 0, + } + + document_id = str((document_item or {}).get('id') or (document_item or {}).get('document_id') or '').strip() + if not document_id: + raise ValueError('document_item must include id or document_id') + + source_scope = _resolve_source_scope(document_item) + existing_rows = _query_existing_projection_rows(source_scope, document_id) + affected_scope_keys = sorted({ + existing_row.get('scope_key') + for existing_row in existing_rows + if existing_row.get('scope_key') + }) or _document_access_scope_keys_from_document_item(document_item) + deleted_count = 0 + try: + for existing_row in existing_rows: + cosmos_document_access_index_container.delete_item( + item=existing_row.get('id'), + partition_key=existing_row.get('scope_key'), + ) + deleted_count += 1 + + _clear_projection_repair_required(source_scope, document_id) + invalidation_result = invalidate_document_access_index_cache_scope_keys( + affected_scope_keys, + reason=operation, + settings=settings, + ) + _raise_for_failed_cache_invalidation(invalidation_result, operation, affected_scope_keys) + except DocumentAccessIndexCacheInvalidationError: + raise + except Exception as exc: + raise DocumentAccessIndexProjectionMutationError(operation, exc, affected_scope_keys) from exc + + return { + 'success': True, + 'status': 'deleted', + 'operation': operation, + 'source_scope': source_scope, + 'document_id': document_id, + 'deleted_count': deleted_count, + } + + +def sync_document_access_index_for_document_fail_open(document_item, operation=DOCUMENT_ACCESS_OPERATION_UPSERT, settings=None): + """Synchronize projection rows without failing the source document mutation.""" + try: + return sync_document_access_index_for_document(document_item, operation=operation, settings=settings) + except Exception as exc: + cache_scope_keys = ( + _cache_scope_keys_from_exception(exc) + or _document_access_scope_keys_from_document_item(document_item) + ) + invalidate_document_access_index_cache_scope_keys( + cache_scope_keys, + reason=f'{operation}_failed', + settings=settings, + ) + log_event( + '[DocumentAccessIndex] Document access index synchronization failed; source document remains authoritative.', + extra={ + 'document_id': (document_item or {}).get('id'), + 'operation': operation, + 'error': str(exc), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + try: + _record_projection_repair_required(document_item, operation, exc) + except Exception as repair_exc: + log_event( + '[DocumentAccessIndex] Failed to record projection repair state.', + extra={ + 'document_id': (document_item or {}).get('id'), + 'operation': operation, + 'error': str(repair_exc), + }, + level=logging.ERROR, + exceptionTraceback=True, + ) + return { + 'success': False, + 'status': 'repair_required', + 'operation': operation, + 'error': str(exc), + } + + +def delete_document_access_index_for_document_fail_open(document_item, operation=DOCUMENT_ACCESS_OPERATION_DELETE, settings=None): + """Delete projection rows without failing the source document mutation.""" + try: + return delete_document_access_index_for_document(document_item, operation=operation, settings=settings) + except Exception as exc: + cache_scope_keys = ( + _cache_scope_keys_from_exception(exc) + or _document_access_scope_keys_from_document_item(document_item) + ) + invalidate_document_access_index_cache_scope_keys( + cache_scope_keys, + reason=f'{operation}_failed', + settings=settings, + ) + log_event( + '[DocumentAccessIndex] Document access index delete failed; source document remains authoritative.', + extra={ + 'document_id': (document_item or {}).get('id'), + 'operation': operation, + 'error': str(exc), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + try: + _record_projection_repair_required(document_item, operation, exc) + except Exception as repair_exc: + log_event( + '[DocumentAccessIndex] Failed to record projection delete repair state.', + extra={ + 'document_id': (document_item or {}).get('id'), + 'operation': operation, + 'error': str(repair_exc), + }, + level=logging.ERROR, + exceptionTraceback=True, + ) + return { + 'success': False, + 'status': 'repair_required', + 'operation': operation, + 'error': str(exc), + } + + +def _write_repair_backlog_state( + has_repair_backlog, + repair_required_count=None, + reason=None, + repair_tracking_failed=False, +): + state_doc = { + 'id': DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, + 'type': DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_TYPE, + 'has_repair_backlog': bool(has_repair_backlog), + 'updated_at': _utc_now_iso(), + 'schema_version': DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION, + } + if repair_required_count is not None: + state_doc['repair_required_count'] = _safe_int(repair_required_count) + if reason: + state_doc['reason'] = str(reason) + if repair_tracking_failed: + state_doc['repair_tracking_failed'] = True + cosmos_settings_container.upsert_item(state_doc) + _set_cached_document_access_state(DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, state_doc) + return state_doc + + +def _try_write_repair_backlog_state( + has_repair_backlog, + repair_required_count=None, + reason=None, + repair_tracking_failed=False, +): + try: + return _write_repair_backlog_state( + has_repair_backlog, + repair_required_count=repair_required_count, + reason=reason, + repair_tracking_failed=repair_tracking_failed, + ) + except Exception as exc: + log_event( + '[DocumentAccessIndex] Failed to update projection repair backlog state.', + extra={ + 'has_repair_backlog': bool(has_repair_backlog), + 'repair_required_count': repair_required_count, + 'reason': reason, + 'repair_tracking_failed': bool(repair_tracking_failed), + 'error': str(exc), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None + + +def _try_delete_repair_backlog_state(reason=None): + try: + cosmos_settings_container.delete_item( + item=DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, + partition_key=DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, + ) + _set_cached_document_access_state(DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, None) + return True + except Exception as exc: + if _is_not_found_error(exc): + _set_cached_document_access_state(DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, None) + return True + log_event( + '[DocumentAccessIndex] Failed to delete projection repair backlog state.', + extra={ + 'reason': reason, + 'error': str(exc), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return False + + +def _read_repair_backlog_state(use_cache=True): + if use_cache: + cache_hit, cached_state = _get_cached_document_access_state(DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID) + if cache_hit: + return cached_state + try: + state = cosmos_settings_container.read_item( + item=DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, + partition_key=DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, + ) + except Exception as exc: + if _is_not_found_error(exc): + _set_cached_document_access_state(DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, None) + return None + raise + if not isinstance(state, dict) or state.get('type') != DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_TYPE: + _set_cached_document_access_state(DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, None) + return None + if not isinstance(state.get('has_repair_backlog'), bool): + _set_cached_document_access_state(DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, None) + return None + _set_cached_document_access_state(DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, state) + return state + + +def _repair_backlog_state_has_untracked_failure(state): + return ( + isinstance(state, dict) + and bool(state.get('has_repair_backlog')) + and bool(state.get('repair_tracking_failed')) + ) + + +def _query_repair_backlog_exists(): + query = ( + 'SELECT TOP 1 VALUE c.id FROM c ' + 'WHERE c.type = @type AND c.status = @status' + ) + repair_ids = list(cosmos_settings_container.query_items( + query=query, + parameters=[ + {'name': '@type', 'value': DOCUMENT_ACCESS_REPAIR_TYPE}, + {'name': '@status', 'value': 'repair_required'}, + ], + enable_cross_partition_query=True, + max_item_count=1, + )) + return bool(repair_ids) + + +def _refresh_repair_backlog_state_from_query(reason=None): + existing_state = _read_repair_backlog_state(use_cache=False) + has_backlog = _query_repair_backlog_exists() + if not has_backlog and _repair_backlog_state_has_untracked_failure(existing_state): + return True + _write_repair_backlog_state(has_backlog, reason=reason or 'repair_backlog_query') + return has_backlog + + +def _get_source_scope_container(source_scope): + if source_scope == DOCUMENT_ACCESS_SCOPE_GROUP: + return cosmos_group_documents_container, cosmos_group_documents_container_name + if source_scope == DOCUMENT_ACCESS_SCOPE_PUBLIC: + return cosmos_public_documents_container, cosmos_public_documents_container_name + return cosmos_user_documents_container, cosmos_user_documents_container_name + + +def _normalize_source_scopes(source_scopes=None): + if not source_scopes: + return list(DOCUMENT_ACCESS_SOURCE_SCOPES) + if isinstance(source_scopes, str): + source_scopes = [source_scopes] + normalized_scopes = [] + for source_scope in list(source_scopes or []): + normalized_scope = str(source_scope or '').strip().lower() + if normalized_scope in DOCUMENT_ACCESS_SOURCE_SCOPES and normalized_scope not in normalized_scopes: + normalized_scopes.append(normalized_scope) + return normalized_scopes or list(DOCUMENT_ACCESS_SOURCE_SCOPES) + + +def _query_first_page(container, query, parameters=None, continuation_token=None, max_item_count=100): + query_iterable = container.query_items( + query=query, + parameters=list(parameters or []), + enable_cross_partition_query=True, + max_item_count=max_item_count, + ) + if not hasattr(query_iterable, 'by_page'): + return list(query_iterable), None + + try: + page_iterator = query_iterable.by_page(continuation_token=continuation_token) + except TypeError: + page_iterator = query_iterable.by_page(continuation_token) + try: + page = next(page_iterator) + except StopIteration: + return [], None + + return list(page), getattr(page_iterator, 'continuation_token', None) + + +def _query_source_document_page(source_scope, continuation_token=None, max_item_count=100): + source_container, source_container_name = _get_source_scope_container(source_scope) + query = ( + 'SELECT * FROM c ' + 'WHERE (NOT IS_DEFINED(c.type) OR c.type = @document_metadata_type)' + ) + documents, next_token = _query_first_page( + source_container, + query=query, + parameters=[{'name': '@document_metadata_type', 'value': 'document_metadata'}], + continuation_token=continuation_token, + max_item_count=max_item_count, + ) + return { + 'source_scope': source_scope, + 'source_container': source_container_name, + 'documents': documents, + 'continuation_token': next_token, + } + + +def _query_repair_documents(max_item_count=DOCUMENT_ACCESS_DEFAULT_REPAIR_BATCH_SIZE): + query = ( + 'SELECT * FROM c ' + 'WHERE c.type = @type AND c.status = @status' + ) + repair_docs, _next_token = _query_first_page( + cosmos_settings_container, + query=query, + parameters=[ + {'name': '@type', 'value': DOCUMENT_ACCESS_REPAIR_TYPE}, + {'name': '@status', 'value': 'repair_required'}, + ], + max_item_count=max_item_count, + ) + return repair_docs + + +def _read_backfill_state(use_cache=True): + if use_cache: + cache_hit, cached_state = _get_cached_document_access_state(DOCUMENT_ACCESS_BACKFILL_STATE_DOC_ID) + if cache_hit: + return cached_state + try: + state = cosmos_settings_container.read_item( + item=DOCUMENT_ACCESS_BACKFILL_STATE_DOC_ID, + partition_key=DOCUMENT_ACCESS_BACKFILL_STATE_DOC_ID, + ) + except Exception as exc: + if _is_not_found_error(exc): + _set_cached_document_access_state(DOCUMENT_ACCESS_BACKFILL_STATE_DOC_ID, None) + return None + log_event( + '[DocumentAccessIndex] Failed to read document access index backfill state.', + extra={'error': str(exc)}, + level=logging.ERROR, + exceptionTraceback=True, + ) + raise + _set_cached_document_access_state(DOCUMENT_ACCESS_BACKFILL_STATE_DOC_ID, state) + return state + + +def _write_backfill_state(state): + state_body = copy.deepcopy(state or {}) + state_body.update({ + 'id': DOCUMENT_ACCESS_BACKFILL_STATE_DOC_ID, + 'type': DOCUMENT_ACCESS_BACKFILL_STATE_TYPE, + 'updated_at': _utc_now_iso(), + 'schema_version': DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION, + }) + cosmos_settings_container.upsert_item(state_body) + _set_cached_document_access_state(DOCUMENT_ACCESS_BACKFILL_STATE_DOC_ID, state_body) + return state_body + + +def _build_initial_backfill_state(source_scopes): + return { + 'id': DOCUMENT_ACCESS_BACKFILL_STATE_DOC_ID, + 'type': DOCUMENT_ACCESS_BACKFILL_STATE_TYPE, + 'status': 'not_started', + 'source_scopes': list(source_scopes), + 'completed_source_scopes': [], + 'continuation_tokens': {}, + 'current_source_scope': None, + 'total_documents_processed': 0, + 'total_documents_failed': 0, + 'total_rows_upserted': 0, + 'total_rows_deleted': 0, + 'last_error': None, + 'schema_version': DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION, + } + + +def _get_next_backfill_scope(source_scopes, completed_source_scopes): + completed = set(completed_source_scopes or []) + for source_scope in source_scopes: + if source_scope not in completed: + return source_scope + return None + + +def _repair_document_stub(source_scope, document_id): + document_stub = {'id': document_id} + if source_scope == DOCUMENT_ACCESS_SCOPE_GROUP: + document_stub['group_id'] = 'repair' + elif source_scope == DOCUMENT_ACCESS_SCOPE_PUBLIC: + document_stub['public_workspace_id'] = 'repair' + return document_stub + + +def _read_source_document_for_repair(source_scope, document_id): + source_container, _source_container_name = _get_source_scope_container(source_scope) + return source_container.read_item(item=document_id, partition_key=document_id) + + +def _repair_operation_represents_delete(operation): + normalized_operation = str(operation or '').strip().lower() + return 'delete' in normalized_operation or 'cleanup' in normalized_operation + + +def _mark_repair_document_failed(repair_doc, error): + updated_repair_doc = copy.deepcopy(repair_doc or {}) + updated_repair_doc.update({ + 'status': 'repair_required', + 'last_repair_error': str(error), + 'last_repair_attempted_at': _utc_now_iso(), + 'repair_attempt_count': int(updated_repair_doc.get('repair_attempt_count') or 0) + 1, + }) + cosmos_settings_container.upsert_item(updated_repair_doc) + return updated_repair_doc + + +def count_document_access_index_repair_documents(): + """Return the current number of projection repair documents.""" + query = ( + 'SELECT VALUE COUNT(1) FROM c ' + 'WHERE c.type = @type AND c.status = @status' + ) + try: + count_items = list(cosmos_settings_container.query_items( + query=query, + parameters=[ + {'name': '@type', 'value': DOCUMENT_ACCESS_REPAIR_TYPE}, + {'name': '@status', 'value': 'repair_required'}, + ], + enable_cross_partition_query=True, + max_item_count=1, + )) + repair_count = _safe_int(count_items[0]) if count_items else 0 + if repair_count <= 0: + state = _read_repair_backlog_state() + if _repair_backlog_state_has_untracked_failure(state): + return None + _try_write_repair_backlog_state(False, repair_required_count=0, reason='repair_count') + return 0 + _try_write_repair_backlog_state( + repair_count > 0, + repair_required_count=repair_count, + reason='repair_count', + ) + return repair_count + except Exception as exc: + log_event( + '[DocumentAccessIndex] Failed to count projection repair documents.', + extra={'error': str(exc)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None + + +def has_document_access_index_repair_backlog(): + """Return whether any projection repair record is currently pending.""" + try: + state = _read_repair_backlog_state() + if state is not None: + if state.get('has_repair_backlog'): + return True + has_backlog = _query_repair_backlog_exists() + if has_backlog: + _try_write_repair_backlog_state(True, reason='repair_backlog_false_state_corrected') + return has_backlog + return _refresh_repair_backlog_state_from_query(reason='repair_backlog_state_initialize') + except Exception as exc: + log_event( + '[DocumentAccessIndex] Failed to check projection repair backlog.', + extra={'error': str(exc)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None + + +def _build_document_access_index_maintenance_summary(state, repair_required_count, normalized_settings): + state = state if isinstance(state, dict) else {} + normalized_status = str(state.get('status') or 'not_started').strip().lower() or 'not_started' + repair_count_known = repair_required_count is not None + has_repair_work = repair_count_known and int(repair_required_count or 0) > 0 + has_unknown_repair_work = not repair_count_known + has_backfill_work = normalized_status not in DOCUMENT_ACCESS_BACKFILL_COMPLETE_STATUSES + + if not normalized_settings.get('container_enabled'): + next_action = 'disabled' + elif has_unknown_repair_work: + next_action = 'check_repair_backlog' + elif has_repair_work: + next_action = 'repair' + elif has_backfill_work: + next_action = 'backfill' + else: + next_action = 'monitor' + + return { + 'mode': 'automatic', + 'auto_maintenance_enabled': bool(normalized_settings.get('container_enabled')), + 'has_more_work': bool(normalized_settings.get('container_enabled') and ( + has_unknown_repair_work or has_repair_work or has_backfill_work + )), + 'next_action': next_action, + 'backfill_status': normalized_status, + 'repair_required_count': repair_required_count, + 'last_batch_processed_count': _safe_int(state.get('last_processed_count')), + 'last_batch_failed_count': _safe_int(state.get('last_failed_count')), + 'last_batch_rows_upserted': _safe_int(state.get('last_rows_upserted')), + 'last_batch_rows_deleted': _safe_int(state.get('last_rows_deleted')), + 'last_completed_at': state.get('last_completed_at'), + 'last_error': state.get('last_error'), + } + + +def is_document_access_index_maintenance_pending(status_payload): + """Return True when DAI repair/backfill work should keep active maintenance polling.""" + if not isinstance(status_payload, dict): + return False + backfill_status = status_payload.get('document_access_index_backfill') + if isinstance(backfill_status, dict): + status_payload = backfill_status + maintenance_summary = status_payload.get('maintenance') + if isinstance(maintenance_summary, dict) and 'has_more_work' in maintenance_summary: + return bool(maintenance_summary.get('has_more_work')) + + state = status_payload.get('state') if isinstance(status_payload.get('state'), dict) else {} + repair_required_count = status_payload.get('repair_required_count') + settings = status_payload.get('settings') if isinstance(status_payload.get('settings'), dict) else {} + return _build_document_access_index_maintenance_summary( + state, + repair_required_count, + { + 'container_enabled': settings.get('container_enabled', True), + }, + ).get('has_more_work') + + +def get_document_access_index_backfill_status(settings=None): + """Return persisted backfill state and repair backlog diagnostics.""" + normalized_settings = get_document_access_index_settings(settings) + state = _read_backfill_state() or _build_initial_backfill_state(DOCUMENT_ACCESS_SOURCE_SCOPES) + repair_required_count = count_document_access_index_repair_documents() + shadow_validation = { + 'status': 'not_run', + 'missing_count': 0, + 'extra_count': 0, + 'source_query_ru': None, + 'projection_query_ru': None, + 'validation_index_ru': None, + 'candidate_read_ru': None, + 'estimated_ru_savings': None, + 'estimated_wave5_ru_savings': None, + 'shadow_overhead_ru': None, + 'source_query_ms': None, + 'projection_query_ms': None, + 'validation_index_ms': None, + 'candidate_read_ms': None, + 'estimated_ms_savings': None, + 'estimated_wave5_ms_savings': None, + 'shadow_overhead_ms': None, + } + if normalized_settings.get('shadow_validation_enabled'): + shadow_validation = _read_shadow_validation_state() or shadow_validation + if isinstance(shadow_validation, dict) and not isinstance(shadow_validation.get('rolling_metrics'), dict): + shadow_validation['rolling_metrics'] = _empty_shadow_rolling_metrics() + return { + 'success': True, + 'state': state, + 'repair_required_count': repair_required_count, + 'maintenance': _build_document_access_index_maintenance_summary( + state, + repair_required_count, + normalized_settings, + ), + 'read_metrics': get_document_access_index_read_metrics(), + 'cache_metrics': get_document_access_index_cache_metrics(), + 'shadow_validation': shadow_validation, + 'settings': { + 'container_enabled': normalized_settings.get('container_enabled'), + 'write_through_enabled': normalized_settings.get('write_through_enabled'), + 'reads_enabled': normalized_settings.get('reads_enabled'), + 'shadow_validation_enabled': normalized_settings.get('shadow_validation_enabled'), + 'startup_backfill_enabled': normalized_settings.get('startup_backfill_enabled'), + 'backfill_batch_size': normalized_settings.get('backfill_batch_size'), + 'repair_batch_size': normalized_settings.get('repair_batch_size'), + 'cache_enabled': normalized_settings.get('cache_enabled'), + 'cache_ttl_seconds': normalized_settings.get('cache_ttl_seconds'), + }, + } + + +def reconcile_document_access_index_repair_documents(settings=None, max_repairs=None, force=False): + """Repair projection rows recorded by fail-open write-through operations.""" + if not _is_container_enabled(settings) and not force: + return { + 'success': True, + 'status': 'skipped_disabled', + 'repairs_processed': 0, + 'repairs_succeeded': 0, + 'repairs_failed': 0, + } + + normalized_settings = get_document_access_index_settings(settings) + max_repairs = _normalize_positive_int( + max_repairs, + normalized_settings.get('repair_batch_size'), + max_value=DOCUMENT_ACCESS_MAX_REPAIR_BATCH_SIZE, + ) + repair_docs = _query_repair_documents(max_item_count=max_repairs) + repairs_succeeded = 0 + repairs_failed = 0 + + for repair_doc in repair_docs: + document_id = str(repair_doc.get('source_document_id') or '').strip() + source_scope = str(repair_doc.get('source_scope') or '').strip().lower() + operation = repair_doc.get('operation') + if not document_id or source_scope not in DOCUMENT_ACCESS_SOURCE_SCOPES: + repairs_failed += 1 + _mark_repair_document_failed(repair_doc, 'Repair document is missing source scope or document id.') + continue + + try: + repair_cache_scope_keys = _normalize_document_access_cache_scope_keys( + repair_doc.get('cache_scope_keys') + ) + if repair_cache_scope_keys: + invalidation_result = invalidate_document_access_index_cache_scope_keys( + repair_cache_scope_keys, + reason='document_access_repair_historical_scope', + settings=settings, + ) + _raise_for_failed_cache_invalidation( + invalidation_result, + 'document_access_repair_historical_scope', + repair_cache_scope_keys, + ) + + if _repair_operation_represents_delete(operation): + delete_document_access_index_for_document( + _repair_document_stub(source_scope, document_id), + operation='document_access_repair_delete', + settings=settings, + force=True, + ) + else: + try: + source_document = _read_source_document_for_repair(source_scope, document_id) + except Exception as source_error: + if not _is_not_found_error(source_error): + raise + delete_document_access_index_for_document( + _repair_document_stub(source_scope, document_id), + operation='document_access_repair_source_missing', + settings=settings, + force=True, + ) + else: + sync_document_access_index_for_document( + source_document, + operation='document_access_repair_sync', + settings=settings, + force=True, + ) + repairs_succeeded += 1 + except Exception as exc: + repairs_failed += 1 + _mark_repair_document_failed(repair_doc, exc) + log_event( + '[DocumentAccessIndex] Projection repair reconciliation failed.', + extra={ + 'document_id': document_id, + 'source_scope': source_scope, + 'operation': operation, + 'error': str(exc), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + + try: + _refresh_repair_backlog_state_from_query(reason='repair_reconciliation') + except Exception as exc: + log_event( + '[DocumentAccessIndex] Projection repair backlog state refresh failed after reconciliation.', + extra={'error': str(exc)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + return { + 'success': repairs_failed == 0, + 'status': 'reconciled' if repairs_failed == 0 else 'reconciled_with_errors', + 'repairs_processed': len(repair_docs), + 'repairs_succeeded': repairs_succeeded, + 'repairs_failed': repairs_failed, + } + + +def run_document_access_index_backfill_once( + settings=None, + batch_size=None, + source_scopes=None, + reset=False, + force=False, +): + """Run one resumable batch of document access index backfill work.""" + normalized_settings = get_document_access_index_settings(settings) + if not normalized_settings.get('container_enabled'): + return { + 'success': True, + 'status': 'skipped_disabled', + 'processed_count': 0, + 'failed_count': 0, + } + if not force and not normalized_settings.get('startup_backfill_enabled'): + return { + 'success': True, + 'status': 'skipped_disabled', + 'processed_count': 0, + 'failed_count': 0, + } + + source_scopes = _normalize_source_scopes(source_scopes) + batch_size = _normalize_positive_int( + batch_size, + normalized_settings.get('backfill_batch_size'), + max_value=DOCUMENT_ACCESS_MAX_BACKFILL_BATCH_SIZE, + ) + state = None if reset else _read_backfill_state() + if ( + not state + or state.get('type') != DOCUMENT_ACCESS_BACKFILL_STATE_TYPE + or _safe_int(state.get('schema_version')) != DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION + ): + state = _build_initial_backfill_state(source_scopes) + else: + state = copy.deepcopy(state) + state['source_scopes'] = source_scopes + state.setdefault('completed_source_scopes', []) + state.setdefault('continuation_tokens', {}) + state.setdefault('total_documents_processed', 0) + state.setdefault('total_documents_failed', 0) + state.setdefault('total_rows_upserted', 0) + state.setdefault('total_rows_deleted', 0) + + if state.get('status') == 'succeeded' and not reset: + return { + 'success': True, + 'status': 'skipped_completed', + 'processed_count': 0, + 'failed_count': 0, + 'state': state, + } + + run_processed = 0 + run_failed = 0 + run_rows_upserted = 0 + run_rows_deleted = 0 + state['status'] = 'running' + state['last_started_at'] = _utc_now_iso() + state['last_error'] = None + + try: + while run_processed < batch_size: + source_scope = _get_next_backfill_scope( + source_scopes, + state.get('completed_source_scopes', []), + ) + if not source_scope: + break + + continuation_token = state.get('continuation_tokens', {}).get(source_scope) + remaining_count = batch_size - run_processed + page_result = _query_source_document_page( + source_scope, + continuation_token=continuation_token, + max_item_count=remaining_count, + ) + documents = page_result.get('documents', []) + next_token = page_result.get('continuation_token') + + if not documents: + completed_scopes = state.setdefault('completed_source_scopes', []) + if source_scope not in completed_scopes: + completed_scopes.append(source_scope) + state.setdefault('continuation_tokens', {}).pop(source_scope, None) + state['current_source_scope'] = None + continue + + state['current_source_scope'] = source_scope + for document_item in documents: + run_processed += 1 + state['total_documents_processed'] = int(state.get('total_documents_processed') or 0) + 1 + try: + result = sync_document_access_index_for_document( + document_item, + operation='document_access_backfill', + settings=settings, + force=True, + ) + except Exception as exc: + run_failed += 1 + state['total_documents_failed'] = int(state.get('total_documents_failed') or 0) + 1 + try: + _record_projection_repair_required(document_item, 'document_access_backfill', exc) + except Exception as repair_error: + log_event( + '[DocumentAccessIndex] Failed to record backfill repair state.', + extra={ + 'document_id': (document_item or {}).get('id'), + 'source_scope': source_scope, + 'error': str(repair_error), + }, + level=logging.ERROR, + exceptionTraceback=True, + ) + log_event( + '[DocumentAccessIndex] Backfill projection failed for one source document.', + extra={ + 'document_id': (document_item or {}).get('id'), + 'source_scope': source_scope, + 'error': str(exc), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + continue + if result.get('success') is False: + run_failed += 1 + state['total_documents_failed'] = int(state.get('total_documents_failed') or 0) + 1 + continue + run_rows_upserted += int(result.get('upserted_count') or 0) + run_rows_deleted += int(result.get('deleted_count') or 0) + state['total_rows_upserted'] = ( + int(state.get('total_rows_upserted') or 0) + + int(result.get('upserted_count') or 0) + ) + state['total_rows_deleted'] = ( + int(state.get('total_rows_deleted') or 0) + + int(result.get('deleted_count') or 0) + ) + + if next_token: + state.setdefault('continuation_tokens', {})[source_scope] = next_token + break + + state.setdefault('continuation_tokens', {}).pop(source_scope, None) + completed_scopes = state.setdefault('completed_source_scopes', []) + if source_scope not in completed_scopes: + completed_scopes.append(source_scope) + state['current_source_scope'] = None + + if _get_next_backfill_scope(source_scopes, state.get('completed_source_scopes', [])): + state['status'] = 'in_progress' + elif int(state.get('total_documents_failed') or 0) > 0: + state['status'] = 'succeeded_with_errors' + else: + state['status'] = 'succeeded' + + state.update({ + 'last_completed_at': _utc_now_iso(), + 'last_processed_count': run_processed, + 'last_failed_count': run_failed, + 'last_rows_upserted': run_rows_upserted, + 'last_rows_deleted': run_rows_deleted, + }) + state = _write_backfill_state(state) + return { + 'success': run_failed == 0, + 'status': state.get('status'), + 'processed_count': run_processed, + 'failed_count': run_failed, + 'rows_upserted': run_rows_upserted, + 'rows_deleted': run_rows_deleted, + 'state': state, + } + except Exception as exc: + state['status'] = 'failed' + state['last_error'] = str(exc) + state['last_completed_at'] = _utc_now_iso() + state = _write_backfill_state(state) + log_event( + '[DocumentAccessIndex] Document access index backfill batch failed.', + extra={'error': str(exc), 'current_source_scope': state.get('current_source_scope')}, + level=logging.ERROR, + exceptionTraceback=True, + ) + return { + 'success': False, + 'status': 'failed', + 'processed_count': run_processed, + 'failed_count': run_failed, + 'error': str(exc), + 'state': state, + } + + +def run_document_access_index_backfill_maintenance( + settings=None, + run_backfill=None, + reset=False, + batch_size=None, + repair_batch_size=None, +): + """Run repair reconciliation and one optional backfill batch for app maintenance.""" + normalized_settings = get_document_access_index_settings(settings) + if run_backfill is None: + run_backfill = normalized_settings.get('startup_backfill_enabled', False) + if not normalized_settings.get('container_enabled'): + return { + 'success': True, + 'status': 'skipped_disabled', + 'repair_reconciliation': { + 'success': True, + 'status': 'skipped_disabled', + }, + 'backfill': { + 'success': True, + 'status': 'skipped_disabled', + }, + 'current_status': get_document_access_index_backfill_status(settings=settings), + } + + repair_result = reconcile_document_access_index_repair_documents( + settings=settings, + max_repairs=repair_batch_size, + force=True, + ) + if run_backfill: + backfill_result = run_document_access_index_backfill_once( + settings=settings, + batch_size=batch_size, + reset=reset, + force=True, + ) + else: + backfill_result = { + 'success': True, + 'status': 'skipped_disabled', + 'processed_count': 0, + 'failed_count': 0, + } + success = repair_result.get('success') and backfill_result.get('success') + current_status = get_document_access_index_backfill_status(settings=settings) + maintenance_pending = is_document_access_index_maintenance_pending(current_status) + status = 'completed' if success else 'completed_with_errors' + if success and maintenance_pending: + status = 'completed_more_work_pending' + return { + 'success': bool(success), + 'status': status, + 'maintenance_pending': maintenance_pending, + 'next_action': (current_status.get('maintenance') or {}).get('next_action'), + 'repair_reconciliation': repair_result, + 'backfill': backfill_result, + 'current_status': current_status, + } diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index 3cb0ce3e..df5d1da8 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -9,6 +9,16 @@ from flask import make_response from config import * from functions_appinsights import log_event +from functions_document_access_index import ( + DOCUMENT_ACCESS_SCOPE_GROUP, + DOCUMENT_ACCESS_SCOPE_PERSONAL, + DOCUMENT_ACCESS_SCOPE_PUBLIC, + delete_document_access_index_for_document_fail_open, + is_document_access_shadow_validation_enabled, + query_items_with_cosmos_diagnostics, + sync_document_access_index_for_document_fail_open, + validate_document_access_index_shadow, +) from functions_visio import build_visio_page_markdown, parse_vsdx_pages from functions_content import * from functions_settings import * @@ -678,7 +688,7 @@ def sort_key(document_item): return sorted(documents or [], key=sort_key, reverse=reverse) -def _query_accessible_documents(user_id, group_id=None, public_workspace_id=None): +def _query_accessible_documents(user_id, group_id=None, public_workspace_id=None, collect_diagnostics=False): cosmos_container = _get_documents_container(group_id=group_id, public_workspace_id=public_workspace_id) if public_workspace_id is not None: @@ -715,13 +725,26 @@ def _query_accessible_documents(user_id, group_id=None, public_workspace_id=None {"name": "@user_id_prefix", "value": f"{user_id},"} ] - return list( - cosmos_container.query_items( - query=query, - parameters=parameters, - enable_cross_partition_query=True, - ) + documents, diagnostics = query_items_with_cosmos_diagnostics( + cosmos_container, + diagnostics_label='source_documents', + collect_diagnostics=collect_diagnostics, + query=query, + parameters=parameters, + enable_cross_partition_query=True, + ) + if collect_diagnostics: + return documents, diagnostics + return documents + + +def _upsert_document_and_sync_access_index(cosmos_container, document_item, operation): + persisted_document = cosmos_container.upsert_item(document_item) + sync_document_access_index_for_document_fail_open( + persisted_document if isinstance(persisted_document, dict) else document_item, + operation=operation, ) + return persisted_document if isinstance(persisted_document, dict) else document_item def _build_archived_scope_value(scope_value): @@ -815,7 +838,11 @@ def normalize_document_revision_families(user_id, group_id=None, public_workspac update_occurred = True if update_occurred: - cosmos_container.upsert_item(document_item) + _upsert_document_and_sync_access_index( + cosmos_container, + document_item, + operation='document_revision_normalized', + ) changes_made = True return changes_made @@ -985,7 +1012,11 @@ def create_document(file_name, user_id, document_id, num_file_chunks, status, gr update_existing_document = True if update_existing_document: - cosmos_container.upsert_item(existing_document) + _upsert_document_and_sync_access_index( + cosmos_container, + existing_document, + operation='document_revision_archived', + ) if is_public_workspace: document_metadata = { @@ -1086,7 +1117,11 @@ def create_document(file_name, user_id, document_id, num_file_chunks, status, gr "tags": carried_forward.get("tags", []) } - cosmos_container.upsert_item(document_metadata) + _upsert_document_and_sync_access_index( + cosmos_container, + document_metadata, + operation='document_created', + ) add_file_task_to_file_processing_log( document_id, @@ -2414,7 +2449,11 @@ def update_document(**kwargs): # 5. Upsert the document if changes were made if update_occurred: - cosmos_container.upsert_item(existing_document) + _upsert_document_and_sync_access_index( + cosmos_container, + existing_document, + operation='document_updated', + ) except CosmosResourceNotFoundError as e: # Error already logged where it was first detected @@ -3176,12 +3215,38 @@ def get_ordered_document_chunks(document_id, user_id, group_id=None, public_work def get_documents(user_id, group_id=None, public_workspace_id=None): try: - documents = _query_accessible_documents( + collect_shadow_metrics = is_document_access_shadow_validation_enabled() + if collect_shadow_metrics: + documents, source_query_metrics = _query_accessible_documents( + user_id=user_id, + group_id=group_id, + public_workspace_id=public_workspace_id, + collect_diagnostics=True, + ) + else: + documents = _query_accessible_documents( + user_id=user_id, + group_id=group_id, + public_workspace_id=public_workspace_id, + ) + source_query_metrics = None + current_documents = sort_documents(select_current_documents(documents)) + source_scope = DOCUMENT_ACCESS_SCOPE_PERSONAL + shadow_group_ids = None + if public_workspace_id is not None: + source_scope = DOCUMENT_ACCESS_SCOPE_PUBLIC + elif group_id is not None: + source_scope = DOCUMENT_ACCESS_SCOPE_GROUP + shadow_group_ids = [group_id] + validate_document_access_index_shadow( + current_documents, + source_scope=source_scope, user_id=user_id, - group_id=group_id, + group_ids=shadow_group_ids, public_workspace_id=public_workspace_id, + source_query_metrics=source_query_metrics, + context='functions_documents.get_documents', ) - current_documents = sort_documents(select_current_documents(documents)) return jsonify({"documents": current_documents}), 200 except Exception as e: return jsonify({'error': f'Error retrieving documents: {str(e)}'}), 500 @@ -3411,6 +3476,10 @@ def delete_document(user_id, document_id, group_id=None, public_workspace_id=Non item=document_id, partition_key=document_id ) + delete_document_access_index_for_document_fail_open( + document_item, + operation='document_deleted', + ) except CosmosResourceNotFoundError: raise Exception("Document not found") @@ -3483,7 +3552,11 @@ def delete_document_revision(user_id, document_id, delete_mode="all_versions", g public_workspace_id=public_workspace_id, ) set_document_chunk_visibility(promoted_document, active=True) - cosmos_container.upsert_item(promoted_document) + _upsert_document_and_sync_access_index( + cosmos_container, + promoted_document, + operation='document_revision_promoted', + ) promoted_document_id = promoted_document.get('id') return { @@ -3722,11 +3795,15 @@ def sync_chat_upload_workspace_document_sharing_for_collaboration(conversation_d continue document_item['last_updated'] = current_time - cosmos_user_documents_container.upsert_item(document_item) + persisted_document = _upsert_document_and_sync_access_index( + cosmos_user_documents_container, + document_item, + operation='chat_upload_collaboration_share_synced', + ) try: set_document_chunk_visibility( - document_item, - active=str(document_item.get('search_visibility_state') or 'active').strip().lower() != 'archived', + persisted_document, + active=str(persisted_document.get('search_visibility_state') or 'active').strip().lower() != 'archived', ) except Exception as chunk_sync_error: log_event( @@ -4898,7 +4975,11 @@ def upload_to_blob(temp_file_path, user_id, document_id, blob_filename, update_c public_workspace_id=public_workspace_id, ) if archived_blob_path: - cosmos_container.upsert_item(previous_document) + _upsert_document_and_sync_access_index( + cosmos_container, + previous_document, + operation='document_blob_archived', + ) blob_service_client = _get_blob_service_client() @@ -4928,7 +5009,11 @@ def upload_to_blob(temp_file_path, user_id, document_id, blob_filename, update_c current_document["enhanced_citations"] = bool(mark_enhanced_citations) if current_document.get("archived_blob_path") is None: current_document["archived_blob_path"] = None - cosmos_container.upsert_item(current_document) + _upsert_document_and_sync_access_index( + cosmos_container, + current_document, + operation='document_blob_uploaded', + ) print(f"Successfully uploaded {blob_filename} to blob storage at {blob_path}") return blob_path @@ -8139,6 +8224,13 @@ def queue_personal_workspace_upload_from_temp_file( item=workspace_document_id, partition_key=workspace_document_id, ) + delete_document_access_index_for_document_fail_open( + { + 'id': workspace_document_id, + 'user_id': user_id, + }, + operation='queued_personal_document_cleanup', + ) except Exception as cleanup_error: debug_print(f"Failed to clean up queued workspace document metadata: {cleanup_error}") if workspace_temp_file_path and os.path.exists(workspace_temp_file_path) and not temp_file_queued: @@ -8289,6 +8381,13 @@ def queue_group_workspace_upload_from_temp_file( item=workspace_document_id, partition_key=workspace_document_id, ) + delete_document_access_index_for_document_fail_open( + { + 'id': workspace_document_id, + 'group_id': group_id, + }, + operation='queued_group_document_cleanup', + ) except Exception as cleanup_error: debug_print(f"Failed to clean up queued group workspace document metadata: {cleanup_error}") if workspace_temp_file_path and os.path.exists(workspace_temp_file_path) and not temp_file_queued: @@ -8946,7 +9045,11 @@ def share_document_with_user(document_id, owner_user_id, target_user_id): document_item['last_updated'] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') # Update the document - cosmos_user_documents_container.upsert_item(document_item) + _upsert_document_and_sync_access_index( + cosmos_user_documents_container, + document_item, + operation='document_shared_with_user', + ) # Update all chunks with the new shared_user_ids try: @@ -9010,7 +9113,11 @@ def unshare_document_from_user(document_id, owner_user_id, target_user_id): document_item['shared_user_ids'] = new_shared_user_ids document_item['last_updated'] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') # Update the document - cosmos_user_documents_container.upsert_item(document_item) + _upsert_document_and_sync_access_index( + cosmos_user_documents_container, + document_item, + operation='document_unshared_from_user', + ) # Update all chunks with the new shared_user_ids try: @@ -9173,7 +9280,11 @@ def share_document_with_group(document_id, owner_group_id, target_group_id): document_item['last_updated'] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') # Update the document - cosmos_group_documents_container.upsert_item(document_item) + _upsert_document_and_sync_access_index( + cosmos_group_documents_container, + document_item, + operation='document_shared_with_group', + ) return True return True # Already shared @@ -9212,7 +9323,11 @@ def unshare_document_from_group(document_id, owner_group_id, target_group_id): document_item['last_updated'] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') # Update the document - cosmos_group_documents_container.upsert_item(document_item) + _upsert_document_and_sync_access_index( + cosmos_group_documents_container, + document_item, + operation='document_unshared_from_group', + ) return True @@ -9463,13 +9578,81 @@ def validate_tag_color(color, tag_name): return True, None, normalized_color +def get_workspace_tag_definitions(user_id, group_id=None, public_workspace_id=None): + """Return tag definition metadata for a personal, group, or public workspace.""" + # Local imports avoid circular initialization with workspace/settings helpers. + if public_workspace_id is not None: + from functions_public_workspaces import find_public_workspace_by_id + ws_doc = find_public_workspace_by_id(public_workspace_id) + workspace_tag_defs = (ws_doc or {}).get('tag_definitions', {}) + elif group_id is not None: + from functions_group import find_group_by_id + group_doc = find_group_by_id(group_id) + workspace_tag_defs = (group_doc or {}).get('tag_definitions', {}) + else: + from functions_settings import get_user_settings + user_settings = get_user_settings(user_id) + settings_dict = user_settings.get('settings', {}) + tag_definitions = settings_dict.get('tag_definitions', {}) + workspace_tag_defs = tag_definitions.get('personal', {}) + + return workspace_tag_defs if isinstance(workspace_tag_defs, dict) else {} + + +def build_workspace_tags_from_counts(tag_counts, user_id, group_id=None, public_workspace_id=None): + """ + Merge normalized tag counts with workspace tag definitions and safe colors. + Returns: [{'name': 'tag1', 'count': 5, 'color': '#3b82f6'}, ...] + """ + normalized_counts = {} + for tag_name, count in (tag_counts or {}).items(): + normalized_tag = normalize_tag(tag_name) + if not normalized_tag: + continue + try: + normalized_count = int(count or 0) + except (TypeError, ValueError): + normalized_count = 0 + normalized_counts[normalized_tag] = normalized_counts.get(normalized_tag, 0) + normalized_count + + workspace_tag_defs = get_workspace_tag_definitions( + user_id, + group_id=group_id, + public_workspace_id=public_workspace_id, + ) + + results = [] + for tag_name, count in normalized_counts.items(): + tag_def = workspace_tag_defs.get(tag_name, {}) + if not isinstance(tag_def, dict): + tag_def = {} + results.append({ + 'name': tag_name, + 'count': count, + 'color': get_safe_tag_color(tag_def.get('color'), tag_name) + }) + + for tag_name, tag_def in workspace_tag_defs.items(): + normalized_tag = normalize_tag(tag_name) + if not normalized_tag or normalized_tag in normalized_counts: + continue + if not isinstance(tag_def, dict): + tag_def = {} + results.append({ + 'name': normalized_tag, + 'count': 0, + 'color': get_safe_tag_color(tag_def.get('color'), tag_name) + }) + + results.sort(key=lambda x: (-x['count'], x['name'])) + return results + + def get_workspace_tags(user_id, group_id=None, public_workspace_id=None): """ Get all unique tags used in a workspace with document counts. Returns: [{'name': 'tag1', 'count': 5, 'color': '#3b82f6'}, ...] """ - from functions_settings import get_user_settings - is_group = group_id is not None is_public_workspace = public_workspace_id is not None @@ -9534,47 +9717,12 @@ def get_workspace_tags(user_id, group_id=None, public_workspace_id=None): if normalized_tag: tag_counts[normalized_tag] = tag_counts.get(normalized_tag, 0) + 1 - # Get tag definitions (colors) from the appropriate source - if is_public_workspace: - # Read from public workspace record (shared across all users) - from functions_public_workspaces import find_public_workspace_by_id - ws_doc = find_public_workspace_by_id(public_workspace_id) - workspace_tag_defs = (ws_doc or {}).get('tag_definitions', {}) - elif is_group: - # Read from group record (shared across all group members) - from functions_group import find_group_by_id - group_doc = find_group_by_id(group_id) - workspace_tag_defs = (group_doc or {}).get('tag_definitions', {}) - else: - # Personal: read from user settings - user_settings = get_user_settings(user_id) - settings_dict = user_settings.get('settings', {}) - tag_definitions = settings_dict.get('tag_definitions', {}) - workspace_tag_defs = tag_definitions.get('personal', {}) - - # Build result with colors from used tags - results = [] - for tag_name, count in tag_counts.items(): - tag_def = workspace_tag_defs.get(tag_name, {}) - results.append({ - 'name': tag_name, - 'count': count, - 'color': get_safe_tag_color(tag_def.get('color'), tag_name) - }) - - # Add defined tags that haven't been used yet (count = 0) - for tag_name, tag_def in workspace_tag_defs.items(): - if tag_name not in tag_counts: - results.append({ - 'name': tag_name, - 'count': 0, - 'color': get_safe_tag_color(tag_def.get('color'), tag_name) - }) - - # Sort by count descending, then name ascending - results.sort(key=lambda x: (-x['count'], x['name'])) - - return results + return build_workspace_tags_from_counts( + tag_counts, + user_id, + group_id=group_id, + public_workspace_id=public_workspace_id, + ) except Exception as e: print(f"Error getting workspace tags: {e}") diff --git a/application/single_app/functions_global_actions.py b/application/single_app/functions_global_actions.py index 19f0eb60..5203927d 100644 --- a/application/single_app/functions_global_actions.py +++ b/application/single_app/functions_global_actions.py @@ -18,6 +18,7 @@ hydrate_action_identity_reference, validate_action_identity_reference, ) +from functions_chat_bootstrap_cache import bump_chat_bootstrap_global_cache_version def get_global_actions(return_type=SecretReturnType.TRIGGER, include_disabled=False): """ @@ -154,6 +155,7 @@ def save_global_action(action_data, user_id=None): existing_plugin=existing_action, ) result = cosmos_global_actions_container.upsert_item(body=action_data) + bump_chat_bootstrap_global_cache_version(reason="global_action_saved") print(f"✅ Global action saved successfully: {result['id']}") return result @@ -183,6 +185,7 @@ def delete_global_action(action_id): item=action_id, partition_key=action_id ) + bump_chat_bootstrap_global_cache_version(reason="global_action_deleted") print(f"✅ Global action deleted successfully: {action_id}") return True @@ -220,10 +223,10 @@ def update_global_action_enabled(action_id, is_enabled, user_id=None): action['modified_at'] = now action['updated_at'] = now result = cosmos_global_actions_container.upsert_item(body=action) + bump_chat_bootstrap_global_cache_version(reason="global_action_enabled_updated") return result except Exception as e: print(f"❌ Error updating enabled state for global action {action_id}: {str(e)}") traceback.print_exc() return None - diff --git a/application/single_app/functions_global_agents.py b/application/single_app/functions_global_agents.py index 6bfe6f10..4b23e205 100644 --- a/application/single_app/functions_global_agents.py +++ b/application/single_app/functions_global_agents.py @@ -17,6 +17,7 @@ from functions_keyvault import keyvault_agent_save_helper, keyvault_agent_get_helper, keyvault_agent_delete_helper from functions_settings import * from functions_agent_payload import sanitize_agent_payload, AgentPayloadError +from functions_chat_bootstrap_cache import bump_chat_bootstrap_global_cache_version def ensure_default_global_agent_exists(): @@ -256,6 +257,7 @@ def save_global_agent(agent_data, user_id=None): "Global agent saved successfully.", extra={"agent_id": result['id'], "user_id": user_id}, ) + bump_chat_bootstrap_global_cache_version(reason="global_agent_saved") print(f"Global agent saved successfully: {result['id']}") return result except Exception as e: @@ -293,6 +295,7 @@ def delete_global_agent(agent_id): "Global agent deleted successfully.", extra={"agent_id": agent_id, "user_id": user_id}, ) + bump_chat_bootstrap_global_cache_version(reason="global_agent_deleted") print(f"Global agent deleted successfully: {agent_id}") return True except Exception as e: @@ -335,6 +338,7 @@ def update_global_agent_enabled(agent_id, is_enabled, user_id=None): agent['modified_at'] = now agent['updated_at'] = now result = cosmos_global_agents_container.upsert_item(body=agent) + bump_chat_bootstrap_global_cache_version(reason="global_agent_enabled_updated") return result except Exception as e: log_event( diff --git a/application/single_app/functions_group.py b/application/single_app/functions_group.py index 55e8e85b..94ce8a07 100644 --- a/application/single_app/functions_group.py +++ b/application/single_app/functions_group.py @@ -5,6 +5,7 @@ import functions_settings from typing import Iterable +from functions_chat_bootstrap_cache import bump_chat_bootstrap_global_cache_version from functions_workspace_branding import DEFAULT_WORKSPACE_HERO_COLOR @@ -45,6 +46,7 @@ def create_group(name, description): "modifiedDate": now_str } cosmos_groups_container.create_item(group_doc) + bump_chat_bootstrap_global_cache_version(reason="group_created") return group_doc def search_groups(search_query, user_id): @@ -222,6 +224,7 @@ def delete_group(group_id): Deletes a group from Cosmos DB. Typically only owner can do this. """ cosmos_groups_container.delete_item(item=group_id, partition_key=group_id) + bump_chat_bootstrap_global_cache_version(reason="group_deleted") def is_user_in_group(group_doc, user_id): """ @@ -338,4 +341,5 @@ def update_group_model_endpoints(group_id: str, endpoints): group_doc["model_endpoints"] = endpoints group_doc["modifiedDate"] = datetime.utcnow().isoformat() cosmos_groups_container.upsert_item(group_doc) + bump_chat_bootstrap_global_cache_version(reason="group_model_endpoints_updated") return group_doc \ No newline at end of file diff --git a/application/single_app/functions_group_actions.py b/application/single_app/functions_group_actions.py index 3455486f..19d58d7b 100644 --- a/application/single_app/functions_group_actions.py +++ b/application/single_app/functions_group_actions.py @@ -23,6 +23,7 @@ validate_action_identity_reference, ) from functions_governance import ensure_action_type_access, filter_actions_by_action_type_access +from functions_chat_bootstrap_cache import bump_chat_bootstrap_global_cache_version _NAME_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$") @@ -163,6 +164,7 @@ def save_group_action(group_id: str, action_data: Dict[str, Any], user_id: Optio try: stored = cosmos_group_actions_container.upsert_item(body=payload) + bump_chat_bootstrap_global_cache_version(reason="group_action_saved") return _clean_action(stored, group_id, SecretReturnType.TRIGGER) except Exception as exc: debug_print( @@ -187,6 +189,7 @@ def delete_group_action(group_id: str, action_id: str) -> bool: item=action_id, partition_key=group_id, ) + bump_chat_bootstrap_global_cache_version(reason="group_action_deleted") return True except Exception as exc: debug_print( diff --git a/application/single_app/functions_group_agents.py b/application/single_app/functions_group_agents.py index 179e993b..14321ca8 100644 --- a/application/single_app/functions_group_agents.py +++ b/application/single_app/functions_group_agents.py @@ -18,6 +18,7 @@ ) from functions_agent_payload import sanitize_agent_payload from functions_governance import ensure_governance_access +from functions_chat_bootstrap_cache import bump_chat_bootstrap_global_cache_version _NAME_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$") @@ -141,6 +142,7 @@ def save_group_agent(group_id: str, agent_data: Dict[str, Any], user_id: Optiona try: stored = cosmos_group_agents_container.upsert_item(body=payload) + bump_chat_bootstrap_global_cache_version(reason="group_agent_saved") return _clean_agent(stored) except Exception as exc: debug_print( @@ -165,6 +167,7 @@ def delete_group_agent(group_id: str, agent_id: str) -> bool: item=agent_id, partition_key=group_id, ) + bump_chat_bootstrap_global_cache_version(reason="group_agent_deleted") return True except Exception as exc: debug_print( diff --git a/application/single_app/functions_personal_actions.py b/application/single_app/functions_personal_actions.py index 3bd2d6d1..d51aaf4a 100644 --- a/application/single_app/functions_personal_actions.py +++ b/application/single_app/functions_personal_actions.py @@ -22,6 +22,7 @@ from config import cosmos_personal_actions_container import logging from functions_governance import ensure_action_type_access, filter_actions_by_action_type_access +from functions_chat_bootstrap_cache import bump_chat_bootstrap_user_cache_version def get_governed_personal_actions(user_id, return_type=SecretReturnType.TRIGGER): @@ -216,6 +217,7 @@ def save_personal_action(user_id, action_data, enforce_governance=True): result = cosmos_personal_actions_container.upsert_item(body=action_data) # Remove Cosmos metadata from response cleaned_result = {k: v for k, v in result.items() if not k.startswith('_')} + bump_chat_bootstrap_user_cache_version(user_id, reason="personal_action_saved") return cleaned_result except Exception as e: @@ -247,6 +249,7 @@ def delete_personal_action(user_id, action_id): item=action['id'], partition_key=user_id ) + bump_chat_bootstrap_user_cache_version(user_id, reason="personal_action_deleted") return True except exceptions.CosmosResourceNotFoundError: diff --git a/application/single_app/functions_personal_agents.py b/application/single_app/functions_personal_agents.py index f70e0be5..8b177da0 100644 --- a/application/single_app/functions_personal_agents.py +++ b/application/single_app/functions_personal_agents.py @@ -21,6 +21,7 @@ from functions_agent_payload import sanitize_agent_payload from functions_debug import debug_print from functions_governance import ensure_governance_access +from functions_chat_bootstrap_cache import bump_chat_bootstrap_user_cache_version def get_personal_agents(user_id): """ @@ -216,6 +217,7 @@ def save_personal_agent(user_id, agent_data, actor_user_id=None): cleaned_result.setdefault('agent_type', 'local') cleaned_result.setdefault('tags', []) cleaned_result.setdefault('icon', {}) + bump_chat_bootstrap_user_cache_version(user_id, reason="personal_agent_saved") return cleaned_result except Exception as e: @@ -249,6 +251,7 @@ def delete_personal_agent(user_id, agent_id): item=agent['id'], partition_key=user_id ) + bump_chat_bootstrap_user_cache_version(user_id, reason="personal_agent_deleted") return True except exceptions.CosmosResourceNotFoundError: debug_print(f"Agent {agent_id} not found for user {user_id}") @@ -333,4 +336,3 @@ def migrate_agents_from_user_settings(user_id): except Exception as e: debug_print(f"Error during agent migration for user {user_id}: {e}") return 0 - diff --git a/application/single_app/functions_prompts.py b/application/single_app/functions_prompts.py index e2e6377b..b45f8cd4 100644 --- a/application/single_app/functions_prompts.py +++ b/application/single_app/functions_prompts.py @@ -1,6 +1,17 @@ # functions_prompts.py from config import * +from functions_chat_bootstrap_cache import ( + bump_chat_bootstrap_global_cache_version, + bump_chat_bootstrap_user_cache_version, +) + + +def _invalidate_prompt_chat_bootstrap_cache(user_id, group_id=None, public_workspace_id=None, reason="prompt_changed"): + if group_id is not None or public_workspace_id is not None: + bump_chat_bootstrap_global_cache_version(reason=reason) + else: + bump_chat_bootstrap_user_cache_version(user_id, reason=reason) def get_pagination_params(args): try: @@ -284,6 +295,12 @@ def create_prompt_doc(name, content, prompt_type, user_id, group_id=None, public doc["user_id"] = user_id created = cosmos_container.create_item(body=doc) + _invalidate_prompt_chat_bootstrap_cache( + user_id, + group_id=group_id, + public_workspace_id=public_workspace_id, + reason="prompt_created", + ) return { "id": created["id"], "name": created["name"], @@ -325,6 +342,12 @@ def update_prompt_doc(user_id, prompt_id, prompt_type, updates, group_id=None, p item["updated_at"] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') updated = cosmos_container.replace_item(item=prompt_id, body=item) + _invalidate_prompt_chat_bootstrap_cache( + user_id, + group_id=group_id, + public_workspace_id=public_workspace_id, + reason="prompt_updated", + ) return { "id": updated["id"], @@ -349,4 +372,10 @@ def delete_prompt_doc(user_id, prompt_id, group_id=None, public_workspace_id=Non return False cosmos_container.delete_item(item=prompt_id, partition_key=prompt_id) + _invalidate_prompt_chat_bootstrap_cache( + user_id, + group_id=group_id, + public_workspace_id=public_workspace_id, + reason="prompt_deleted", + ) return True \ No newline at end of file diff --git a/application/single_app/functions_public_workspaces.py b/application/single_app/functions_public_workspaces.py index 7fcaba73..eda51075 100644 --- a/application/single_app/functions_public_workspaces.py +++ b/application/single_app/functions_public_workspaces.py @@ -6,6 +6,7 @@ from functions_group import * from typing import Iterable +from functions_chat_bootstrap_cache import bump_chat_bootstrap_global_cache_version from functions_workspace_branding import ( DEFAULT_WORKSPACE_HERO_COLOR, get_workspace_logo_metadata, @@ -43,6 +44,7 @@ def create_public_workspace(name: str, description: str) -> dict: "modifiedDate": now_iso } cosmos_public_workspaces_container.create_item(ws_doc) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_created") return ws_doc @@ -144,6 +146,7 @@ def delete_public_workspace(ws_id: str) -> None: item=ws_id, partition_key=ws_id ) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_deleted") def get_user_role_in_public_workspace(ws_doc: dict, user_id: str) -> str | None: @@ -246,6 +249,7 @@ def add_document_manager(ws_id: str, user_id: str, email: str, display_name: str }) ws["modifiedDate"] = datetime.utcnow().isoformat() cosmos_public_workspaces_container.upsert_item(ws) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_document_manager_added") def remove_document_manager(ws_id: str, user_id: str) -> None: @@ -262,6 +266,7 @@ def remove_document_manager(ws_id: str, user_id: str) -> None: ] ws["modifiedDate"] = datetime.utcnow().isoformat() cosmos_public_workspaces_container.upsert_item(ws) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_document_manager_removed") def approve_document_manager_request(ws_id: str, request_user_id: str) -> None: @@ -276,13 +281,23 @@ def approve_document_manager_request(ws_id: str, request_user_id: str) -> None: new_pend = [] for p in pend: if p["userId"] == request_user_id: - add_document_manager(ws_id, p["userId"], p["email"], p["displayName"]) + existing_manager_ids = { + dm.get("userId") if isinstance(dm, dict) else dm + for dm in ws.get("documentManagers", []) + } + if p["userId"] not in existing_manager_ids: + ws.setdefault("documentManagers", []).append({ + "userId": p["userId"], + "email": p["email"], + "displayName": p["displayName"] + }) else: new_pend.append(p) ws["pendingDocumentManagers"] = new_pend ws["modifiedDate"] = datetime.utcnow().isoformat() cosmos_public_workspaces_container.upsert_item(ws) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_document_manager_request_approved") def reject_document_manager_request(ws_id: str, request_user_id: str) -> None: @@ -299,6 +314,7 @@ def reject_document_manager_request(ws_id: str, request_user_id: str) -> None: ] ws["modifiedDate"] = datetime.utcnow().isoformat() cosmos_public_workspaces_container.upsert_item(ws) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_document_manager_request_rejected") def count_public_workspace_documents(ws_id: str) -> int: diff --git a/application/single_app/functions_redis_monitoring.py b/application/single_app/functions_redis_monitoring.py new file mode 100644 index 00000000..c691e9ff --- /dev/null +++ b/application/single_app/functions_redis_monitoring.py @@ -0,0 +1,904 @@ +# functions_redis_monitoring.py + +from datetime import datetime, timezone +import json +import re +import time + +import app_settings_cache + + +REDIS_MONITORING_STATUS_DISABLED = "disabled" +REDIS_MONITORING_STATUS_NOT_CONFIGURED = "not_configured" +REDIS_MONITORING_STATUS_UNAVAILABLE = "unavailable" +REDIS_MONITORING_STATUS_HEALTHY = "healthy" +REDIS_MONITORING_STATUS_DEGRADED = "degraded" +REDIS_MONITORING_STATUS_ERROR = "error" +REDIS_EXPLORER_DEFAULT_PAGE_SIZE = 25 +REDIS_EXPLORER_MAX_PAGE_SIZE = 100 +REDIS_EXPLORER_MAX_SCAN_ITERATIONS = 5 +REDIS_EXPLORER_MAX_FILTER_LENGTH = 128 +REDIS_EXPLORER_MAX_KEY_LENGTH = 1024 +REDIS_EXPLORER_VALUE_ITEM_LIMIT = 20 +REDIS_EXPLORER_PREVIEW_MAX_CHARS = 4096 +REDIS_EXPLORER_RESTRICTED_KEY_TOKENS = ( + "authorization", + "cookie", + "credential", + "csrf", + "password", + "secret", + "session", + "token", +) +REDIS_EXPLORER_SENSITIVE_FIELD_TOKENS = REDIS_EXPLORER_RESTRICTED_KEY_TOKENS + ( + "api_key", + "connection", + "key", +) +REDIS_EXPLORER_REDACTED_VALUE = "[redacted]" +REDIS_EXPLORER_RESTRICTED_PREVIEW = ( + "Preview restricted because the Redis key name indicates session, token, cookie, or credential data." +) +REDIS_DAI_CACHE_KEY_PREFIX = "DAI_LIST_CACHE" +REDIS_DAI_CACHE_VERSION_KEY_PREFIX = "DAI_LIST_CACHE_VERSION" +REDIS_DAI_CACHE_PAYLOAD_TTL_DEFAULT_SECONDS = 900 +REDIS_DAI_CACHE_PAYLOAD_TTL_MIN_SECONDS = 60 +REDIS_DAI_CACHE_PAYLOAD_TTL_MAX_SECONDS = 900 +REDIS_DAI_CACHE_VERSION_TTL_MIN_SECONDS = 3600 +REDIS_DAI_CACHE_VERSION_TTL_MAX_SECONDS = 86400 +REDIS_DAI_CACHE_VERSION_TTL_MULTIPLIER = 4 +REDIS_DAI_CACHE_KEYSPACE_SCAN_ITERATIONS = 5 +REDIS_EXPLORER_SENSITIVE_TEXT_PATTERN = re.compile( + r"(?i)(api[_-]?key|authorization|connection[_-]?string|password|secret|token)(\s*[:=]\s*)([^\s,;}\]]+)" +) +REDIS_DAI_VERSION_KEY_PATTERN = re.compile(r"^DAI_LIST_CACHE_VERSION:([a-fA-F0-9]{64})$") +REDIS_DAI_PAYLOAD_KEY_PATTERN = re.compile(r"^DAI_LIST_CACHE:([^:]+):([^:]+):([a-fA-F0-9]{64})$") + + +def _utc_now_iso(): + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _safe_int(value): + if value is None or value == "": + return None + if isinstance(value, bytes): + value = value.decode("utf-8") + try: + return int(value) + except (TypeError, ValueError): + try: + return int(float(value)) + except (TypeError, ValueError): + return None + + +def _safe_str(value): + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if value is None: + return "" + return str(value) + + +def _safe_float(value): + if value is None or value == "": + return None + if isinstance(value, bytes): + value = value.decode("utf-8") + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _safe_percentage(numerator, denominator): + if numerator is None or denominator is None or denominator <= 0: + return None + return round((numerator / denominator) * 100, 2) + + +def _safe_error_summary(context, exc): + return f"{context} ({exc.__class__.__name__})." + + +def _clamp_int(value, default_value, minimum_value, maximum_value): + numeric_value = _safe_int(value) + if numeric_value is None: + return default_value + return min(max(numeric_value, minimum_value), maximum_value) + + +def _calculate_dai_cache_version_ttl_seconds(settings): + payload_ttl_seconds = _clamp_int( + (settings or {}).get("document_access_index_cache_ttl_seconds"), + REDIS_DAI_CACHE_PAYLOAD_TTL_DEFAULT_SECONDS, + REDIS_DAI_CACHE_PAYLOAD_TTL_MIN_SECONDS, + REDIS_DAI_CACHE_PAYLOAD_TTL_MAX_SECONDS, + ) + return min( + max( + payload_ttl_seconds * REDIS_DAI_CACHE_VERSION_TTL_MULTIPLIER, + REDIS_DAI_CACHE_VERSION_TTL_MIN_SECONDS, + ), + REDIS_DAI_CACHE_VERSION_TTL_MAX_SECONDS, + ) + + +def _build_empty_dai_cache_keyspace(settings): + payload_ttl_seconds = _clamp_int( + (settings or {}).get("document_access_index_cache_ttl_seconds"), + REDIS_DAI_CACHE_PAYLOAD_TTL_DEFAULT_SECONDS, + REDIS_DAI_CACHE_PAYLOAD_TTL_MIN_SECONDS, + REDIS_DAI_CACHE_PAYLOAD_TTL_MAX_SECONDS, + ) + return { + "payload_ttl_seconds": payload_ttl_seconds, + "version_marker_ttl_seconds": _calculate_dai_cache_version_ttl_seconds(settings), + "payload_key_count": 0, + "version_marker_count": 0, + "version_marker_no_expiry_count": 0, + "version_marker_expiring_count": 0, + "version_marker_low_ttl_count": 0, + "version_marker_memory_usage_bytes": 0, + "scan_complete": True, + "scan_error": None, + } + + +def _scan_count_matching_keys(redis_client, pattern): + cursor = 0 + total = 0 + iterations = 0 + while iterations < REDIS_DAI_CACHE_KEYSPACE_SCAN_ITERATIONS: + cursor, keys = redis_client.scan( + cursor=cursor, + match=pattern, + count=REDIS_EXPLORER_MAX_PAGE_SIZE, + ) + cursor = _normalize_redis_cursor(cursor) + total += len(list(keys or [])) + iterations += 1 + if cursor == 0: + break + return total, cursor == 0 + + +def _build_dai_cache_keyspace(redis_client, settings): + keyspace = _build_empty_dai_cache_keyspace(settings) + if redis_client is None: + keyspace["scan_complete"] = False + keyspace["scan_error"] = "Redis client is not available." + return keyspace + + try: + payload_count, payload_scan_complete = _scan_count_matching_keys( + redis_client, + f"{REDIS_DAI_CACHE_KEY_PREFIX}:*", + ) + keyspace["payload_key_count"] = payload_count + keyspace["scan_complete"] = payload_scan_complete + + cursor = 0 + iterations = 0 + while iterations < REDIS_DAI_CACHE_KEYSPACE_SCAN_ITERATIONS: + cursor, keys = redis_client.scan( + cursor=cursor, + match=f"{REDIS_DAI_CACHE_VERSION_KEY_PREFIX}:*", + count=REDIS_EXPLORER_MAX_PAGE_SIZE, + ) + cursor = _normalize_redis_cursor(cursor) + iterations += 1 + for key in list(keys or []): + keyspace["version_marker_count"] += 1 + ttl_seconds = _safe_int(redis_client.ttl(key)) + if ttl_seconds == -1: + keyspace["version_marker_no_expiry_count"] += 1 + elif ttl_seconds is not None and ttl_seconds >= 0: + keyspace["version_marker_expiring_count"] += 1 + if ttl_seconds < keyspace["payload_ttl_seconds"]: + keyspace["version_marker_low_ttl_count"] += 1 + keyspace["version_marker_memory_usage_bytes"] += _get_redis_memory_usage(redis_client, key) or 0 + if cursor == 0: + break + keyspace["scan_complete"] = keyspace["scan_complete"] and cursor == 0 + except Exception as exc: + keyspace["scan_complete"] = False + keyspace["scan_error"] = _safe_error_summary("DAI cache keyspace scan failed", exc) + return keyspace + + +def _redact_sensitive_text(value): + return REDIS_EXPLORER_SENSITIVE_TEXT_PATTERN.sub( + lambda match: f"{match.group(1)}{match.group(2)}{REDIS_EXPLORER_REDACTED_VALUE}", + _safe_str(value), + ) + + +def _contains_sensitive_token(value, sensitive_tokens=REDIS_EXPLORER_RESTRICTED_KEY_TOKENS): + normalized_value = _safe_str(value).lower() + return any(token in normalized_value for token in sensitive_tokens) + + +def _sanitize_preview_payload(value, parent_key=""): + if _contains_sensitive_token(parent_key, REDIS_EXPLORER_SENSITIVE_FIELD_TOKENS): + return REDIS_EXPLORER_REDACTED_VALUE + if isinstance(value, dict): + return { + _safe_str(key): _sanitize_preview_payload(item_value, parent_key=_safe_str(key)) + for key, item_value in list(value.items())[:REDIS_EXPLORER_VALUE_ITEM_LIMIT] + } + if isinstance(value, (list, tuple, set)): + sanitized_items = [ + _sanitize_preview_payload(item, parent_key=parent_key) + for item in list(value)[:REDIS_EXPLORER_VALUE_ITEM_LIMIT] + ] + if len(value) > REDIS_EXPLORER_VALUE_ITEM_LIMIT: + sanitized_items.append(f"... {len(value) - REDIS_EXPLORER_VALUE_ITEM_LIMIT} more item(s)") + return sanitized_items + if isinstance(value, bytes): + return _redact_sensitive_text(value) + if isinstance(value, str): + return _redact_sensitive_text(value) + return value + + +def _build_preview_response(value, preview_format="text"): + truncated = False + if preview_format == "json": + sanitized_payload = _sanitize_preview_payload(value) + preview = json.dumps(sanitized_payload, indent=2, sort_keys=True, default=str) + else: + preview = _redact_sensitive_text(value) + + if len(preview) > REDIS_EXPLORER_PREVIEW_MAX_CHARS: + preview = preview[:REDIS_EXPLORER_PREVIEW_MAX_CHARS] + truncated = True + + return { + "preview": preview, + "preview_format": preview_format, + "redacted": REDIS_EXPLORER_REDACTED_VALUE in preview, + "truncated": truncated, + } + + +def _build_string_preview(value): + text_value = _safe_str(value) + try: + decoded_json = json.loads(text_value) + except (TypeError, ValueError, json.JSONDecodeError): + return _build_preview_response(text_value, preview_format="text") + return _build_preview_response(decoded_json, preview_format="json") + + +def _build_collection_preview(value): + return _build_preview_response(value, preview_format="json") + + +def _escape_redis_glob(value): + escaped_value = "" + for character in _safe_str(value): + if character in ("*", "?", "[", "]", "\\"): + escaped_value += f"\\{character}" + else: + escaped_value += character + return escaped_value + + +def _normalize_key_filter(raw_filter): + filter_text = _safe_str(raw_filter).strip() + if not filter_text: + return "" + return filter_text[:REDIS_EXPLORER_MAX_FILTER_LENGTH] + + +def _build_scan_pattern(raw_filter): + normalized_filter = _normalize_key_filter(raw_filter) + if not normalized_filter: + return "*" + return f"*{_escape_redis_glob(normalized_filter)}*" + + +def _normalize_redis_cursor(raw_cursor): + cursor_value = _safe_int(raw_cursor) + if cursor_value is None or cursor_value < 0: + return 0 + return cursor_value + + +def _normalize_redis_key(raw_key): + key = _safe_str(raw_key).strip() + if not key: + raise ValueError("Redis key is required.") + if len(key) > REDIS_EXPLORER_MAX_KEY_LENGTH: + raise ValueError("Redis key is too long.") + return key + + +def _get_redis_type(redis_client, key): + return _safe_str(redis_client.type(key)).lower() + + +def _get_redis_memory_usage(redis_client, key): + try: + return _safe_int(redis_client.memory_usage(key)) + except (AttributeError, TypeError): + return None + + +def _build_resolution_payload(kind, label, resolved=False, **kwargs): + payload = { + "kind": kind, + "label": label, + "resolved": bool(resolved), + } + payload.update(kwargs) + return payload + + +def _resolve_dai_version_hashes(scope_hashes, dai_hash_resolver=None): + resolver = dai_hash_resolver + if resolver is None: + try: + # Imported lazily because the DAI module initializes Cosmos-backed containers. + from functions_document_access_index import resolve_document_access_cache_version_hashes + + resolver = resolve_document_access_cache_version_hashes + except Exception as exc: + return { + scope_hash: _build_resolution_payload( + "document_access_index_scope_version", + "DAI scope version marker", + resolved=False, + resolution_status="unavailable", + scope_hash=scope_hash, + note=_safe_error_summary("DAI hash resolution unavailable", exc), + ) + for scope_hash in scope_hashes + } + try: + return resolver(scope_hashes) + except Exception as exc: + return { + scope_hash: _build_resolution_payload( + "document_access_index_scope_version", + "DAI scope version marker", + resolved=False, + resolution_status="unavailable", + scope_hash=scope_hash, + note=_safe_error_summary("DAI hash resolution failed", exc), + ) + for scope_hash in scope_hashes + } + + +def _resolve_redis_keys(keys, dai_hash_resolver=None): + resolutions = {} + dai_version_hashes = [] + dai_version_keys_by_hash = {} + for key in list(keys or []): + normalized_key = _safe_str(key) + dai_version_match = REDIS_DAI_VERSION_KEY_PATTERN.match(normalized_key) + if dai_version_match: + scope_hash = dai_version_match.group(1).lower() + dai_version_hashes.append(scope_hash) + dai_version_keys_by_hash[scope_hash] = normalized_key + continue + + dai_payload_match = REDIS_DAI_PAYLOAD_KEY_PATTERN.match(normalized_key) + if dai_payload_match: + operation, source_scope, cache_hash = dai_payload_match.groups() + resolutions[normalized_key] = _build_resolution_payload( + "document_access_index_payload", + "DAI read-through cache payload", + resolved=False, + operation=operation, + source_scope=source_scope, + cache_hash=cache_hash.lower(), + note="Payload keys include filters, scope versions, and process epoch, so the key hash is not reversible from Redis alone.", + ) + continue + + if normalized_key == app_settings_cache.APP_SETTINGS_CACHE_KEY: + resolutions[normalized_key] = _build_resolution_payload( + "app_settings_cache", + "App settings cache payload", + resolved=True, + note="Global app settings cache payload.", + ) + elif normalized_key == app_settings_cache.APP_SETTINGS_CACHE_VERSION_KEY: + resolutions[normalized_key] = _build_resolution_payload( + "app_settings_cache_version", + "App settings cache version", + resolved=True, + note="Global app settings cache invalidation version.", + ) + + if dai_version_hashes: + dai_resolutions = _resolve_dai_version_hashes(dai_version_hashes, dai_hash_resolver=dai_hash_resolver) + for scope_hash, resolution in (dai_resolutions or {}).items(): + key = dai_version_keys_by_hash.get(scope_hash) + if key: + resolutions[key] = resolution + return resolutions + + +def _build_redis_key_summary(redis_client, key, resolution=None): + normalized_key = _safe_str(key) + return { + "key": normalized_key, + "type": _get_redis_type(redis_client, normalized_key), + "ttl_seconds": _safe_int(redis_client.ttl(normalized_key)), + "memory_usage_bytes": _get_redis_memory_usage(redis_client, normalized_key), + "preview_restricted": _contains_sensitive_token(normalized_key), + "resolution": resolution, + } + + +def _build_explorer_unavailable_status(settings, app_cache_client, session_redis_client, session_type): + monitoring_status = get_redis_monitoring_status( + settings, + app_cache_client=app_cache_client, + session_redis_client=session_redis_client, + session_type=session_type, + ) + return { + "success": False, + "status": monitoring_status.get("health", {}).get("status") or REDIS_MONITORING_STATUS_UNAVAILABLE, + "checked_at": monitoring_status.get("checked_at"), + "runtime": monitoring_status.get("runtime", {}), + "last_error": monitoring_status.get("health", {}).get("last_error"), + } + + +def _resolve_explorer_client(settings, app_cache_client=None, session_redis_client=None, session_type=None): + safe_settings = settings if isinstance(settings, dict) else {} + resolved_app_cache_client = ( + app_cache_client + if app_cache_client is not None + else app_settings_cache.get_app_cache_redis_client() + ) + monitoring_client, monitoring_source = _select_monitoring_client( + resolved_app_cache_client, + session_redis_client, + ) + + if not bool(safe_settings.get("enable_redis_cache")): + return None, _build_explorer_unavailable_status( + safe_settings, + resolved_app_cache_client, + session_redis_client, + session_type, + ) + if not str(safe_settings.get("redis_url") or "").strip(): + return None, _build_explorer_unavailable_status( + safe_settings, + resolved_app_cache_client, + session_redis_client, + session_type, + ) + if monitoring_client is None: + return None, _build_explorer_unavailable_status( + safe_settings, + resolved_app_cache_client, + session_redis_client, + session_type, + ) + + return monitoring_client, { + "success": True, + "status": REDIS_MONITORING_STATUS_HEALTHY, + "checked_at": _utc_now_iso(), + "runtime": { + "monitoring_source": monitoring_source, + "client_available": True, + }, + } + + +def _parse_keyspace_database(value): + if isinstance(value, dict): + return { + "keys": _safe_int(value.get("keys")), + "expires": _safe_int(value.get("expires")), + "avg_ttl": _safe_int(value.get("avg_ttl")), + } + + if isinstance(value, bytes): + value = value.decode("utf-8") + + parsed = {} + for segment in str(value or "").split(","): + if "=" not in segment: + continue + key, raw_value = segment.split("=", 1) + parsed[key.strip()] = _safe_int(raw_value.strip()) + + return { + "keys": parsed.get("keys"), + "expires": parsed.get("expires"), + "avg_ttl": parsed.get("avg_ttl"), + } + + +def _extract_keyspace(info): + databases = {} + total_keys = 0 + expiring_keys = 0 + for key, value in (info or {}).items(): + key_name = str(key) + if not key_name.startswith("db"): + continue + database = _parse_keyspace_database(value) + databases[key_name] = database + total_keys += database.get("keys") or 0 + expiring_keys += database.get("expires") or 0 + + return { + "databases": databases, + "total_keys": total_keys, + "expiring_keys": expiring_keys, + } + + +def _build_empty_metric_sections(): + return { + "server": {}, + "clients": {}, + "memory": {}, + "stats": {}, + "keyspace": { + "databases": {}, + "total_keys": 0, + "expiring_keys": 0, + }, + } + + +def _build_info_metric_sections(info): + used_memory = _safe_int(info.get("used_memory")) + maxmemory = _safe_int(info.get("maxmemory")) + keyspace_hits = _safe_int(info.get("keyspace_hits")) + keyspace_misses = _safe_int(info.get("keyspace_misses")) + keyspace_total = (keyspace_hits or 0) + (keyspace_misses or 0) + + return { + "server": { + "redis_version": info.get("redis_version"), + "uptime_in_seconds": _safe_int(info.get("uptime_in_seconds")), + "uptime_in_days": _safe_int(info.get("uptime_in_days")), + }, + "clients": { + "connected_clients": _safe_int(info.get("connected_clients")), + "blocked_clients": _safe_int(info.get("blocked_clients")), + "tracking_clients": _safe_int(info.get("tracking_clients")), + "maxclients": _safe_int(info.get("maxclients")), + }, + "memory": { + "used_memory": used_memory, + "used_memory_human": info.get("used_memory_human"), + "maxmemory": maxmemory, + "maxmemory_human": info.get("maxmemory_human"), + "maxmemory_policy": info.get("maxmemory_policy"), + "usage_percent": _safe_percentage(used_memory, maxmemory), + "mem_fragmentation_ratio": _safe_float(info.get("mem_fragmentation_ratio")), + }, + "stats": { + "total_connections_received": _safe_int(info.get("total_connections_received")), + "total_commands_processed": _safe_int(info.get("total_commands_processed")), + "instantaneous_ops_per_sec": _safe_int(info.get("instantaneous_ops_per_sec")), + "keyspace_hits": keyspace_hits, + "keyspace_misses": keyspace_misses, + "keyspace_hit_rate_percent": _safe_percentage(keyspace_hits, keyspace_total), + "expired_keys": _safe_int(info.get("expired_keys")), + "evicted_keys": _safe_int(info.get("evicted_keys")), + "rejected_connections": _safe_int(info.get("rejected_connections")), + "total_error_replies": _safe_int(info.get("total_error_replies")), + }, + "keyspace": _extract_keyspace(info), + } + + +def _select_monitoring_client(app_cache_client, session_redis_client): + if app_cache_client is not None: + return app_cache_client, "app_cache" + if session_redis_client is not None: + return session_redis_client, "session" + return None, "none" + + +def get_redis_monitoring_status( + settings, + app_cache_client=None, + session_redis_client=None, + session_type=None, + now_func=None, +): + """Return sanitized Redis health and capacity metrics for admin monitoring.""" + safe_settings = settings if isinstance(settings, dict) else {} + enabled = bool(safe_settings.get("enable_redis_cache")) + configured = bool(str(safe_settings.get("redis_url") or "").strip()) + auth_type = str(safe_settings.get("redis_auth_type") or "key").strip().lower() or "key" + resolved_app_cache_client = ( + app_cache_client + if app_cache_client is not None + else app_settings_cache.get_app_cache_redis_client() + ) + app_cache_using_redis = bool(app_settings_cache.app_cache_is_using_redis and resolved_app_cache_client) + session_using_redis = str(session_type or "").strip().lower() == "redis" and session_redis_client is not None + monitoring_client, monitoring_source = _select_monitoring_client( + resolved_app_cache_client, + session_redis_client, + ) + checked_at = (now_func or _utc_now_iso)() + + status = { + "checked_at": checked_at, + "configuration": { + "enabled": enabled, + "configured": configured, + "auth_type": auth_type, + }, + "runtime": { + "app_cache_using_redis": app_cache_using_redis, + "session_using_redis": session_using_redis, + "monitoring_source": monitoring_source, + "client_available": monitoring_client is not None, + }, + "health": { + "status": REDIS_MONITORING_STATUS_DISABLED, + "ping_success": False, + "ping_latency_ms": None, + "last_error": None, + }, + "dai_cache": _build_empty_dai_cache_keyspace(safe_settings), + **_build_empty_metric_sections(), + } + + if not enabled: + return status + + if not configured: + status["health"]["status"] = REDIS_MONITORING_STATUS_NOT_CONFIGURED + status["health"]["last_error"] = "Redis cache is enabled but no host name is configured." + return status + + if monitoring_client is None: + status["health"]["status"] = REDIS_MONITORING_STATUS_UNAVAILABLE + status["health"]["last_error"] = "Redis is configured, but no active runtime Redis client is available." + return status + + ping_started_at = time.perf_counter() + try: + status["health"]["ping_success"] = bool(monitoring_client.ping()) + status["health"]["ping_latency_ms"] = round((time.perf_counter() - ping_started_at) * 1000, 2) + except Exception as exc: + status["health"]["status"] = REDIS_MONITORING_STATUS_ERROR + status["health"]["last_error"] = _safe_error_summary("Redis ping failed", exc) + return status + + try: + info = monitoring_client.info() + except Exception as exc: + status["health"]["status"] = REDIS_MONITORING_STATUS_DEGRADED + status["health"]["last_error"] = _safe_error_summary("Redis INFO metrics failed", exc) + return status + + if not isinstance(info, dict): + status["health"]["status"] = REDIS_MONITORING_STATUS_DEGRADED + status["health"]["last_error"] = "Redis INFO metrics returned an unexpected response." + return status + + status.update(_build_info_metric_sections(info)) + status["dai_cache"] = _build_dai_cache_keyspace(monitoring_client, safe_settings) + status["health"]["status"] = ( + REDIS_MONITORING_STATUS_HEALTHY + if status["health"]["ping_success"] + else REDIS_MONITORING_STATUS_DEGRADED + ) + return status + + +def get_redis_explorer_keys( + settings, + cursor=0, + page_size=REDIS_EXPLORER_DEFAULT_PAGE_SIZE, + key_filter="", + app_cache_client=None, + session_redis_client=None, + session_type=None, + dai_hash_resolver=None, +): + """Return one cursor-paginated Redis key page with safe metadata only.""" + redis_client, status = _resolve_explorer_client( + settings, + app_cache_client=app_cache_client, + session_redis_client=session_redis_client, + session_type=session_type, + ) + if redis_client is None: + return { + **status, + "keys": [], + "cursor": str(_normalize_redis_cursor(cursor)), + "next_cursor": "0", + "has_more": False, + "page_size": _clamp_int( + page_size, + REDIS_EXPLORER_DEFAULT_PAGE_SIZE, + 1, + REDIS_EXPLORER_MAX_PAGE_SIZE, + ), + "filter": _normalize_key_filter(key_filter), + } + + normalized_cursor = _normalize_redis_cursor(cursor) + normalized_page_size = _clamp_int( + page_size, + REDIS_EXPLORER_DEFAULT_PAGE_SIZE, + 1, + REDIS_EXPLORER_MAX_PAGE_SIZE, + ) + normalized_filter = _normalize_key_filter(key_filter) + scan_pattern = _build_scan_pattern(normalized_filter) + next_cursor = normalized_cursor + collected_keys = [] + scan_iterations = 0 + + while len(collected_keys) < normalized_page_size and scan_iterations < REDIS_EXPLORER_MAX_SCAN_ITERATIONS: + raw_cursor, scanned_keys = redis_client.scan( + cursor=next_cursor, + match=scan_pattern, + count=normalized_page_size, + ) + next_cursor = _normalize_redis_cursor(raw_cursor) + scan_iterations += 1 + for key in list(scanned_keys or []): + if len(collected_keys) >= normalized_page_size: + break + collected_keys.append(_safe_str(key)) + if next_cursor == 0: + break + + resolutions = _resolve_redis_keys(collected_keys, dai_hash_resolver=dai_hash_resolver) + return { + **status, + "keys": [ + _build_redis_key_summary(redis_client, key, resolution=resolutions.get(key)) + for key in collected_keys + ], + "cursor": str(normalized_cursor), + "next_cursor": str(next_cursor), + "has_more": next_cursor != 0, + "page_size": normalized_page_size, + "filter": normalized_filter, + "scan_iterations": scan_iterations, + } + + +def _read_redis_value_preview(redis_client, key, key_type): + if key_type == "string": + return _build_string_preview(redis_client.get(key)) + if key_type == "hash": + _cursor, values = redis_client.hscan(key, cursor=0, count=REDIS_EXPLORER_VALUE_ITEM_LIMIT) + return _build_collection_preview({ + _safe_str(field): _safe_str(value) + for field, value in dict(values or {}).items() + }) + if key_type == "list": + return _build_collection_preview([ + _safe_str(value) + for value in list(redis_client.lrange(key, 0, REDIS_EXPLORER_VALUE_ITEM_LIMIT - 1) or []) + ]) + if key_type == "set": + _cursor, values = redis_client.sscan(key, cursor=0, count=REDIS_EXPLORER_VALUE_ITEM_LIMIT) + return _build_collection_preview([_safe_str(value) for value in list(values or [])]) + if key_type == "zset": + return _build_collection_preview([ + { + "value": _safe_str(value), + "score": score, + } + for value, score in list(redis_client.zrange( + key, + 0, + REDIS_EXPLORER_VALUE_ITEM_LIMIT - 1, + withscores=True, + ) or []) + ]) + if key_type == "stream": + return _build_collection_preview([ + { + "id": _safe_str(entry_id), + "fields": { + _safe_str(field): _safe_str(value) + for field, value in dict(fields or {}).items() + }, + } + for entry_id, fields in list(redis_client.xrange( + key, + min="-", + max="+", + count=REDIS_EXPLORER_VALUE_ITEM_LIMIT, + ) or []) + ]) + return { + "preview": f"Preview is not available for Redis key type '{key_type}'.", + "preview_format": "text", + "redacted": False, + "truncated": False, + } + + +def get_redis_explorer_value( + settings, + key, + app_cache_client=None, + session_redis_client=None, + session_type=None, + dai_hash_resolver=None, +): + """Return sanitized metadata and preview content for one Redis key.""" + normalized_key = _normalize_redis_key(key) + redis_client, status = _resolve_explorer_client( + settings, + app_cache_client=app_cache_client, + session_redis_client=session_redis_client, + session_type=session_type, + ) + if redis_client is None: + return { + **status, + "key": normalized_key, + "type": "unknown", + "preview": "", + "preview_format": "text", + "redacted": False, + "truncated": False, + "preview_restricted": False, + } + + key_type = _get_redis_type(redis_client, normalized_key) + if key_type in ("", "none"): + return { + **status, + "success": False, + "status": "not_found", + "key": normalized_key, + "type": "none", + "ttl_seconds": -2, + "memory_usage_bytes": None, + "preview": "Redis key was not found.", + "preview_format": "text", + "redacted": False, + "truncated": False, + "preview_restricted": False, + } + + resolutions = _resolve_redis_keys([normalized_key], dai_hash_resolver=dai_hash_resolver) + metadata = { + "key": normalized_key, + "type": key_type, + "ttl_seconds": _safe_int(redis_client.ttl(normalized_key)), + "memory_usage_bytes": _get_redis_memory_usage(redis_client, normalized_key), + "preview_restricted": _contains_sensitive_token(normalized_key), + "resolution": resolutions.get(normalized_key), + } + if metadata["preview_restricted"]: + return { + **status, + **metadata, + "preview": REDIS_EXPLORER_RESTRICTED_PREVIEW, + "preview_format": "text", + "redacted": True, + "truncated": False, + } + + preview = _read_redis_value_preview(redis_client, normalized_key, key_type) + return { + **status, + **metadata, + **preview, + } diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index 59a73c33..d48079f5 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -90,6 +90,25 @@ def resolve_admin_settings_secret_value(field_name, submitted_value, existing_se return str(_get_nested_setting_value(existing_settings, field_name) or '').strip() +def normalize_document_access_index_required_settings(settings): + """Force DAI operational settings that are required for the default read path.""" + if not isinstance(settings, dict): + return False + + changed = False + required_flags = { + 'enable_document_access_index_container': True, + 'enable_document_access_index_write_through': True, + 'enable_document_access_index_reads': True, + 'enable_startup_document_access_index_backfill': True, + } + for key, required_value in required_flags.items(): + if settings.get(key) != required_value: + settings[key] = required_value + changed = True + return changed + + def redact_admin_settings_secrets_for_form(settings): redacted_settings = copy.deepcopy(settings or {}) for field_name in ADMIN_SETTINGS_FORM_SECRET_FIELDS: @@ -715,44 +734,6 @@ def _update_cache(stage): _update_cache("after_version_bump") -def _refresh_app_settings_cache_after_write(settings_payload, context="app_settings_write"): - """Update shared/local settings cache around a version bump.""" - cache_updater = getattr(app_settings_cache, "update_settings_cache", None) - version_bumper = getattr(app_settings_cache, "bump_app_settings_cache_version", None) - - def _update_cache(stage): - if not callable(cache_updater): - return - try: - cache_updater(copy.deepcopy(settings_payload)) - except Exception as cache_error: - log_event( - "App settings cache update failed after settings write.", - extra={ - "context": context, - "stage": stage, - "error": str(cache_error) - }, - level=logging.WARNING - ) - - _update_cache("before_version_bump") - - if callable(version_bumper): - try: - version_bumper() - except Exception as version_error: - log_event( - "App settings cache version bump failed after settings write.", - extra={ - "context": context, - "error": str(version_error) - }, - level=logging.WARNING - ) - - _update_cache("after_version_bump") - def get_settings(use_cosmos=False, include_source=False): import secrets default_settings = { @@ -931,6 +912,28 @@ def get_settings(use_cosmos=False, include_source=False): 'redis_key': '', 'redis_auth_type': '', + # App Maintenance Settings + 'enable_app_maintenance': True, + 'enable_startup_app_maintenance': True, + 'app_maintenance_check_interval_seconds': 3600, + 'app_maintenance_job_lease_seconds': 300, + 'app_maintenance_apply_cosmos_indexing_policies': False, + 'enable_document_access_index_container': True, + 'enable_document_access_index_write_through': True, + 'enable_document_access_index_reads': True, + 'enable_dai_debug': False, + 'enable_document_access_index_shadow_validation': False, + 'enable_startup_document_access_index_backfill': True, + 'document_access_index_backfill_batch_size': 200, + 'document_access_index_repair_batch_size': 100, + 'document_access_index_active_maintenance_interval_seconds': 30, + 'enable_document_access_index_cache': True, + 'document_access_index_cache_ttl_seconds': 900, + 'custom_pages_nav_cache_ttl_seconds': 60, + 'chat_bootstrap_cache_ttl_seconds': 300, + 'enable_conversation_cache': True, + 'conversation_cache_ttl_seconds': 120, + # Cosmos DB Throughput Scale Settings **get_default_cosmos_throughput_settings(), @@ -1343,11 +1346,18 @@ def _format_result(settings_payload, source): migration_updated = apply_custom_endpoint_setting_migration(merged) assignment_settings_updated = normalize_group_workflow_assignment_settings(merged) promoted_popular_settings_updated = normalize_agents_page_promoted_popular_settings(merged) + document_access_index_settings_updated = normalize_document_access_index_required_settings(merged) merged['enable_tabular_processing_plugin'] = is_tabular_processing_enabled(merged) # If merging added anything new, upsert back to Cosmos so future reads remain up to date - if merge_changed or migration_updated or assignment_settings_updated or promoted_popular_settings_updated: + if ( + merge_changed + or migration_updated + or assignment_settings_updated + or promoted_popular_settings_updated + or document_access_index_settings_updated + ): cosmos_settings_container.upsert_item(merged) _refresh_app_settings_cache_after_write(merged, context="merge_upsert") @@ -1396,6 +1406,7 @@ def update_settings(new_settings): settings_item.update(new_settings) normalize_group_workflow_assignment_settings(settings_item) normalize_agents_page_promoted_popular_settings(settings_item) + normalize_document_access_index_required_settings(settings_item) settings_item['enable_multi_model_endpoints'] = coerce_multi_model_endpoint_enablement( existing_multi_endpoint_enabled, settings_item.get('enable_multi_model_endpoints', False), diff --git a/application/single_app/functions_shared_cache.py b/application/single_app/functions_shared_cache.py new file mode 100644 index 00000000..11c2c5ba --- /dev/null +++ b/application/single_app/functions_shared_cache.py @@ -0,0 +1,464 @@ +# functions_shared_cache.py +"""Shared versioned cache helpers with Redis-first and Cosmos fallback behavior.""" + +import copy +import hashlib +import json +import logging +import threading +import time +from datetime import datetime, timedelta, timezone + +from azure.core import MatchConditions +from config import cosmos_settings_container +from functions_appinsights import log_event + + +SHARED_CACHE_ENTRY_DOC_TYPE = 'shared_cache_entry' +SHARED_CACHE_ENTRY_PREFIX = 'shared_cache_entry:' +SHARED_CACHE_VERSION_DOC_TYPE = 'cache_version' +SHARED_CACHE_VERSION_READ_TTL_SECONDS = 15 +SHARED_CACHE_VERSION_BUMP_MAX_RETRIES = 5 + +_version_cache = {} +_version_cache_lock = None +_shared_cache_metrics = { + 'counts': {}, + 'last_event': None, +} +_shared_cache_metrics_lock = threading.Lock() + + +def _get_version_cache_lock(): + global _version_cache_lock + if _version_cache_lock is None: + _version_cache_lock = threading.Lock() + return _version_cache_lock + + +def _utc_now(): + return datetime.now(timezone.utc) + + +def _utc_now_iso(): + return _utc_now().isoformat() + + +def _normalize_cache_version(value): + if isinstance(value, bytes): + value = value.decode('utf-8') + try: + return int(value) + except (TypeError, ValueError): + return 0 + + +def _is_not_found_error(exc): + return getattr(exc, 'status_code', None) == 404 + + +def _log_cache_warning(operation, error, extra=None): + context = { + 'operation': operation, + 'error': str(error), + } + if extra: + context.update(extra) + log_event( + f"[SharedCache] {operation} failed.", + extra=context, + level=logging.WARNING, + exceptionTraceback=True, + ) + + +def _hash_cache_key(cache_key): + normalized_key = str(cache_key or '').strip() + if not normalized_key: + return '' + return hashlib.sha256(normalized_key.encode('utf-8')).hexdigest()[:16] + + +def _record_cache_event(operation, namespace=None, key=None, backend='cosmos', result='success', extra=None): + safe_event = { + 'operation': str(operation or '').strip() or 'unknown', + 'namespace': str(namespace or '').strip() or 'shared', + 'backend': str(backend or '').strip() or 'unknown', + 'result': str(result or '').strip() or 'unknown', + } + if key is not None: + safe_event['cache_key_hash'] = _hash_cache_key(key) + if extra: + safe_event.update(extra) + + metric_key = ':'.join([ + safe_event['namespace'], + safe_event['operation'], + safe_event['backend'], + safe_event['result'], + ]) + with _shared_cache_metrics_lock: + counts = _shared_cache_metrics.setdefault('counts', {}) + counts[metric_key] = int(counts.get(metric_key) or 0) + 1 + _shared_cache_metrics['last_event'] = copy.deepcopy(safe_event) + + log_event( + f"[SharedCache] {safe_event['operation']} {safe_event['result']}.", + extra=safe_event, + level=logging.INFO, + debug_only=True, + ) + + +def get_shared_cache_metrics(): + """Return lightweight in-process shared cache counters for diagnostics/tests.""" + with _shared_cache_metrics_lock: + return copy.deepcopy(_shared_cache_metrics) + + +def reset_shared_cache_metrics(): + """Reset lightweight shared cache counters for deterministic functional tests.""" + with _shared_cache_metrics_lock: + _shared_cache_metrics['counts'] = {} + _shared_cache_metrics['last_event'] = None + + +def _build_entry_doc_id(namespace, key): + normalized_namespace = str(namespace or '').strip() + normalized_key = str(key or '').strip() + if not normalized_namespace or not normalized_key: + raise ValueError('Shared cache namespace and key are required.') + return f"{SHARED_CACHE_ENTRY_PREFIX}{normalized_namespace}:{normalized_key}" + + +def _build_redis_key(namespace, key): + return _build_entry_doc_id(namespace, key) + + +def _serialize_for_cache(value): + return json.dumps(value, default=str) + + +def _deserialize_from_cache(value): + if value is None: + return None + if isinstance(value, bytes): + value = value.decode('utf-8') + return json.loads(value) + + +def _get_expires_at(ttl_seconds=None): + if ttl_seconds is None: + return None + return _utc_now() + timedelta(seconds=max(int(ttl_seconds), 0)) + + +def _is_expired(entry): + expires_at = entry.get('expires_at') if isinstance(entry, dict) else None + if not expires_at: + return False + try: + return datetime.fromisoformat(expires_at) <= _utc_now() + except (TypeError, ValueError): + return True + + +def set_shared_cache_entry(namespace, key, value, ttl_seconds=None, redis_client=None, allow_cosmos_fallback=True): + """Set a shared cache entry, optionally falling back to Cosmos when Redis is unavailable.""" + redis_key = _build_redis_key(namespace, key) + if redis_client is not None: + try: + serialized_value = _serialize_for_cache(value) + if ttl_seconds is None: + redis_client.set(redis_key, serialized_value) + else: + redis_client.setex(redis_key, int(ttl_seconds), serialized_value) + _record_cache_event('set', namespace, redis_key, backend='redis', result='success') + return True + except Exception as ex: + _record_cache_event('set', namespace, redis_key, backend='redis', result='error') + _log_cache_warning('redis_set_shared_cache_entry', ex, {'cache_key': redis_key}) + if not allow_cosmos_fallback: + _record_cache_event('set', namespace, redis_key, backend='cosmos', result='skipped') + return False + elif not allow_cosmos_fallback: + _record_cache_event('set', namespace, redis_key, backend='cosmos', result='skipped') + return False + + expires_at = _get_expires_at(ttl_seconds) + body = { + 'id': redis_key, + 'type': SHARED_CACHE_ENTRY_DOC_TYPE, + 'namespace': str(namespace), + 'key': str(key), + 'value': copy.deepcopy(value), + 'expires_at': expires_at.isoformat() if expires_at else None, + 'updated_at': _utc_now_iso(), + } + try: + cosmos_settings_container.upsert_item(body) + _record_cache_event('set', namespace, redis_key, backend='cosmos', result='success') + return True + except Exception as ex: + _record_cache_event('set', namespace, redis_key, backend='cosmos', result='error') + _log_cache_warning('cosmos_set_shared_cache_entry', ex, {'cache_key': redis_key}) + return False + + +def get_shared_cache_entry(namespace, key, redis_client=None, allow_cosmos_fallback=True): + """Read a shared cache entry, optionally falling back to Cosmos when Redis is unavailable.""" + redis_key = _build_redis_key(namespace, key) + if redis_client is not None: + try: + value = redis_client.get(redis_key) + if value is not None: + _record_cache_event('get', namespace, redis_key, backend='redis', result='hit') + return _deserialize_from_cache(value) + _record_cache_event('get', namespace, redis_key, backend='redis', result='miss') + if not allow_cosmos_fallback: + _record_cache_event('get', namespace, redis_key, backend='cosmos', result='skipped') + return None + except Exception as ex: + _record_cache_event('get', namespace, redis_key, backend='redis', result='error') + _log_cache_warning('redis_get_shared_cache_entry', ex, {'cache_key': redis_key}) + if not allow_cosmos_fallback: + _record_cache_event('get', namespace, redis_key, backend='cosmos', result='skipped') + return None + elif not allow_cosmos_fallback: + _record_cache_event('get', namespace, redis_key, backend='cosmos', result='skipped') + return None + + try: + entry = cosmos_settings_container.read_item(item=redis_key, partition_key=redis_key) + if _is_expired(entry): + delete_shared_cache_entry(namespace, key) + _record_cache_event('get', namespace, redis_key, backend='cosmos', result='expired') + return None + _record_cache_event('get', namespace, redis_key, backend='cosmos', result='hit') + return copy.deepcopy(entry.get('value')) + except Exception as ex: + if _is_not_found_error(ex): + _record_cache_event('get', namespace, redis_key, backend='cosmos', result='miss') + return None + _record_cache_event('get', namespace, redis_key, backend='cosmos', result='error') + _log_cache_warning('cosmos_get_shared_cache_entry', ex, {'cache_key': redis_key}) + return None + + +def delete_shared_cache_entry(namespace, key, redis_client=None, allow_cosmos_fallback=True): + """Delete a shared cache entry from Redis and optionally Cosmos without failing callers.""" + redis_key = _build_redis_key(namespace, key) + deleted = True + if redis_client is not None: + try: + redis_client.delete(redis_key) + _record_cache_event('delete', namespace, redis_key, backend='redis', result='success') + if not allow_cosmos_fallback: + _record_cache_event('delete', namespace, redis_key, backend='cosmos', result='skipped') + return deleted + except Exception as ex: + deleted = False + _record_cache_event('delete', namespace, redis_key, backend='redis', result='error') + _log_cache_warning('redis_delete_shared_cache_entry', ex, {'cache_key': redis_key}) + if not allow_cosmos_fallback: + _record_cache_event('delete', namespace, redis_key, backend='cosmos', result='skipped') + return False + elif not allow_cosmos_fallback: + _record_cache_event('delete', namespace, redis_key, backend='cosmos', result='skipped') + return True + + try: + cosmos_settings_container.delete_item(item=redis_key, partition_key=redis_key) + _record_cache_event('delete', namespace, redis_key, backend='cosmos', result='success') + except Exception as ex: + if not _is_not_found_error(ex): + deleted = False + _record_cache_event('delete', namespace, redis_key, backend='cosmos', result='error') + _log_cache_warning('cosmos_delete_shared_cache_entry', ex, {'cache_key': redis_key}) + else: + _record_cache_event('delete', namespace, redis_key, backend='cosmos', result='miss') + return deleted + + +def _get_version_cache_key(container, version_doc_id): + return f"{id(container)}:{version_doc_id}" + + +def _set_cached_shared_cache_version(container, version_doc_id, version): + cache_lock = _get_version_cache_lock() + with cache_lock: + _version_cache[_get_version_cache_key(container, version_doc_id)] = { + 'value': version, + 'expires_at': time.time() + SHARED_CACHE_VERSION_READ_TTL_SECONDS, + } + + +def get_shared_cache_version(version_doc_id, default_version=0, container=None, use_local_cache=True): + """Read a Cosmos-backed shared cache version with a short local TTL.""" + cache_container = container or cosmos_settings_container + cache_key = _get_version_cache_key(cache_container, version_doc_id) + now = time.time() + cache_lock = _get_version_cache_lock() + if use_local_cache: + with cache_lock: + cached = _version_cache.get(cache_key) + if cached and cached.get('expires_at', 0) > now: + _record_cache_event( + 'version_get', + namespace='shared_cache_version', + key=version_doc_id, + backend='local', + result='hit', + ) + return _normalize_cache_version(cached.get('value')) + + try: + doc = cache_container.read_item(item=version_doc_id, partition_key=version_doc_id) + version = _normalize_cache_version(doc.get('version', default_version)) + _record_cache_event( + 'version_get', + namespace='shared_cache_version', + key=version_doc_id, + backend='cosmos', + result='hit', + ) + except Exception as ex: + if not _is_not_found_error(ex): + _record_cache_event( + 'version_get', + namespace='shared_cache_version', + key=version_doc_id, + backend='cosmos', + result='error', + ) + _log_cache_warning('cosmos_get_shared_cache_version', ex, {'version_doc_id': version_doc_id}) + else: + _record_cache_event( + 'version_get', + namespace='shared_cache_version', + key=version_doc_id, + backend='cosmos', + result='miss', + ) + version = _normalize_cache_version(default_version) + + if use_local_cache: + _set_cached_shared_cache_version(cache_container, version_doc_id, version) + return version + + +def ensure_shared_cache_version_doc(version_doc_id, initial_version=0, description='', container=None): + """Create a cache version document if it is missing and return its current state.""" + cache_container = container or cosmos_settings_container + try: + doc = cache_container.read_item(item=version_doc_id, partition_key=version_doc_id) + return { + 'id': version_doc_id, + 'created': False, + 'version': _normalize_cache_version(doc.get('version', initial_version)), + 'description': doc.get('description') or description, + } + except Exception as ex: + if not _is_not_found_error(ex): + _log_cache_warning('cosmos_read_shared_cache_version_doc', ex, {'version_doc_id': version_doc_id}) + raise + + version = _normalize_cache_version(initial_version) + body = { + 'id': version_doc_id, + 'type': SHARED_CACHE_VERSION_DOC_TYPE, + 'description': description, + 'version': version, + 'created_at': _utc_now_iso(), + 'updated_at': _utc_now_iso(), + } + try: + cache_container.create_item(body=body) + return { + 'id': version_doc_id, + 'created': True, + 'version': version, + 'description': description, + } + except Exception as ex: + if getattr(ex, 'status_code', None) == 409: + doc = cache_container.read_item(item=version_doc_id, partition_key=version_doc_id) + return { + 'id': version_doc_id, + 'created': False, + 'version': _normalize_cache_version(doc.get('version', initial_version)), + 'description': doc.get('description') or description, + } + _log_cache_warning('cosmos_create_shared_cache_version_doc', ex, {'version_doc_id': version_doc_id}) + raise + + +def bump_shared_cache_version(version_doc_id, description='', container=None): + """Increment a Cosmos-backed shared cache version document with optimistic concurrency.""" + cache_container = container or cosmos_settings_container + ensure_shared_cache_version_doc(version_doc_id, description=description, container=cache_container) + + last_conflict = None + for _ in range(SHARED_CACHE_VERSION_BUMP_MAX_RETRIES): + try: + doc = cache_container.read_item(item=version_doc_id, partition_key=version_doc_id) + current_version = _normalize_cache_version(doc.get('version')) + next_version = current_version + 1 + replacement_doc = dict(doc) + replacement_doc.update({ + 'id': version_doc_id, + 'type': SHARED_CACHE_VERSION_DOC_TYPE, + 'description': doc.get('description') or description, + 'version': next_version, + 'updated_at': _utc_now_iso(), + }) + cache_container.replace_item( + item=version_doc_id, + body=replacement_doc, + etag=doc.get('_etag'), + match_condition=MatchConditions.IfNotModified, + ) + _set_cached_shared_cache_version(cache_container, version_doc_id, next_version) + _record_cache_event( + 'version_bump', + namespace='shared_cache_version', + key=version_doc_id, + backend='cosmos', + result='success', + extra={'version': next_version}, + ) + return next_version + except Exception as ex: + status_code = getattr(ex, 'status_code', None) + if _is_not_found_error(ex): + ensure_shared_cache_version_doc(version_doc_id, description=description, container=cache_container) + last_conflict = ex + continue + if status_code in (409, 412): + last_conflict = ex + _record_cache_event( + 'version_bump', + namespace='shared_cache_version', + key=version_doc_id, + backend='cosmos', + result='conflict', + ) + continue + _record_cache_event( + 'version_bump', + namespace='shared_cache_version', + key=version_doc_id, + backend='cosmos', + result='error', + ) + _log_cache_warning('cosmos_bump_shared_cache_version', ex, {'version_doc_id': version_doc_id}) + raise + + _log_cache_warning( + 'cosmos_bump_shared_cache_version_conflict', + last_conflict or RuntimeError('Version bump retry limit exceeded.'), + {'version_doc_id': version_doc_id, 'max_retries': SHARED_CACHE_VERSION_BUMP_MAX_RETRIES}, + ) + if last_conflict: + raise last_conflict + raise RuntimeError('Shared cache version bump retry limit exceeded.') diff --git a/application/single_app/functions_simplechat_operations.py b/application/single_app/functions_simplechat_operations.py index 6b6cba7e..10c17d70 100644 --- a/application/single_app/functions_simplechat_operations.py +++ b/application/single_app/functions_simplechat_operations.py @@ -49,6 +49,7 @@ persist_collaboration_message, ) from functions_documents import allowed_file, create_document, process_document_upload_background, update_document +from functions_chat_bootstrap_cache import bump_chat_bootstrap_global_cache_version from functions_group import ( assert_group_role, check_group_status_allows_operation, @@ -885,6 +886,7 @@ def make_group_inactive_for_current_user( } ) updated_group_doc = cosmos_groups_container.upsert_item(group_doc) + bump_chat_bootstrap_global_cache_version(reason="group_marked_inactive") log_group_status_change( group_id=resolved_group_id, @@ -1106,6 +1108,7 @@ def add_group_member_for_current_user( group_doc["modifiedDate"] = datetime.utcnow().isoformat() updated_group_doc = cosmos_groups_container.upsert_item(group_doc) + bump_chat_bootstrap_global_cache_version(reason="group_member_added") _log_group_member_addition( actor_user=current_user, diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index eecc5763..e73a05c0 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -104,6 +104,7 @@ normalize_chart_kind, user_request_supports_proactive_charts, ) +from functions_conversation_cache import invalidate_conversation_cache_for_item from functions_conversation_metadata import collect_conversation_metadata, update_conversation_with_metadata from functions_conversation_unread import mark_conversation_unread from functions_image_messages import build_image_message_documents, decode_image_content @@ -2263,6 +2264,7 @@ def _create_personal_conversation(user_id, conversation_id=None): conversation_item['added_to_activity_log'] = True cosmos_conversations_container.upsert_item(conversation_item) + invalidate_conversation_cache_for_item(conversation_item, reason="conversation_created") return conversation_item @@ -11655,6 +11657,7 @@ def _load_or_create_analyze_conversation(user_id, conversation_id=None): ) conversation_item['added_to_activity_log'] = True cosmos_conversations_container.upsert_item(conversation_item) + invalidate_conversation_cache_for_item(conversation_item, reason="conversation_created") return conversation_item def execute_document_action_chat_request(data=None, publish_background_event=None, forced_action_type=None): @@ -11912,6 +11915,7 @@ def execute_document_action_chat_request(data=None, publish_background_event=Non if title_updated: conversation_item['last_updated'] = datetime.utcnow().isoformat() cosmos_conversations_container.upsert_item(conversation_item) + invalidate_conversation_cache_for_item(conversation_item, reason="conversation_title_initialized") if callable(publish_background_event): publish_background_event(_build_conversation_metadata_stream_event(conversation_item)) @@ -12191,6 +12195,7 @@ def execute_document_action_chat_request(data=None, publish_background_event=Non debug_print(f'[ChatDocumentAnalysis] Conversation metadata update failed: {exc}') cosmos_conversations_container.upsert_item(conversation_item) + invalidate_conversation_cache_for_item(conversation_item, reason="document_action_chat_completed") debug_print( '[ChatDocumentAction] Execution completed | ' f'user_id={user_id} | ' @@ -12419,6 +12424,7 @@ def generate_image_from_proposal(): conversation_item['last_updated'] = datetime.utcnow().isoformat() cosmos_conversations_container.upsert_item(conversation_item) + invalidate_conversation_cache_for_item(conversation_item, reason="chat_image_proposal_generated") image_doc = image_result.pop('image_message', {}) or {} image_doc_metadata = image_doc.get('metadata') if isinstance(image_doc.get('metadata'), dict) else {} @@ -13324,6 +13330,7 @@ def result_requires_message_reload(result: Any) -> bool: conversation_item['last_updated'] = datetime.utcnow().isoformat() cosmos_conversations_container.upsert_item(conversation_item) # Update timestamp and potentially title + invalidate_conversation_cache_for_item(conversation_item, reason="conversation_title_initialized") assistant_message_id, thought_tracker, assistant_thread_attempt, response_message_context = _initialize_assistant_response_tracking( conversation_id=conversation_id, @@ -13432,6 +13439,7 @@ def result_requires_message_reload(result: Any) -> bool: # Update conversation's last_updated conversation_item['last_updated'] = datetime.utcnow().isoformat() cosmos_conversations_container.upsert_item(conversation_item) + invalidate_conversation_cache_for_item(conversation_item, reason="chat_safety_blocked") # Return a normal 200 with a special field: blocked=True return jsonify({ @@ -14247,6 +14255,7 @@ def result_requires_message_reload(result: Any) -> bool: conversation_item['last_updated'] = datetime.utcnow().isoformat() cosmos_conversations_container.upsert_item(conversation_item) + invalidate_conversation_cache_for_item(conversation_item, reason="chat_image_generated") return jsonify({ 'reply': "Image loading...", @@ -15866,6 +15875,7 @@ def gpt_error(e): # Add any other final updates to conversation_item if needed (like classifications if not done earlier) cosmos_conversations_container.upsert_item(conversation_item) + invalidate_conversation_cache_for_item(conversation_item, reason="chat_completed") # --------------------------------------------------------------------- # 8) Return final success (even if AI generated an error message) @@ -16887,6 +16897,7 @@ def build_streaming_capability_usage(): conversation_item['last_updated'] = datetime.utcnow().isoformat() cosmos_conversations_container.upsert_item(conversation_item) + invalidate_conversation_cache_for_item(conversation_item, reason="conversation_title_initialized") if title_updated: yield _build_conversation_metadata_stream_event(conversation_item) @@ -17056,6 +17067,7 @@ def record_and_publish_streaming_thought(thought_payload): conversation_item['last_updated'] = datetime.utcnow().isoformat() cosmos_conversations_container.upsert_item(conversation_item) + invalidate_conversation_cache_for_item(conversation_item, reason="chat_safety_blocked") final_data = make_json_serializable({ 'content': blocked_msg_content.strip(), @@ -18260,6 +18272,7 @@ def finalize_cancelled_stream_response(): cosmos_messages_container.upsert_item(assistant_doc) conversation_item['last_updated'] = datetime.utcnow().isoformat() cosmos_conversations_container.upsert_item(conversation_item) + invalidate_conversation_cache_for_item(conversation_item, reason="chat_stream_stopped") message_persisted = True log_event( @@ -18964,6 +18977,7 @@ def finalize_cancelled_agent_stream_response(): ) cosmos_conversations_container.upsert_item(conversation_item) + invalidate_conversation_cache_for_item(conversation_item, reason="chat_stream_completed") # Send final message with metadata final_data = make_json_serializable({ diff --git a/application/single_app/route_backend_collaboration.py b/application/single_app/route_backend_collaboration.py index 78a09a48..1d00cec8 100644 --- a/application/single_app/route_backend_collaboration.py +++ b/application/single_app/route_backend_collaboration.py @@ -44,6 +44,7 @@ update_personal_collaboration_member_role, update_personal_collaboration_title, ) +from functions_conversation_cache import bump_conversation_cache_version from functions_group import assert_group_role, check_group_status_allows_operation, find_group_by_id, require_active_group from functions_image_messages import decode_image_content, get_complete_image_content, is_blob_backed_image_message, is_external_image_url from functions_message_masking import ( @@ -1806,6 +1807,7 @@ def mark_collaboration_conversation_read_api(conversation_id): current_user['user_id'], conversation_id, ) + bump_conversation_cache_version(current_user['user_id'], reason="collaboration_marked_read") return jsonify({ 'success': True, diff --git a/application/single_app/route_backend_control_center.py b/application/single_app/route_backend_control_center.py index 40dba76b..c820fd6b 100644 --- a/application/single_app/route_backend_control_center.py +++ b/application/single_app/route_backend_control_center.py @@ -10,6 +10,7 @@ from config import * from functions_authentication import * +from functions_chat_bootstrap_cache import bump_chat_bootstrap_global_cache_version from functions_settings import * from functions_logging import * from functions_activity_logging import * @@ -3265,6 +3266,7 @@ def api_update_group_status(group_id): # Update in database cosmos_groups_container.upsert_item(group) + bump_chat_bootstrap_global_cache_version(reason="group_status_updated") # Log to activity_logs container for audit trail from functions_activity_logging import log_group_status_change @@ -3708,6 +3710,7 @@ def api_admin_add_group_member(group_id): # Save group cosmos_groups_container.upsert_item(group) + bump_chat_bootstrap_global_cache_version(reason="group_member_added") # Determine the action source (single add vs bulk CSV) source = data.get('source', 'csv') # Default to 'csv' for backward compatibility @@ -4139,6 +4142,7 @@ def api_update_public_workspace_status(workspace_id): # Update in database cosmos_public_workspaces_container.upsert_item(workspace) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_status_updated") # Log to activity_logs container for audit trail from functions_activity_logging import log_public_workspace_status_change @@ -4297,6 +4301,7 @@ def api_bulk_public_workspace_action(): }) cosmos_public_workspaces_container.upsert_item(workspace) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_bulk_status_updated") # Log activity from functions_activity_logging import log_public_workspace_status_change @@ -4527,6 +4532,7 @@ def api_admin_add_workspace_member(workspace_id): # Save workspace cosmos_public_workspaces_container.upsert_item(workspace) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_member_added") # Determine the action source source = data.get('source', 'csv') @@ -4636,6 +4642,7 @@ def api_admin_add_workspace_member_single(workspace_id): # Save workspace cosmos_public_workspaces_container.upsert_item(workspace) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_member_added") # Log to activity logs activity_record = { @@ -7147,6 +7154,7 @@ def _execute_take_ownership(approval, executor_id, executor_email, executor_name group['modifiedDate'] = datetime.utcnow().isoformat() cosmos_groups_container.upsert_item(group) + bump_chat_bootstrap_global_cache_version(reason="group_ownership_transferred") # Log to activity logs activity_record = { @@ -7284,6 +7292,7 @@ def _execute_take_workspace_ownership(approval, executor_id, executor_email, exe workspace['modifiedDate'] = datetime.utcnow().isoformat() cosmos_public_workspaces_container.upsert_item(workspace) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_ownership_transferred") # Log to activity logs activity_record = { @@ -7369,6 +7378,7 @@ def _execute_transfer_ownership(approval, executor_id, executor_email, executor_ group['modifiedDate'] = datetime.utcnow().isoformat() cosmos_groups_container.upsert_item(group) + bump_chat_bootstrap_global_cache_version(reason="group_ownership_transferred") # Log to activity logs activity_record = { @@ -7509,6 +7519,7 @@ def _execute_transfer_workspace_ownership(approval, executor_id, executor_email, workspace['modifiedDate'] = datetime.utcnow().isoformat() cosmos_public_workspaces_container.upsert_item(workspace) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_ownership_transferred") # Log to activity logs activity_record = { diff --git a/application/single_app/route_backend_conversations.py b/application/single_app/route_backend_conversations.py index cd611726..d53fbb43 100644 --- a/application/single_app/route_backend_conversations.py +++ b/application/single_app/route_backend_conversations.py @@ -32,6 +32,14 @@ ) from functions_conversation_metadata import get_conversation_metadata, update_conversation_with_metadata from functions_conversation_unread import clear_conversation_unread, normalize_conversation_unread_state +from functions_conversation_cache import ( + build_conversation_cache_key, + bump_conversation_cache_version, + get_cached_conversation_payload, + get_conversation_cache_settings, + invalidate_conversation_cache_for_item, + set_cached_conversation_payload, +) from functions_image_messages import decode_image_content, get_complete_image_content, hydrate_image_messages, is_blob_backed_image_message, is_external_image_url from functions_notifications import mark_chat_response_notifications_read_for_conversation from flask import Response, request, stream_with_context @@ -40,6 +48,7 @@ delete_chat_upload_workspace_documents_for_conversation, serialize_chat_upload_workspace_documents_for_conversation, ) +from functions_group import get_user_groups from functions_message_artifacts import ( build_message_artifact_payload_map, filter_assistant_artifact_items, @@ -107,6 +116,43 @@ def normalize_chat_type(conversation_item): } +def _build_conversation_cache_access_parameters(user_id): + """Return current group-access inputs that affect collaboration feed/search visibility.""" + try: + group_docs = get_user_groups(user_id) + except Exception as exc: + log_event( + f"[ConversationCache] Failed to build group access cache fingerprint for {user_id}: {exc}", + level=logging.WARNING, + exceptionTraceback=True, + debug_only=True, + ) + return None + + groups = [] + for group_doc in group_docs or []: + if not isinstance(group_doc, dict): + continue + group_id = str(group_doc.get('id') or '').strip() + if not group_id: + continue + groups.append({ + 'id': group_id, + 'status': str(group_doc.get('status') or 'active'), + 'updated_at': str( + group_doc.get('updated_at') + or group_doc.get('last_updated') + or group_doc.get('_ts') + or '' + ), + }) + + groups.sort(key=lambda group_item: group_item['id']) + return { + 'groups': groups, + } + + def _normalize_workspace_document_delete_ids(raw_document_ids): if raw_document_ids is None: return [] @@ -713,6 +759,23 @@ def _authorize_personal_conversation_read(user_id, conversation_id): return conversation_item +def _invalidate_conversation_cache_after_message_mutation(conversation_id, user_id, reason): + """Invalidate conversation caches after message-level changes without failing the caller.""" + try: + conversation_item = _authorize_personal_conversation_read(user_id, conversation_id) + except Exception as exc: + log_event( + f"[ConversationCache] Failed to load conversation {conversation_id} for message mutation invalidation: {exc}", + level=logging.WARNING, + exceptionTraceback=True, + debug_only=True, + ) + bump_conversation_cache_version(user_id, reason=reason) + return + + invalidate_conversation_cache_for_item(conversation_item, reason=reason) + + def _authorize_image_conversation_read(user_id, conversation_id): """Authorize image reads for either personal or collaborative conversations.""" try: @@ -948,12 +1011,34 @@ def get_conversations(): user_id = get_current_user_id() if not user_id: return jsonify({'error': 'User not authenticated'}), 401 - query = f"SELECT * FROM c WHERE c.user_id = '{user_id}' ORDER BY c.last_updated DESC" - items = list(cosmos_conversations_container.query_items(query=query, enable_cross_partition_query=True)) + settings = get_settings() + cache_settings = get_conversation_cache_settings(settings) + cache_key = None + if cache_settings.get('enabled'): + cache_key = build_conversation_cache_key(user_id, "list", parameters={"include_hidden": True}) + if cache_key: + cached_payload = get_cached_conversation_payload(cache_key, settings=settings) + if isinstance(cached_payload, dict) and isinstance(cached_payload.get('conversations'), list): + return jsonify(cached_payload), 200 + + query = "SELECT * FROM c WHERE c.user_id = @user_id ORDER BY c.last_updated DESC" + items = list(cosmos_conversations_container.query_items( + query=query, + parameters=[{'name': '@user_id', 'value': user_id}], + enable_cross_partition_query=True, + )) normalized_items = [normalize_conversation_unread_state(item) for item in items] - return jsonify({ + payload = { 'conversations': normalized_items - }), 200 + } + if cache_key: + set_cached_conversation_payload( + cache_key, + payload, + ttl_seconds=cache_settings.get('ttl_seconds'), + settings=settings, + ) + return jsonify(payload), 200 @bp.route('/api/conversations/feed', methods=['GET']) @@ -978,6 +1063,29 @@ def get_conversations_feed(): source_offsets = get_conversation_feed_source_offsets(cursor_data) if cursor_is_compatible else {} include_priority = not cursor_is_compatible + settings = get_settings() + cache_settings = get_conversation_cache_settings(settings) + access_parameters = _build_conversation_cache_access_parameters(user_id) + feed_cache_parameters = { + "search_term": search_term, + "include_hidden": include_hidden, + "page_size": page_size, + "cursor": request.args.get('cursor') or "", + "include_priority": include_priority, + "access": access_parameters, + } + feed_cache_key = None + if cache_settings.get('enabled') and access_parameters is not None: + feed_cache_key = build_conversation_cache_key( + user_id, + "feed", + parameters=feed_cache_parameters, + ) + if feed_cache_key: + cached_feed_payload = get_cached_conversation_payload(feed_cache_key, settings=settings) + if isinstance(cached_feed_payload, dict) and isinstance(cached_feed_payload.get('conversations'), list): + return jsonify(cached_feed_payload), 200 + feed_payload = _build_conversation_feed( user_id=user_id, page_size=page_size, @@ -988,6 +1096,13 @@ def get_conversations_feed(): ) feed_payload['search_term'] = search_term feed_payload['include_hidden'] = include_hidden + if feed_cache_key: + set_cached_conversation_payload( + feed_cache_key, + feed_payload, + ttl_seconds=cache_settings.get('ttl_seconds'), + settings=settings, + ) return jsonify(feed_payload), 200 except Exception as exc: log_event( @@ -1012,6 +1127,7 @@ def create_conversation(): data.get('initial_message') or data.get('message') or data.get('title') or '' ) conversation_item = create_personal_conversation_for_current_user(title=initial_title) + bump_conversation_cache_version(user_id, reason="conversation_created") return jsonify({ 'conversation_id': conversation_item.get('id'), @@ -1053,6 +1169,7 @@ def update_conversation_title(conversation_id): # Write back to Cosmos DB cosmos_conversations_container.upsert_item(conversation_item) + bump_conversation_cache_version(user_id, reason="conversation_title_updated") return jsonify({ 'message': 'Conversation updated', @@ -1183,6 +1300,7 @@ def delete_conversation(conversation_id): item=conversation_id, partition_key=conversation_id ) + bump_conversation_cache_version(user_id, reason="conversation_deleted") # TODO: Delete any facts that were stored with this conversation. except Exception as e: return jsonify({ @@ -1289,7 +1407,10 @@ def delete_multiple_conversations(): except Exception as e: print(f"Error deleting conversation {conversation_id}: {str(e)}") failed_ids.append(conversation_id) - + + if success_count: + bump_conversation_cache_version(user_id, reason="conversations_bulk_deleted") + return jsonify({ "success": True, "deleted_count": success_count, @@ -1326,6 +1447,7 @@ def toggle_conversation_pin(conversation_id): # Update in Cosmos DB cosmos_conversations_container.upsert_item(conversation_item) + bump_conversation_cache_version(user_id, reason="conversation_pin_toggled") return jsonify({ 'success': True, @@ -1368,6 +1490,7 @@ def toggle_conversation_hide(conversation_id): # Update in Cosmos DB cosmos_conversations_container.upsert_item(conversation_item) + bump_conversation_cache_version(user_id, reason="conversation_hide_toggled") return jsonify({ 'success': True, @@ -1430,7 +1553,10 @@ def bulk_pin_conversations(): except Exception as e: print(f"Error updating conversation {conversation_id}: {str(e)}") failed_ids.append(conversation_id) - + + if success_count: + bump_conversation_cache_version(user_id, reason="conversations_bulk_pin_updated") + return jsonify({ "success": True, "updated_count": success_count, @@ -1488,7 +1614,10 @@ def bulk_hide_conversations(): except Exception as e: print(f"Error updating conversation {conversation_id}: {str(e)}") failed_ids.append(conversation_id) - + + if success_count: + bump_conversation_cache_version(user_id, reason="conversations_bulk_hide_updated") + return jsonify({ "success": True, "updated_count": success_count, @@ -1523,6 +1652,7 @@ def get_conversation_metadata_api(conversation_id): _, updated = normalize_chat_type(conversation_item) if updated: cosmos_conversations_container.upsert_item(conversation_item) + invalidate_conversation_cache_for_item(conversation_item, reason="conversation_chat_type_normalized") linked_workspace_documents = [] try: @@ -1586,8 +1716,15 @@ def mark_conversation_read_api(conversation_id): if conversation_item.get('user_id') != user_id: return jsonify({'error': 'Forbidden'}), 403 - conversation_item = clear_conversation_unread(conversation_item) - cosmos_conversations_container.upsert_item(conversation_item) + conversation_state_changed = ( + conversation_item.get('has_unread_assistant_response') is True + or conversation_item.get('last_unread_assistant_message_id') is not None + or conversation_item.get('last_unread_assistant_at') is not None + ) + if conversation_state_changed: + conversation_item = clear_conversation_unread(conversation_item) + cosmos_conversations_container.upsert_item(conversation_item) + bump_conversation_cache_version(user_id, reason="conversation_marked_read") notifications_marked_read = mark_chat_response_notifications_read_for_conversation( user_id, @@ -1599,6 +1736,7 @@ def mark_conversation_read_api(conversation_id): 'conversation_id': conversation_id, 'has_unread_assistant_response': False, 'notifications_marked_read': notifications_marked_read, + 'conversation_state_changed': conversation_state_changed, }), 200 except CosmosResourceNotFoundError: return jsonify({'error': 'Conversation not found'}), 404 @@ -1758,6 +1896,7 @@ def patch_conversation_scope_lock(conversation_id): user_id, new_value, ) + invalidate_conversation_cache_for_item(conversation_item, reason="conversation_scope_lock_updated") return jsonify({ "success": True, @@ -1844,6 +1983,34 @@ def search_conversations(): 'success': False, 'error': 'Search term must be at least 3 characters' }), 400 + + settings = get_settings() + cache_settings = get_conversation_cache_settings(settings) + access_parameters = _build_conversation_cache_access_parameters(user_id) + search_cache_parameters = { + 'search_term': search_term, + 'match_mode': match_mode, + 'date_from': date_from, + 'date_to': date_to, + 'chat_types': sorted([str(item) for item in chat_types or []]), + 'classifications': sorted([str(item) for item in classifications or []]), + 'has_files': bool(has_files), + 'has_images': bool(has_images), + 'page': page, + 'per_page': per_page, + 'access': access_parameters, + } + search_cache_key = None + if cache_settings.get('enabled') and access_parameters is not None: + search_cache_key = build_conversation_cache_key( + user_id, + "search", + parameters=search_cache_parameters, + ) + if search_cache_key: + cached_search_payload = get_cached_conversation_payload(search_cache_key, settings=settings) + if isinstance(cached_search_payload, dict) and cached_search_payload.get('success') is True: + return jsonify(cached_search_payload), 200 selected_chat_types = _expand_search_chat_type_filters(chat_types) @@ -2000,14 +2167,22 @@ def search_conversations(): end_idx = start_idx + per_page paginated_results = results[start_idx:end_idx] - return jsonify({ + payload = { 'success': True, 'total_results': total_results, 'page': page, 'total_pages': total_pages, 'per_page': per_page, 'results': paginated_results - }), 200 + } + if search_cache_key: + set_cached_conversation_payload( + search_cache_key, + payload, + ttl_seconds=cache_settings.get('ttl_seconds'), + settings=settings, + ) + return jsonify(payload), 200 except Exception as e: log_event( @@ -2281,7 +2456,12 @@ def delete_message(message_id): cosmos_messages_container.delete_item(msg_id, partition_key=conversation_id) deleted_message_ids.append(msg_id) - + + _invalidate_conversation_cache_after_message_mutation( + conversation_id, + user_id, + "message_deleted", + ) return jsonify({ 'success': True, 'deleted_message_ids': deleted_message_ids, @@ -2454,7 +2634,12 @@ def retry_message(message_id): 'metadata': new_metadata } cosmos_messages_container.upsert_item(new_user_message) - + + _invalidate_conversation_cache_after_message_mutation( + conversation_id, + user_id, + "message_retry_created", + ) # Build chat request parameters from original message metadata chat_request = { 'message': user_content, @@ -2671,7 +2856,12 @@ def edit_message(message_id): 'metadata': new_metadata } cosmos_messages_container.upsert_item(new_user_message) - + + _invalidate_conversation_cache_after_message_mutation( + conversation_id, + user_id, + "message_edit_created", + ) # Build chat request parameters from original message metadata # Keep all original settings (model, reasoning, doc search, etc.) chat_request = { @@ -2833,7 +3023,12 @@ def switch_attempt(message_id): msg_attempt = msg['metadata']['thread_info'].get('thread_attempt', 0) msg['metadata']['thread_info']['active_thread'] = (msg_attempt == target_attempt) cosmos_messages_container.upsert_item(msg) - + + _invalidate_conversation_cache_after_message_mutation( + conversation_id, + user_id, + "message_attempt_switched", + ) return jsonify({ 'success': True, 'target_attempt': target_attempt, diff --git a/application/single_app/route_backend_documents.py b/application/single_app/route_backend_documents.py index 47bf2933..f7d0a6b7 100644 --- a/application/single_app/route_backend_documents.py +++ b/application/single_app/route_backend_documents.py @@ -12,6 +12,14 @@ build_synced_document_delete_guard, ) from functions_notifications import create_notification, delete_notifications_by_metadata +from functions_document_access_index import ( + DOCUMENT_ACCESS_SCOPE_PERSONAL, + is_document_access_shadow_validation_enabled, + query_items_with_cosmos_diagnostics, + query_document_access_index_documents, + query_document_access_index_legacy_count, + query_document_access_index_tag_counts, +) from utils_cache import invalidate_personal_search_cache from functions_debug import * from functions_activity_logging import log_document_upload, log_document_metadata_update_transaction @@ -694,6 +702,7 @@ def api_get_user_documents(): # Add user_id prefix for shared_user_ids with status user_id_prefix = f"{user_id}," query_params.append({"name": "@user_id_prefix", "value": user_id_prefix}) + shadow_tags_filter = [] # Replace the main ownership/shared condition query_conditions[0] = ( @@ -753,6 +762,7 @@ def api_get_user_documents(): tags_list = sanitize_tags_for_filter(tags_filter) if tags_list: + shadow_tags_filter = tags_list # Each tag must exist in the document's tags array for idx, tag in enumerate(tags_list): param_name = f"@tag_{param_count}_{idx}" @@ -762,6 +772,17 @@ def api_get_user_documents(): # Combine conditions into the WHERE clause where_clause = " AND ".join(query_conditions) + shadow_filters = { + 'search': search_term, + 'classification': classification_filter, + 'classification_none_matches_literal': True, + 'author': author_filter, + 'keywords': keywords_filter, + 'abstract': abstract_filter, + 'tags': shadow_tags_filter, + 'array_match_mode': 'contains', + } + used_document_access_index = False # --- 3) Query matching documents, then collapse to current revisions before paginating --- try: @@ -771,17 +792,71 @@ def api_get_user_documents(): FROM c WHERE {where_clause} """ - matching_docs = list(cosmos_user_documents_container.query_items( - query=data_query_str, - parameters=query_params, - enable_cross_partition_query=True - )) - - current_docs = sort_documents( - select_current_documents(matching_docs), - sort_by=sort_by, - sort_order=sort_order, + index_read_result = query_document_access_index_documents( + source_scope=DOCUMENT_ACCESS_SCOPE_PERSONAL, + user_id=user_id, + filters=shadow_filters, ) + + if index_read_result.get('success'): + used_document_access_index = True + current_docs = sort_documents( + index_read_result.get('documents', []), + sort_by=sort_by, + sort_order=sort_order, + ) + if is_document_access_shadow_validation_enabled(): + try: + matching_docs, source_query_metrics = query_items_with_cosmos_diagnostics( + cosmos_user_documents_container, + diagnostics_label='source_documents', + query=data_query_str, + parameters=query_params, + enable_cross_partition_query=True + ) + source_current_docs = sort_documents( + select_current_documents(matching_docs), + sort_by=sort_by, + sort_order=sort_order, + ) + validate_document_access_index_shadow( + source_current_docs, + source_scope=DOCUMENT_ACCESS_SCOPE_PERSONAL, + user_id=user_id, + filters=shadow_filters, + source_query_metrics=source_query_metrics, + context='api_get_user_documents', + ) + except Exception as shadow_error: + log_event( + '[DocumentAccessIndex] Shadow validation source query failed after DAI read succeeded.', + extra={'source_scope': DOCUMENT_ACCESS_SCOPE_PERSONAL, 'error': str(shadow_error)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + else: + matching_docs, source_query_metrics = query_items_with_cosmos_diagnostics( + cosmos_user_documents_container, + diagnostics_label='source_documents', + collect_diagnostics=is_document_access_shadow_validation_enabled(), + query=data_query_str, + parameters=query_params, + enable_cross_partition_query=True + ) + + current_docs = sort_documents( + select_current_documents(matching_docs), + sort_by=sort_by, + sort_order=sort_order, + ) + validate_document_access_index_shadow( + current_docs, + source_scope=DOCUMENT_ACCESS_SCOPE_PERSONAL, + user_id=user_id, + filters=shadow_filters, + source_query_metrics=source_query_metrics, + context='api_get_user_documents', + ) total_count = len(current_docs) docs = current_docs[offset:offset + page_size] @@ -803,23 +878,32 @@ def api_get_user_documents(): # --- new: do we have any legacy documents? --- - try: - legacy_q = """ - SELECT VALUE COUNT(1) - FROM c - WHERE c.user_id = @user_id - AND NOT IS_DEFINED(c.percentage_complete) - """ - legacy_docs = list( - cosmos_user_documents_container.query_items( - query=legacy_q, - parameters=[{"name":"@user_id","value":user_id}], - enable_cross_partition_query=True - ) + legacy_count = 0 + if used_document_access_index: + legacy_count_result = query_document_access_index_legacy_count( + source_scope=DOCUMENT_ACCESS_SCOPE_PERSONAL, + user_id=user_id, ) - legacy_count = legacy_docs[0] if legacy_docs else 0 - except Exception as e: - debug_print(f"Error executing legacy query: {e}") + if legacy_count_result.get('success'): + legacy_count = legacy_count_result.get('legacy_count', 0) + else: + try: + legacy_q = """ + SELECT VALUE COUNT(1) + FROM c + WHERE c.user_id = @user_id + AND NOT IS_DEFINED(c.percentage_complete) + """ + legacy_docs = list( + cosmos_user_documents_container.query_items( + query=legacy_q, + parameters=[{"name":"@user_id","value":user_id}], + enable_cross_partition_query=True + ) + ) + legacy_count = legacy_docs[0] if legacy_docs else 0 + except Exception as e: + debug_print(f"Error executing legacy query: {e}") # --- 5) Return results --- file_downloads_enabled = is_personal_workspace_file_download_enabled(get_settings()) @@ -1349,10 +1433,20 @@ def api_get_workspace_tags(): if not user_id: return jsonify({'error': 'User not authenticated'}), 401 - from functions_documents import get_workspace_tags + from functions_documents import build_workspace_tags_from_counts, get_workspace_tags try: - tags = get_workspace_tags(user_id) + index_tag_result = query_document_access_index_tag_counts( + DOCUMENT_ACCESS_SCOPE_PERSONAL, + user_id=user_id, + ) + if index_tag_result.get('success'): + tags = build_workspace_tags_from_counts( + index_tag_result.get('tag_counts', {}), + user_id, + ) + else: + tags = get_workspace_tags(user_id) return jsonify({'tags': tags}), 200 except Exception as e: return jsonify({'error': str(e)}), 500 @@ -2115,17 +2209,22 @@ def api_approve_shared_document(document_id): if updated: document_item['shared_user_ids'] = new_shared_user_ids document_item['last_updated'] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') - cosmos_user_documents_container.upsert_item(document_item) + persisted_document = cosmos_user_documents_container.upsert_item(document_item) + document_for_sync = persisted_document if isinstance(persisted_document, dict) else document_item + sync_document_access_index_for_document_fail_open( + document_for_sync, + operation='document_share_approved', + ) # Update all chunks with the new shared_user_ids try: - chunks = get_all_chunks(document_id, document_item.get('user_id')) + chunks = get_all_chunks(document_id, document_for_sync.get('user_id')) for chunk in chunks: chunk_id = chunk.get('id') if chunk_id: try: update_chunk_metadata( chunk_id=chunk_id, - user_id=document_item.get('user_id'), + user_id=document_for_sync.get('user_id'), group_id=None, public_workspace_id=None, document_id=document_id, diff --git a/application/single_app/route_backend_group_documents.py b/application/single_app/route_backend_group_documents.py index af6d69ac..386a976a 100644 --- a/application/single_app/route_backend_group_documents.py +++ b/application/single_app/route_backend_group_documents.py @@ -13,6 +13,15 @@ build_synced_document_delete_guard, ) from functions_notifications import create_notification, delete_notifications_by_metadata +from functions_document_access_index import ( + DOCUMENT_ACCESS_SCOPE_GROUP, + build_document_access_scope_key, + is_document_access_shadow_validation_enabled, + query_items_with_cosmos_diagnostics, + query_document_access_index_documents, + query_document_access_index_legacy_count, + query_document_access_index_tag_counts, +) from functions_simplechat_operations import download_blob_content, queue_generated_document_processing from utils_cache import invalidate_group_search_cache from functions_debug import * @@ -503,6 +512,7 @@ def api_get_group_documents(): query_conditions = [group_condition] param_count = 0 + shadow_tags_filter = [] if search_term: param_name = f"@search_term_{param_count}" @@ -541,6 +551,7 @@ def api_get_group_documents(): from functions_documents import sanitize_tags_for_filter tags_list = sanitize_tags_for_filter(tags_filter) if tags_list: + shadow_tags_filter = tags_list for idx, tag in enumerate(tags_list): param_name = f"@tag_{param_count}_{idx}" query_conditions.append(f"ARRAY_CONTAINS(c.tags, {param_name})") @@ -550,6 +561,17 @@ def api_get_group_documents(): where_clause = " AND ".join(query_conditions) # --- 3) Query matching documents, then collapse to current revisions before paginating --- + shadow_filters = { + 'search': search_term, + 'classification': classification_filter, + 'classification_none_matches_literal': False, + 'author': author_filter, + 'keywords': keywords_filter, + 'abstract': abstract_filter, + 'tags': shadow_tags_filter, + 'array_match_mode': 'contains', + } + used_document_access_index = False try: offset = (page - 1) * page_size data_query_str = f""" @@ -557,16 +579,70 @@ def api_get_group_documents(): FROM c WHERE {where_clause} """ - matching_docs = list(cosmos_group_documents_container.query_items( - query=data_query_str, - parameters=query_params, - enable_cross_partition_query=True - )) - current_docs = sort_documents( - select_current_documents(matching_docs), - sort_by=sort_by, - sort_order=sort_order, + index_read_result = query_document_access_index_documents( + source_scope=DOCUMENT_ACCESS_SCOPE_GROUP, + group_ids=validated_group_ids, + filters=shadow_filters, ) + + if index_read_result.get('success'): + used_document_access_index = True + current_docs = sort_documents( + index_read_result.get('documents', []), + sort_by=sort_by, + sort_order=sort_order, + ) + if is_document_access_shadow_validation_enabled(): + try: + matching_docs, source_query_metrics = query_items_with_cosmos_diagnostics( + cosmos_group_documents_container, + diagnostics_label='source_documents', + query=data_query_str, + parameters=query_params, + enable_cross_partition_query=True + ) + source_current_docs = sort_documents( + select_current_documents(matching_docs), + sort_by=sort_by, + sort_order=sort_order, + ) + validate_document_access_index_shadow( + source_current_docs, + source_scope=DOCUMENT_ACCESS_SCOPE_GROUP, + group_ids=validated_group_ids, + filters=shadow_filters, + source_query_metrics=source_query_metrics, + context='api_get_group_documents', + ) + except Exception as shadow_error: + log_event( + '[DocumentAccessIndex] Shadow validation source query failed after DAI read succeeded.', + extra={'source_scope': DOCUMENT_ACCESS_SCOPE_GROUP, 'error': str(shadow_error)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + else: + matching_docs, source_query_metrics = query_items_with_cosmos_diagnostics( + cosmos_group_documents_container, + diagnostics_label='source_documents', + collect_diagnostics=is_document_access_shadow_validation_enabled(), + query=data_query_str, + parameters=query_params, + enable_cross_partition_query=True + ) + current_docs = sort_documents( + select_current_documents(matching_docs), + sort_by=sort_by, + sort_order=sort_order, + ) + validate_document_access_index_shadow( + current_docs, + source_scope=DOCUMENT_ACCESS_SCOPE_GROUP, + group_ids=validated_group_ids, + filters=shadow_filters, + source_query_metrics=source_query_metrics, + context='api_get_group_documents', + ) total_count = len(current_docs) docs = current_docs[offset:offset + page_size] @@ -602,26 +678,16 @@ def api_get_group_documents(): # --- new: do we have any legacy documents? --- - legacy_count = 0 - try: - if len(validated_group_ids) == 1: - legacy_q = """ - SELECT VALUE COUNT(1) - FROM c - WHERE c.group_id = @group_id - AND NOT IS_DEFINED(c.percentage_complete) - """ - legacy_docs = list( - cosmos_group_documents_container.query_items( - query=legacy_q, - parameters=[{"name":"@group_id","value":validated_group_ids[0]}], - enable_cross_partition_query=True - ) - ) - legacy_count = legacy_docs[0] if legacy_docs else 0 - else: - # For multi-group, check each group - for gid in validated_group_ids: + if used_document_access_index: + legacy_count_result = query_document_access_index_legacy_count( + source_scope=DOCUMENT_ACCESS_SCOPE_GROUP, + group_ids=validated_group_ids, + ) + legacy_count = legacy_count_result.get('legacy_count', 0) if legacy_count_result.get('success') else 0 + else: + legacy_count = 0 + try: + if len(validated_group_ids) == 1: legacy_q = """ SELECT VALUE COUNT(1) FROM c @@ -631,13 +697,30 @@ def api_get_group_documents(): legacy_docs = list( cosmos_group_documents_container.query_items( query=legacy_q, - parameters=[{"name":"@group_id","value":gid}], + parameters=[{"name":"@group_id","value":validated_group_ids[0]}], enable_cross_partition_query=True ) ) - legacy_count += legacy_docs[0] if legacy_docs else 0 - except Exception as e: - print(f"Error executing legacy query: {e}") + legacy_count = legacy_docs[0] if legacy_docs else 0 + else: + # For multi-group, check each group + for gid in validated_group_ids: + legacy_q = """ + SELECT VALUE COUNT(1) + FROM c + WHERE c.group_id = @group_id + AND NOT IS_DEFINED(c.percentage_complete) + """ + legacy_docs = list( + cosmos_group_documents_container.query_items( + query=legacy_q, + parameters=[{"name":"@group_id","value":gid}], + enable_cross_partition_query=True + ) + ) + legacy_count += legacy_docs[0] if legacy_docs else 0 + except Exception as e: + print(f"Error executing legacy query: {e}") # --- 5) Return results --- app_settings = get_settings() @@ -1904,9 +1987,10 @@ def api_get_group_document_tags(): except (ValueError, LookupError, PermissionError): group_ids = [] - from functions_documents import get_workspace_tags + from functions_documents import build_workspace_tags_from_counts, get_workspace_tags all_tags = {} + validated_group_ids = [] for gid in group_ids: group_doc = find_group_by_id(gid) if not group_doc: @@ -1914,8 +1998,23 @@ def api_get_group_document_tags(): role = get_user_role_in_group(group_doc, user_id) if not role: continue - - tags = get_workspace_tags(user_id, group_id=gid) + validated_group_ids.append(gid) + + index_tag_result = query_document_access_index_tag_counts( + DOCUMENT_ACCESS_SCOPE_GROUP, + group_ids=validated_group_ids, + ) if validated_group_ids else {'success': False} + + for gid in validated_group_ids: + if index_tag_result.get('success'): + scope_key = build_document_access_scope_key(DOCUMENT_ACCESS_SCOPE_GROUP, gid) + tags = build_workspace_tags_from_counts( + index_tag_result.get('tag_counts_by_scope_key', {}).get(scope_key, {}), + user_id, + group_id=gid, + ) + else: + tags = get_workspace_tags(user_id, group_id=gid) for tag in tags: if tag['name'] in all_tags: all_tags[tag['name']]['count'] += tag['count'] diff --git a/application/single_app/route_backend_groups.py b/application/single_app/route_backend_groups.py index 627ebea1..05d4188b 100644 --- a/application/single_app/route_backend_groups.py +++ b/application/single_app/route_backend_groups.py @@ -2,6 +2,7 @@ from config import * from functions_authentication import * +from functions_chat_bootstrap_cache import bump_chat_bootstrap_global_cache_version from functions_group import * from functions_debug import debug_print from functions_notifications import create_notification @@ -263,6 +264,7 @@ def api_update_group_download_settings(group_id): group_doc["modifiedDate"] = datetime.utcnow().isoformat() try: cosmos_groups_container.upsert_item(group_doc) + bump_chat_bootstrap_global_cache_version(reason="group_updated") except exceptions.CosmosHttpResponseError as ex: return jsonify({"error": str(ex)}), 400 @@ -337,6 +339,7 @@ def api_update_group(group_id): except exceptions.CosmosHttpResponseError as ex: return jsonify({"error": str(ex)}), 400 + bump_chat_bootstrap_global_cache_version(reason="group_updated") return jsonify({"message": "Group updated", "id": group_id}), 200 @bp.route("/api/groups//logo", methods=["GET"]) @@ -557,6 +560,8 @@ def approve_reject_request(group_id, request_id): group_doc["pendingUsers"] = pending_list group_doc["modifiedDate"] = datetime.utcnow().isoformat() cosmos_groups_container.upsert_item(group_doc) + if action == "approve": + bump_chat_bootstrap_global_cache_version(reason="group_member_request_approved") return jsonify({"message": msg}), 200 @@ -634,6 +639,7 @@ def remove_member(group_id, member_id): cosmos_groups_container.upsert_item(group_doc) if removed: + bump_chat_bootstrap_global_cache_version(reason="group_member_removed") # Log activity for self-removal from functions_activity_logging import log_group_member_deleted user_email = user_info.get("email", "unknown") @@ -686,6 +692,7 @@ def remove_member(group_id, member_id): cosmos_groups_container.upsert_item(group_doc) if removed: + bump_chat_bootstrap_global_cache_version(reason="group_member_removed") # Log activity for admin/owner removal from functions_activity_logging import log_group_member_deleted user_email = user_info.get("email", "unknown") @@ -767,6 +774,7 @@ def update_member_role(group_id, member_id): group_doc["modifiedDate"] = datetime.utcnow().isoformat() cosmos_groups_container.upsert_item(group_doc) + bump_chat_bootstrap_global_cache_version(reason="group_member_role_updated") # Log activity for role change try: @@ -935,6 +943,7 @@ def transfer_ownership(group_id): group_doc["modifiedDate"] = datetime.utcnow().isoformat() cosmos_groups_container.upsert_item(group_doc) + bump_chat_bootstrap_global_cache_version(reason="group_ownership_transferred") return jsonify({"message": "Ownership transferred successfully"}), 200 diff --git a/application/single_app/route_backend_notifications.py b/application/single_app/route_backend_notifications.py index defd4305..3754d0d1 100644 --- a/application/single_app/route_backend_notifications.py +++ b/application/single_app/route_backend_notifications.py @@ -2,6 +2,7 @@ from config import * from functions_authentication import * +from functions_conversation_cache import bump_conversation_cache_version from functions_settings import * from functions_notifications import * from swagger_wrapper import swagger_route, get_auth_security @@ -120,6 +121,7 @@ def api_mark_notification_read(notification_id): success = mark_notification_read(notification_id, user_id) if success: + bump_conversation_cache_version(user_id, reason="notification_marked_read") return jsonify({ 'success': True, 'message': 'Notification marked as read' @@ -150,6 +152,7 @@ def api_dismiss_notification(notification_id): success = dismiss_notification(notification_id, user_id) if success: + bump_conversation_cache_version(user_id, reason="notification_dismissed") return jsonify({ 'success': True, 'message': 'Notification dismissed' @@ -178,6 +181,8 @@ def api_mark_all_read(): try: user_id = get_current_user_id() count = mark_all_read(user_id) + if count: + bump_conversation_cache_version(user_id, reason="notifications_marked_read") return jsonify({ 'success': True, diff --git a/application/single_app/route_backend_plugins.py b/application/single_app/route_backend_plugins.py index 4e3d983e..a4acbba3 100644 --- a/application/single_app/route_backend_plugins.py +++ b/application/single_app/route_backend_plugins.py @@ -4,7 +4,7 @@ import re import builtins import json -from azure.cosmos import CosmosClient +import azure.cosmos as azure_cosmos from azure.cosmos.exceptions import CosmosHttpResponseError from azure.identity import DefaultAzureCredential from flask import Blueprint, jsonify, request, current_app, session @@ -1941,7 +1941,7 @@ def capture_response_headers(response_headers, _): headers.clear() headers.update(response_headers) - client = CosmosClient( + client = azure_cosmos.CosmosClient( endpoint, credential=DefaultAzureCredential() if auth_type == 'identity' else auth_key, timeout=timeout, diff --git a/application/single_app/route_backend_public_documents.py b/application/single_app/route_backend_public_documents.py index a39ba286..f9da07f4 100644 --- a/application/single_app/route_backend_public_documents.py +++ b/application/single_app/route_backend_public_documents.py @@ -1,5 +1,6 @@ # route_backend_public_documents.py +import logging from datetime import datetime, timezone from config import * @@ -8,6 +9,17 @@ from functions_settings import * from functions_public_workspaces import * from functions_documents import * +from functions_appinsights import log_event +from functions_document_access_index import ( + DOCUMENT_ACCESS_SCOPE_PUBLIC, + build_document_access_scope_key, + is_document_access_shadow_validation_enabled, + query_document_access_index_documents, + query_document_access_index_legacy_count, + query_document_access_index_tag_counts, + query_items_with_cosmos_diagnostics, + validate_document_access_index_shadow, +) from functions_file_sync import ( FILE_SYNC_SCOPE_PUBLIC, apply_synced_document_delete_action, @@ -205,6 +217,7 @@ def api_list_public_documents(): conds = ['c.public_workspace_id = @ws'] params = [{'name':'@ws','value':active_ws}] param_count = 0 + shadow_tags_filter = [] if search: conds.append('(CONTAINS(LOWER(c.file_name), LOWER(@search)) OR CONTAINS(LOWER(c.title), LOWER(@search)))') params.append({'name':'@search','value':search}) @@ -241,6 +254,7 @@ def api_list_public_documents(): from functions_documents import sanitize_tags_for_filter tags_list = sanitize_tags_for_filter(tags_filter) if tags_list: + shadow_tags_filter = tags_list for idx, tag in enumerate(tags_list): param_name = f"@tag_{param_count}_{idx}" conds.append(f"ARRAY_CONTAINS(c.tags, {param_name})") @@ -248,27 +262,110 @@ def api_list_public_documents(): param_count += len(tags_list) where = ' AND '.join(conds) + shadow_filters = { + 'search': search, + 'classification': classification_filter, + 'classification_none_matches_literal': True, + 'author': author_filter, + 'keywords': keywords_filter, + 'abstract': abstract_filter, + 'tags': shadow_tags_filter, + 'array_match_mode': 'contains', + } + used_document_access_index = False data_q = f'SELECT * FROM c WHERE {where}' - matching_docs = list(cosmos_public_documents_container.query_items( - query=data_q, parameters=params, enable_cross_partition_query=True - )) - current_docs = sort_documents( - select_current_documents(matching_docs), - sort_by=sort_by, - sort_order=sort_order, - ) - total_count = len(current_docs) - docs = current_docs[offset:offset + page_size] + try: + index_read_result = query_document_access_index_documents( + source_scope=DOCUMENT_ACCESS_SCOPE_PUBLIC, + public_workspace_id=active_ws, + filters=shadow_filters, + ) + if index_read_result.get('success'): + used_document_access_index = True + current_docs = sort_documents( + index_read_result.get('documents', []), + sort_by=sort_by, + sort_order=sort_order, + ) + if is_document_access_shadow_validation_enabled(): + try: + matching_docs, source_query_metrics = query_items_with_cosmos_diagnostics( + cosmos_public_documents_container, + diagnostics_label='source_documents', + query=data_q, + parameters=params, + enable_cross_partition_query=True, + ) + source_current_docs = sort_documents( + select_current_documents(matching_docs), + sort_by=sort_by, + sort_order=sort_order, + ) + validate_document_access_index_shadow( + source_current_docs, + source_scope=DOCUMENT_ACCESS_SCOPE_PUBLIC, + user_id=user_id, + public_workspace_id=active_ws, + filters=shadow_filters, + source_query_metrics=source_query_metrics, + context='api_list_public_documents', + ) + except Exception as shadow_error: + log_event( + '[DocumentAccessIndex] Shadow validation source query failed after DAI read succeeded.', + extra={'source_scope': DOCUMENT_ACCESS_SCOPE_PUBLIC, 'error': str(shadow_error)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + else: + matching_docs, source_query_metrics = query_items_with_cosmos_diagnostics( + cosmos_public_documents_container, + diagnostics_label='source_documents', + collect_diagnostics=is_document_access_shadow_validation_enabled(), + query=data_q, + parameters=params, + enable_cross_partition_query=True, + ) + current_docs = sort_documents( + select_current_documents(matching_docs), + sort_by=sort_by, + sort_order=sort_order, + ) + validate_document_access_index_shadow( + current_docs, + source_scope=DOCUMENT_ACCESS_SCOPE_PUBLIC, + user_id=user_id, + public_workspace_id=active_ws, + filters=shadow_filters, + source_query_metrics=source_query_metrics, + context='api_list_public_documents', + ) + total_count = len(current_docs) + docs = current_docs[offset:offset + page_size] + except Exception as e: + log_event( + '[PublicDocuments] Error fetching public documents.', + extra={'public_workspace_id': active_ws, 'error': str(e)}, + level=logging.ERROR, + ) + return jsonify({'error': f'Error fetching documents: {str(e)}'}), 500 # legacy - legacy_q = 'SELECT VALUE COUNT(1) FROM c WHERE c.public_workspace_id = @ws AND NOT IS_DEFINED(c.percentage_complete)' - legacy = list(cosmos_public_documents_container.query_items( - query=legacy_q, - parameters=[{'name':'@ws','value':active_ws}], - enable_cross_partition_query=True - )) - legacy_count = legacy[0] if legacy else 0 + if used_document_access_index: + legacy_count_result = query_document_access_index_legacy_count( + source_scope=DOCUMENT_ACCESS_SCOPE_PUBLIC, + public_workspace_id=active_ws, + ) + legacy_count = legacy_count_result.get('legacy_count', 0) if legacy_count_result.get('success') else 0 + else: + legacy_q = 'SELECT VALUE COUNT(1) FROM c WHERE c.public_workspace_id = @ws AND NOT IS_DEFINED(c.percentage_complete)' + legacy = list(cosmos_public_documents_container.query_items( + query=legacy_q, + parameters=[{'name':'@ws','value':active_ws}], + enable_cross_partition_query=True + )) + legacy_count = legacy[0] if legacy else 0 file_downloads_enabled = is_public_workspace_file_download_enabled(get_settings(), ws_doc) return jsonify({ @@ -320,14 +417,64 @@ def api_list_public_workspace_documents(): workspace_conditions = " OR ".join([f"c.public_workspace_id = @ws_{i}" for i in range(len(workspace_ids))]) query = f'SELECT * FROM c WHERE {workspace_conditions} ORDER BY c._ts DESC' params = [{'name': f'@ws_{i}', 'value': workspace_id} for i, workspace_id in enumerate(workspace_ids)] - - docs = list(cosmos_public_documents_container.query_items( - query=query, - parameters=params, - enable_cross_partition_query=True - )) + try: + index_read_result = query_document_access_index_documents( + source_scope=DOCUMENT_ACCESS_SCOPE_PUBLIC, + public_workspace_ids=workspace_ids, + ) + if index_read_result.get('success'): + docs = sort_documents(index_read_result.get('documents', [])) + if is_document_access_shadow_validation_enabled(): + try: + source_docs, source_query_metrics = query_items_with_cosmos_diagnostics( + cosmos_public_documents_container, + diagnostics_label='source_documents', + query=query, + parameters=params, + enable_cross_partition_query=True, + ) + validate_document_access_index_shadow( + sort_documents(select_current_documents(source_docs)), + source_scope=DOCUMENT_ACCESS_SCOPE_PUBLIC, + user_id=user_id, + public_workspace_ids=workspace_ids, + source_query_metrics=source_query_metrics, + context='api_list_public_workspace_documents', + ) + except Exception as shadow_error: + log_event( + '[DocumentAccessIndex] Shadow validation source query failed after DAI read succeeded.', + extra={'source_scope': DOCUMENT_ACCESS_SCOPE_PUBLIC, 'error': str(shadow_error)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + else: + source_docs, source_query_metrics = query_items_with_cosmos_diagnostics( + cosmos_public_documents_container, + diagnostics_label='source_documents', + collect_diagnostics=is_document_access_shadow_validation_enabled(), + query=query, + parameters=params, + enable_cross_partition_query=True, + ) + docs = sort_documents(select_current_documents(source_docs)) + validate_document_access_index_shadow( + docs, + source_scope=DOCUMENT_ACCESS_SCOPE_PUBLIC, + user_id=user_id, + public_workspace_ids=workspace_ids, + source_query_metrics=source_query_metrics, + context='api_list_public_workspace_documents', + ) + except Exception as e: + log_event( + '[PublicDocuments] Error fetching public workspace chat documents.', + extra={'workspace_count': len(workspace_ids), 'error': str(e)}, + level=logging.ERROR, + ) + return jsonify({'error': f'Error fetching documents: {str(e)}'}), 500 - docs = sort_documents(select_current_documents(docs))[:page_size] + docs = docs[:page_size] return jsonify({ 'documents': docs, @@ -1005,11 +1152,24 @@ def api_get_public_workspace_document_tags(): visible_ids = set(get_user_visible_public_workspace_ids_from_settings(user_id)) validated_ids = [wid for wid in workspace_ids if wid in visible_ids] - from functions_documents import get_workspace_tags + from functions_documents import build_workspace_tags_from_counts, get_workspace_tags + + index_tag_result = query_document_access_index_tag_counts( + DOCUMENT_ACCESS_SCOPE_PUBLIC, + public_workspace_ids=validated_ids, + ) if validated_ids else {'success': False} all_tags = {} for wid in validated_ids: - tags = get_workspace_tags(user_id, public_workspace_id=wid) + if index_tag_result.get('success'): + scope_key = build_document_access_scope_key(DOCUMENT_ACCESS_SCOPE_PUBLIC, wid) + tags = build_workspace_tags_from_counts( + index_tag_result.get('tag_counts_by_scope_key', {}).get(scope_key, {}), + user_id, + public_workspace_id=wid, + ) + else: + tags = get_workspace_tags(user_id, public_workspace_id=wid) for tag in tags: if tag['name'] in all_tags: all_tags[tag['name']]['count'] += tag['count'] diff --git a/application/single_app/route_backend_public_workspaces.py b/application/single_app/route_backend_public_workspaces.py index 8aba7bdf..c74e8b03 100644 --- a/application/single_app/route_backend_public_workspaces.py +++ b/application/single_app/route_backend_public_workspaces.py @@ -2,6 +2,7 @@ from config import * from functions_authentication import * +from functions_chat_bootstrap_cache import bump_chat_bootstrap_global_cache_version from functions_prompts import count_public_prompts_for_workspace from functions_public_workspaces import * from functions_settings import ( @@ -330,6 +331,7 @@ def api_update_public_workspace(ws_id): try: cosmos_public_workspaces_container.upsert_item(ws) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_updated") return jsonify({"message": "Updated"}), 200 except exceptions.CosmosHttpResponseError as ex: return jsonify({"error": str(ex)}), 400 @@ -558,6 +560,8 @@ def api_handle_public_request(ws_id, req_id): ws["pendingDocumentManagers"] = pend ws["modifiedDate"] = datetime.utcnow().isoformat() cosmos_public_workspaces_container.upsert_item(ws) + if action == "approve": + bump_chat_bootstrap_global_cache_version(reason="public_workspace_member_request_approved") return jsonify({"message": msg}), 200 @bp.route("/api/public_workspaces//members", methods=["GET"]) @@ -679,6 +683,7 @@ def api_add_public_member(ws_id): }) ws["modifiedDate"] = datetime.utcnow().isoformat() cosmos_public_workspaces_container.upsert_item(ws) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_member_added") # Send notification to the added member try: @@ -740,6 +745,7 @@ def api_remove_public_member(ws_id, member_id): ] ws["modifiedDate"] = datetime.utcnow().isoformat() cosmos_public_workspaces_container.upsert_item(ws) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_member_removed") return jsonify({"success": True, "message": "Removed"}), 200 @bp.route("/api/public_workspaces//members/", methods=["PATCH"]) @@ -820,6 +826,7 @@ def api_update_public_member_role(ws_id, member_id): ws["modifiedDate"] = datetime.utcnow().isoformat() cosmos_public_workspaces_container.upsert_item(ws) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_member_role_updated") # Send notification to the member whose role changed try: @@ -916,6 +923,7 @@ def api_transfer_public_ownership(ws_id): ws["modifiedDate"] = datetime.utcnow().isoformat() cosmos_public_workspaces_container.upsert_item(ws) + bump_chat_bootstrap_global_cache_version(reason="public_workspace_ownership_transferred") return jsonify({"message": "Ownership transferred"}), 200 @bp.route("/api/public_workspaces//fileCount", methods=["GET"]) @@ -1259,4 +1267,3 @@ def api_public_workspace_activity(ws_id): traceback.print_exc() return jsonify(activities), 200 - diff --git a/application/single_app/route_backend_settings.py b/application/single_app/route_backend_settings.py index 9f1ccaa3..5d20fa3a 100644 --- a/application/single_app/route_backend_settings.py +++ b/application/single_app/route_backend_settings.py @@ -28,6 +28,12 @@ normalize_cosmos_throughput_settings, set_database_throughput, ) +from functions_app_maintenance import get_app_maintenance_status, run_app_maintenance_once +from functions_redis_monitoring import ( + get_redis_explorer_keys, + get_redis_explorer_value, + get_redis_monitoring_status, +) from azure.identity import DefaultAzureCredential from azure.keyvault.secrets import SecretClient from swagger_wrapper import swagger_route, get_auth_security @@ -195,6 +201,83 @@ def auto_fix_index_fields(idx_type: str, user_id: str = 'system', admin_email: s def register_route_backend_settings(bp): + @bp.route('/api/admin/settings/app-maintenance/status', methods=['GET']) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def get_app_maintenance_admin_status(): + """Return the current app maintenance status for admin diagnostics.""" + try: + return jsonify(get_app_maintenance_status(settings=get_settings())), 200 + except Exception as exc: + log_event( + '[AppMaintenance] Failed to load admin maintenance status.', + extra={'error': str(exc)}, + level=logging.ERROR, + exceptionTraceback=True, + ) + return jsonify({'error': 'Failed to load app maintenance status.'}), 500 + + @bp.route('/api/admin/settings/app-maintenance/run', methods=['POST']) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def run_app_maintenance_admin(): + """Run app maintenance tasks immediately from the admin settings area.""" + user = session.get('user', {}) + admin_email = user.get('preferred_username', user.get('email', 'unknown')) + admin_user_id = get_current_user_id() or 'unknown' + try: + payload = request.get_json(silent=True) or {} + apply_indexing_policies = None + if 'apply_cosmos_indexing_policies' in payload: + apply_indexing_policies = bool(payload.get('apply_cosmos_indexing_policies')) + apply_stale_cache_cleanup = None + if 'apply_stale_cache_cleanup' in payload: + apply_stale_cache_cleanup = bool(payload.get('apply_stale_cache_cleanup')) + result = run_app_maintenance_once( + triggered_by='admin_manual', + requested_by=admin_email, + settings=get_settings(), + apply_indexing_policies=apply_indexing_policies, + run_document_access_backfill=payload.get('run_document_access_index_backfill'), + reset_document_access_backfill=bool(payload.get('reset_document_access_index_backfill', False)), + run_stale_cache_cleanup=payload.get('run_stale_cache_cleanup'), + apply_stale_cache_cleanup=apply_stale_cache_cleanup, + ) + except Exception as exc: + log_event( + '[AppMaintenance] Manual admin maintenance trigger failed.', + extra={'admin_email': admin_email, 'error': str(exc)}, + level=logging.ERROR, + exceptionTraceback=True, + ) + return jsonify({'error': 'Failed to run app maintenance.'}), 500 + + if result.get('success'): + log_general_admin_action( + admin_user_id=admin_user_id, + admin_email=admin_email, + action='app_maintenance_manual_run', + description='Manually ran app maintenance foundation tasks.', + additional_context={ + 'run_id': result.get('run_id'), + 'status': result.get('state', {}).get('last_status'), + }, + ) + return jsonify(result), 200 + + log_event( + '[AppMaintenance] Manual admin maintenance run failed.', + extra={ + 'run_id': result.get('run_id'), + 'admin_email': admin_email, + 'error': result.get('error'), + }, + level=logging.ERROR, + ) + return jsonify(result), 500 + @bp.route('/api/admin/settings/check_index_fields', methods=['POST']) @swagger_route(security=get_auth_security()) @login_required @@ -521,6 +604,160 @@ def test_connection(): except Exception as e: return jsonify({'error': str(e)}), 500 + @bp.route('/api/admin/settings/redis-monitoring/status', methods=['GET']) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def get_redis_monitoring_admin_status(): + """Return sanitized Redis health and capacity metrics for the admin Scale tab.""" + refresh_id = str(uuid.uuid4()) + refresh_start = time.perf_counter() + try: + user = session.get('user', {}) + admin_email = user.get('preferred_username', user.get('email', 'unknown')) + log_event( + '[RedisMonitoring] Admin status refresh requested.', + extra={'refresh_id': refresh_id, 'admin_email': admin_email}, + level=logging.INFO, + ) + status = get_redis_monitoring_status( + get_settings(), + session_redis_client=current_app.config.get('SESSION_REDIS'), + session_type=current_app.config.get('SESSION_TYPE'), + ) + log_event( + '[RedisMonitoring] Admin status refresh completed.', + extra={ + 'refresh_id': refresh_id, + 'status': status.get('health', {}).get('status'), + 'monitoring_source': status.get('runtime', {}).get('monitoring_source'), + 'elapsed_ms': int((time.perf_counter() - refresh_start) * 1000), + }, + level=logging.INFO, + ) + return jsonify(status), 200 + except Exception as e: + log_event( + '[RedisMonitoring] Failed to load admin status.', + extra={ + 'refresh_id': refresh_id, + 'error': str(e), + 'elapsed_ms': int((time.perf_counter() - refresh_start) * 1000), + }, + level=logging.ERROR, + exceptionTraceback=True, + ) + return jsonify({'error': 'Failed to load Redis monitoring status.'}), 500 + + @bp.route('/api/admin/settings/redis-explorer/keys', methods=['GET']) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def get_redis_explorer_admin_keys(): + """Return a cursor-paginated Redis key page for the admin Redis Explorer.""" + refresh_id = str(uuid.uuid4()) + refresh_start = time.perf_counter() + try: + user = session.get('user', {}) + admin_email = user.get('preferred_username', user.get('email', 'unknown')) + log_event( + '[RedisExplorer] Admin key page requested.', + extra={'refresh_id': refresh_id, 'admin_email': admin_email}, + level=logging.INFO, + ) + result = get_redis_explorer_keys( + get_settings(), + cursor=request.args.get('cursor', 0), + page_size=request.args.get('page_size', 25), + key_filter=request.args.get('filter', ''), + session_redis_client=current_app.config.get('SESSION_REDIS'), + session_type=current_app.config.get('SESSION_TYPE'), + ) + log_event( + '[RedisExplorer] Admin key page completed.', + extra={ + 'refresh_id': refresh_id, + 'success': result.get('success'), + 'status': result.get('status'), + 'key_count': len(result.get('keys') or []), + 'has_more': result.get('has_more'), + 'elapsed_ms': int((time.perf_counter() - refresh_start) * 1000), + }, + level=logging.INFO, + ) + return jsonify(result), 200 if result.get('success') else 503 + except Exception as e: + log_event( + '[RedisExplorer] Failed to load key page.', + extra={ + 'refresh_id': refresh_id, + 'error': str(e), + 'elapsed_ms': int((time.perf_counter() - refresh_start) * 1000), + }, + level=logging.ERROR, + exceptionTraceback=True, + ) + return jsonify({'error': 'Failed to load Redis Explorer keys.'}), 500 + + @bp.route('/api/admin/settings/redis-explorer/value', methods=['POST']) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def get_redis_explorer_admin_value(): + """Return sanitized Redis key metadata and preview content for admins.""" + refresh_id = str(uuid.uuid4()) + refresh_start = time.perf_counter() + try: + payload = request.get_json(silent=True) or {} + user = session.get('user', {}) + admin_email = user.get('preferred_username', user.get('email', 'unknown')) + log_event( + '[RedisExplorer] Admin key preview requested.', + extra={'refresh_id': refresh_id, 'admin_email': admin_email}, + level=logging.INFO, + ) + result = get_redis_explorer_value( + get_settings(), + key=payload.get('key'), + session_redis_client=current_app.config.get('SESSION_REDIS'), + session_type=current_app.config.get('SESSION_TYPE'), + ) + status_code = 200 + if not result.get('success'): + status_code = 404 if result.get('status') == 'not_found' else 503 + log_event( + '[RedisExplorer] Admin key preview completed.', + extra={ + 'refresh_id': refresh_id, + 'success': result.get('success'), + 'status': result.get('status'), + 'type': result.get('type'), + 'preview_restricted': result.get('preview_restricted'), + 'elapsed_ms': int((time.perf_counter() - refresh_start) * 1000), + }, + level=logging.INFO, + ) + return jsonify(result), status_code + except ValueError as e: + log_event( + '[RedisExplorer] Invalid key preview request.', + extra={'refresh_id': refresh_id, 'error': str(e)}, + level=logging.WARNING, + ) + return jsonify({'error': str(e)}), 400 + except Exception as e: + log_event( + '[RedisExplorer] Failed to load key preview.', + extra={ + 'refresh_id': refresh_id, + 'error': str(e), + 'elapsed_ms': int((time.perf_counter() - refresh_start) * 1000), + }, + level=logging.ERROR, + exceptionTraceback=True, + ) + return jsonify({'error': 'Failed to load Redis Explorer key preview.'}), 500 + @bp.route('/api/admin/settings/cosmos-throughput/status', methods=['GET']) @swagger_route(security=get_auth_security()) @login_required @@ -538,7 +775,6 @@ def get_cosmos_throughput_admin_status(): level=logging.INFO, ) status = get_cosmos_throughput_status(get_settings(), include_metrics=True, refresh_id=refresh_id) - update_settings(build_runtime_update(status=status)) log_event( '[CosmosThroughput] Admin status refresh completed.', extra={ diff --git a/application/single_app/route_custom_pages.py b/application/single_app/route_custom_pages.py index c68fef49..f20aad92 100644 --- a/application/single_app/route_custom_pages.py +++ b/application/single_app/route_custom_pages.py @@ -203,7 +203,7 @@ def admin_create_custom_page(): if errors: return jsonify({"error": "; ".join(errors)}), 400 slug = str(payload.get("slug") or payload.get("id") or "").strip().lower() - if get_custom_page(slug, include_python=True): + if get_custom_page(slug, include_python=True, use_cache=False): return jsonify({"error": "A custom page with this slug already exists."}), 409 saved = save_custom_page(payload, user_id=_current_admin_user_id()) return jsonify(saved), 201 diff --git a/application/single_app/route_external_public_documents.py b/application/single_app/route_external_public_documents.py index e525099a..66596d7a 100644 --- a/application/single_app/route_external_public_documents.py +++ b/application/single_app/route_external_public_documents.py @@ -7,6 +7,13 @@ from functions_settings import * from functions_public_workspaces import * from functions_documents import * +from functions_document_access_index import ( + DOCUMENT_ACCESS_SCOPE_PUBLIC, + is_document_access_shadow_validation_enabled, + query_items_with_cosmos_diagnostics, + query_document_access_index_documents, + query_document_access_index_legacy_count, +) from functions_appinsights import log_event from functions_activity_logging import log_document_metadata_update_transaction from swagger_wrapper import swagger_route, get_auth_security @@ -236,6 +243,16 @@ def external_get_public_documents(): param_count += 1 where_clause = " AND ".join(query_conditions) + shadow_filters = { + 'search': search_term, + 'classification': classification_filter, + 'classification_none_matches_literal': False, + 'author': author_filter, + 'keywords': keywords_filter, + 'abstract': abstract_filter, + 'array_match_mode': 'exact', + } + used_document_access_index = False # --- 3) Query matching documents, then collapse to current revisions before paginating --- try: @@ -245,12 +262,59 @@ def external_get_public_documents(): FROM c WHERE {where_clause} """ - matching_docs = list(cosmos_public_documents_container.query_items( - query=data_query_str, - parameters=query_params, - enable_cross_partition_query=True - )) - current_docs = sort_documents(select_current_documents(matching_docs)) + index_read_result = query_document_access_index_documents( + source_scope=DOCUMENT_ACCESS_SCOPE_PUBLIC, + public_workspace_id=active_workspace_id, + filters=shadow_filters, + ) + if index_read_result.get('success'): + used_document_access_index = True + current_docs = sort_documents(index_read_result.get('documents', [])) + if is_document_access_shadow_validation_enabled(): + try: + matching_docs, source_query_metrics = query_items_with_cosmos_diagnostics( + cosmos_public_documents_container, + diagnostics_label='source_documents', + query=data_query_str, + parameters=query_params, + enable_cross_partition_query=True + ) + source_current_docs = sort_documents(select_current_documents(matching_docs)) + validate_document_access_index_shadow( + source_current_docs, + source_scope=DOCUMENT_ACCESS_SCOPE_PUBLIC, + user_id=user_id, + public_workspace_id=active_workspace_id, + filters=shadow_filters, + source_query_metrics=source_query_metrics, + context='external_get_public_documents', + ) + except Exception as shadow_error: + log_event( + '[DocumentAccessIndex] Shadow validation source query failed after DAI read succeeded.', + extra={'source_scope': DOCUMENT_ACCESS_SCOPE_PUBLIC, 'error': str(shadow_error)}, + level=logging.WARNING, + exceptionTraceback=True, + ) + else: + matching_docs, source_query_metrics = query_items_with_cosmos_diagnostics( + cosmos_public_documents_container, + diagnostics_label='source_documents', + collect_diagnostics=is_document_access_shadow_validation_enabled(), + query=data_query_str, + parameters=query_params, + enable_cross_partition_query=True + ) + current_docs = sort_documents(select_current_documents(matching_docs)) + validate_document_access_index_shadow( + current_docs, + source_scope=DOCUMENT_ACCESS_SCOPE_PUBLIC, + user_id=user_id, + public_workspace_id=active_workspace_id, + filters=shadow_filters, + source_query_metrics=source_query_metrics, + context='external_get_public_documents', + ) total_count = len(current_docs) docs = current_docs[offset:offset + page_size] except Exception as e: @@ -263,27 +327,34 @@ def external_get_public_documents(): # --- new: do we have any legacy documents? --- - try: - legacy_q = """ - SELECT VALUE COUNT(1) - FROM c - WHERE c.public_workspace_id = @public_workspace_id - AND NOT IS_DEFINED(c.percentage_complete) - """ - legacy_docs = list( - cosmos_public_documents_container.query_items( - query=legacy_q, - parameters=[{"name":"@public_workspace_id","value":active_workspace_id}], - enable_cross_partition_query=True - ) - ) - legacy_count = legacy_docs[0] if legacy_docs else 0 - except Exception as e: - log_event( - '[ExternalPublicDocuments] Error executing public legacy document query.', - extra={'public_workspace_id': active_workspace_id, 'error': str(e)}, - level=logging.ERROR, + if used_document_access_index: + legacy_count_result = query_document_access_index_legacy_count( + source_scope=DOCUMENT_ACCESS_SCOPE_PUBLIC, + public_workspace_id=active_workspace_id, ) + legacy_count = legacy_count_result.get('legacy_count', 0) if legacy_count_result.get('success') else 0 + else: + try: + legacy_q = """ + SELECT VALUE COUNT(1) + FROM c + WHERE c.public_workspace_id = @public_workspace_id + AND NOT IS_DEFINED(c.percentage_complete) + """ + legacy_docs = list( + cosmos_public_documents_container.query_items( + query=legacy_q, + parameters=[{"name":"@public_workspace_id","value":active_workspace_id}], + enable_cross_partition_query=True + ) + ) + legacy_count = legacy_docs[0] if legacy_docs else 0 + except Exception as e: + log_event( + '[ExternalPublicDocuments] Error executing public legacy document query.', + extra={'public_workspace_id': active_workspace_id, 'error': str(e)}, + level=logging.ERROR, + ) # --- 5) Return results --- return jsonify({ diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index d324a552..751b0c1f 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -1671,6 +1671,63 @@ def is_valid_url(url): cosmos_throughput_settings = normalize_cosmos_throughput_settings(cosmos_throughput_candidate_settings) + document_access_index_backfill_batch_size = min( + 1000, + max( + 1, + parse_admin_int( + form_data.get('document_access_index_backfill_batch_size'), + settings.get('document_access_index_backfill_batch_size', 200), + 'document_access_index_backfill_batch_size', + 200, + ), + ), + ) + document_access_index_repair_batch_size = min( + 500, + max( + 1, + parse_admin_int( + form_data.get('document_access_index_repair_batch_size'), + settings.get('document_access_index_repair_batch_size', 100), + 'document_access_index_repair_batch_size', + 100, + ), + ), + ) + document_access_index_cache_ttl_seconds = min( + 900, + max( + 60, + parse_admin_int( + form_data.get('document_access_index_cache_ttl_seconds'), + settings.get('document_access_index_cache_ttl_seconds', 900), + 'document_access_index_cache_ttl_seconds', + 900, + ), + ), + ) + conversation_cache_ttl_seconds = max( + 0, + parse_admin_int( + form_data.get('conversation_cache_ttl_seconds'), + settings.get('conversation_cache_ttl_seconds', 120), + 'conversation_cache_ttl_seconds', + 120, + ), + ) + dai_debug_enabled = bool(settings.get('enable_dai_debug', False)) + document_access_index_shadow_validation_enabled = ( + form_data.get('enable_document_access_index_shadow_validation') == 'on' + if dai_debug_enabled + else bool(settings.get('enable_document_access_index_shadow_validation', False)) + ) + document_access_index_cache_enabled = ( + form_data.get('enable_document_access_index_cache') == 'on' + if dai_debug_enabled + else bool(settings.get('enable_document_access_index_cache', True)) + ) + # --- Chunk Size Overrides --- chunk_size_defaults = get_chunk_size_defaults() existing_chunk_sizes = settings.get('chunk_size', {}) if isinstance(settings, dict) else {} @@ -1829,6 +1886,24 @@ def is_valid_url(url): 'redis_url': form_data.get('redis_url', '').strip(), 'redis_key': admin_secret('redis_key'), 'redis_auth_type': form_data.get('redis_auth_type', '').strip(), + 'enable_conversation_cache': form_data.get('enable_conversation_cache') == 'on', + 'conversation_cache_ttl_seconds': conversation_cache_ttl_seconds, + + # Document Access Index + 'enable_document_access_index_container': True, + 'enable_document_access_index_write_through': True, + 'enable_document_access_index_reads': True, + 'enable_dai_debug': dai_debug_enabled, + 'enable_document_access_index_shadow_validation': document_access_index_shadow_validation_enabled, + 'enable_startup_document_access_index_backfill': True, + 'document_access_index_backfill_batch_size': document_access_index_backfill_batch_size, + 'document_access_index_repair_batch_size': document_access_index_repair_batch_size, + 'document_access_index_active_maintenance_interval_seconds': settings.get( + 'document_access_index_active_maintenance_interval_seconds', + 30, + ), + 'enable_document_access_index_cache': document_access_index_cache_enabled, + 'document_access_index_cache_ttl_seconds': document_access_index_cache_ttl_seconds, # Workspaces 'enable_user_workspace': form_data.get('enable_user_workspace') == 'on', diff --git a/application/single_app/route_frontend_chats.py b/application/single_app/route_frontend_chats.py index 55401dfb..5af7d93a 100644 --- a/application/single_app/route_frontend_chats.py +++ b/application/single_app/route_frontend_chats.py @@ -35,6 +35,11 @@ from functions_public_workspaces import find_public_workspace_by_id, get_user_visible_public_workspace_ids_from_settings from functions_simplechat_operations import upload_chat_image_bytes_for_user from functions_appinsights import log_event +from functions_chat_bootstrap_cache import ( + build_chat_bootstrap_cache_key, + get_cached_chat_bootstrap_payload, + set_cached_chat_bootstrap_payload, +) from swagger_wrapper import swagger_route, get_auth_security from functions_debug import debug_print from utils_cache import invalidate_group_search_cache, invalidate_personal_search_cache @@ -647,6 +652,16 @@ def _build_chat_prompt_catalog(*, user_id, settings, user_groups_raw, user_visib return catalog + +def _is_valid_chat_bootstrap_payload(payload): + return ( + isinstance(payload, dict) + and isinstance(payload.get('chat_agent_options'), list) + and isinstance(payload.get('chat_model_options'), list) + and isinstance(payload.get('chat_prompt_options'), list) + ) + + def register_route_frontend_chats(bp): @bp.route('/chats', methods=['GET']) @swagger_route(security=get_auth_security()) @@ -754,33 +769,77 @@ def chats(): logger.warning(f"Failed to load visible public workspaces for chats page: {e}") chat_agent_options = [] - try: - all_chat_agent_options = build_accessible_agent_catalog( - user_id, - settings=settings, - user_groups=user_groups_raw, - ) - chat_agent_options = [ - agent for agent in all_chat_agent_options - if _is_chat_agent_allowed_by_governance( - user_id, - agent, - str(agent.get('scope_type') or '').strip().lower() or 'personal', - ) - ] - except Exception as e: - logger.warning(f"Failed to load chat agent options: {e}") - chat_model_options = [] + chat_prompt_options = [] + bootstrap_cache_key = None + cached_bootstrap_payload = None try: - chat_model_options = _build_chat_model_catalog( - user_id=user_id, + bootstrap_cache_key = build_chat_bootstrap_cache_key( + user_id, settings=settings, user_settings_dict=user_settings_dict, user_groups_raw=user_groups_raw, + user_visible_public_workspaces=user_visible_public_workspaces, ) + cached_bootstrap_payload = get_cached_chat_bootstrap_payload(bootstrap_cache_key) except Exception as e: - logger.warning(f"Failed to load chat model options: {e}") + logger.warning(f"Failed to load chat bootstrap cache: {e}") + + if _is_valid_chat_bootstrap_payload(cached_bootstrap_payload): + chat_agent_options = cached_bootstrap_payload.get('chat_agent_options') or [] + chat_model_options = cached_bootstrap_payload.get('chat_model_options') or [] + chat_prompt_options = cached_bootstrap_payload.get('chat_prompt_options') or [] + else: + try: + all_chat_agent_options = build_accessible_agent_catalog( + user_id, + settings=settings, + user_groups=user_groups_raw, + ) + chat_agent_options = [ + agent for agent in all_chat_agent_options + if _is_chat_agent_allowed_by_governance( + user_id, + agent, + str(agent.get('scope_type') or '').strip().lower() or 'personal', + ) + ] + except Exception as e: + logger.warning(f"Failed to load chat agent options: {e}") + + try: + chat_model_options = _build_chat_model_catalog( + user_id=user_id, + settings=settings, + user_settings_dict=user_settings_dict, + user_groups_raw=user_groups_raw, + ) + except Exception as e: + logger.warning(f"Failed to load chat model options: {e}") + + try: + chat_prompt_options = _build_chat_prompt_catalog( + user_id=user_id, + settings=settings, + user_groups_raw=user_groups_raw, + user_visible_public_workspaces=user_visible_public_workspaces, + ) + except Exception as e: + logger.warning(f"Failed to load chat prompt options: {e}") + + if bootstrap_cache_key: + try: + set_cached_chat_bootstrap_payload( + bootstrap_cache_key, + { + 'chat_agent_options': chat_agent_options, + 'chat_model_options': chat_model_options, + 'chat_prompt_options': chat_prompt_options, + }, + ttl_seconds=int(settings.get('chat_bootstrap_cache_ttl_seconds') or 300), + ) + except Exception as e: + logger.warning(f"Failed to store chat bootstrap cache: {e}") initial_chat_model_selection = _build_initial_chat_model_selection( chat_model_options=chat_model_options, @@ -788,17 +847,6 @@ def chats(): preferred_model_deployment=user_settings_dict.get('preferredModelDeployment'), ) - chat_prompt_options = [] - try: - chat_prompt_options = _build_chat_prompt_catalog( - user_id=user_id, - settings=settings, - user_groups_raw=user_groups_raw, - user_visible_public_workspaces=user_visible_public_workspaces, - ) - except Exception as e: - logger.warning(f"Failed to load chat prompt options: {e}") - return render_template( 'chats.html', settings=public_settings, diff --git a/application/single_app/semantic_kernel_plugins/cosmos_query_plugin.py b/application/single_app/semantic_kernel_plugins/cosmos_query_plugin.py index d760e28a..d28166b6 100644 --- a/application/single_app/semantic_kernel_plugins/cosmos_query_plugin.py +++ b/application/single_app/semantic_kernel_plugins/cosmos_query_plugin.py @@ -9,7 +9,7 @@ import re from typing import Any, Dict, List, Optional, Sequence, Union -from azure.cosmos import CosmosClient +import azure.cosmos as azure_cosmos from azure.cosmos.exceptions import CosmosHttpResponseError from azure.identity import DefaultAzureCredential from semantic_kernel.functions import kernel_function @@ -34,7 +34,7 @@ def __repr__(self) -> str: class CosmosQueryPlugin(BasePlugin): """Read-only Azure Cosmos DB for NoSQL query plugin.""" - _client_cache: Dict[str, CosmosClient] = {} + _client_cache: Dict[str, azure_cosmos.CosmosClient] = {} def __init__(self, manifest: Dict[str, Any]): super().__init__(manifest) @@ -310,7 +310,7 @@ def _get_container_client(self): client_cache_key = self._get_client_cache_key() client = self._client_cache.get(client_cache_key) if client is None: - client = CosmosClient( + client = azure_cosmos.CosmosClient( self.endpoint, credential=self._get_client_credential(), timeout=self.timeout, diff --git a/application/single_app/static/js/admin/admin_settings.js b/application/single_app/static/js/admin/admin_settings.js index 1de01978..b5e996ac 100644 --- a/application/single_app/static/js/admin/admin_settings.js +++ b/application/single_app/static/js/admin/admin_settings.js @@ -37,6 +37,16 @@ let currentCosmosContainers = []; let currentCosmosMetricsWindowMinutes = 0; let currentCosmosStatusLoaded = false; let currentCosmosContainerSort = { field: 'container_name', direction: 'asc' }; +let documentAccessIndexStatusPollId = null; +let redisExplorerState = { + cursor: '0', + nextCursor: '0', + cursorHistory: [], + filter: '', + pageSize: 25, + loaded: false, + selectedKey: '' +}; const COSMOS_CONTAINER_SORT_FIELDS = new Set([ 'container_name', @@ -48,6 +58,7 @@ const COSMOS_CONTAINER_SORT_FIELDS = new Set([ const COSMOS_CONTAINER_TEXT_SORT_FIELDS = new Set(['container_name', 'policy']); const COSMOS_THROUGHPUT_SIMPLECHAT_MAX_RU = 10000; const COSMOS_THROUGHPUT_PORTAL_MANAGED_MESSAGE = 'Throughput above 10,000 RU/s is monitored only in SimpleChat. Use the Azure portal to change capacity; capacity changes above this level can take 4 to 6 hours.'; +const DOCUMENT_ACCESS_INDEX_STATUS_POLL_INTERVAL_MS = 7500; const GROUP_WORKFLOW_ASSIGNMENT_PARSE_DEPTH_LIMIT = 5; const enableClassificationToggle = document.getElementById('enable_document_classification'); @@ -495,6 +506,106 @@ function formatNumber(value) { return numericValue.toLocaleString(); } +function formatRedisStatus(value) { + const normalizedValue = String(value || '').trim(); + if (!normalizedValue) { + return 'Not loaded'; + } + + return normalizedValue + .replace(/_/g, ' ') + .replace(/\b\w/g, character => character.toUpperCase()); +} + +function getRedisMonitoringStatusVariant(value) { + const normalizedValue = String(value || '').trim().toLowerCase(); + if (normalizedValue === 'healthy' || normalizedValue === 'active') { + return 'success'; + } + if (['degraded', 'not_configured', 'unavailable'].includes(normalizedValue)) { + return 'warning'; + } + if (normalizedValue === 'error') { + return 'danger'; + } + return 'secondary'; +} + +function setRedisMonitoringMessage(message, variant = 'info') { + const messageElement = document.getElementById('redis-monitoring-message'); + if (!messageElement) { + return; + } + + messageElement.textContent = message || ''; + messageElement.className = `alert alert-${variant} small mb-3`; + messageElement.classList.toggle('d-none', !message); +} + +function setRedisMonitoringBadge(elementId, value, variant = 'secondary') { + const badge = document.getElementById(elementId); + if (!badge) { + return; + } + + const safeVariants = new Set(['primary', 'secondary', 'success', 'danger', 'warning', 'info']); + badge.textContent = value || 'Not loaded'; + badge.className = `badge text-bg-${safeVariants.has(variant) ? variant : 'secondary'}`; +} + +function formatRedisMetric(value, unit) { + if (value === null || value === undefined || value === '') { + return 'Not available'; + } + const numericValue = Number(value); + if (Number.isNaN(numericValue)) { + return 'Not available'; + } + return `${numericValue.toLocaleString(undefined, { maximumFractionDigits: 2 })} ${unit}`; +} + +function formatRedisPercent(value) { + if (value === null || value === undefined || value === '') { + return 'Not available'; + } + const numericValue = Number(value); + if (Number.isNaN(numericValue)) { + return 'Not available'; + } + return `${numericValue.toLocaleString(undefined, { maximumFractionDigits: 2 })}%`; +} + +function formatRedisMemoryUsage(memory) { + const usedMemory = memory?.used_memory_human || formatRedisMetric(memory?.used_memory, 'bytes'); + const maxMemory = memory?.maxmemory_human || formatRedisMetric(memory?.maxmemory, 'bytes'); + const usagePercent = formatRedisPercent(memory?.usage_percent); + + if (usedMemory === 'Not available') { + return 'Not available'; + } + if (maxMemory === 'Not available' || Number(memory?.maxmemory || 0) === 0) { + return `${usedMemory} / no maxmemory limit`; + } + if (usagePercent === 'Not available') { + return `${usedMemory} / ${maxMemory}`; + } + return `${usedMemory} / ${maxMemory} (${usagePercent})`; +} + +function formatRedisRuntime(value) { + return value ? 'Active' : 'Inactive'; +} + +function setRedisTestResult(resultDiv, message, variant = 'muted') { + if (!resultDiv) { + return; + } + + const safeVariants = new Set(['success', 'danger', 'muted', 'info']); + resultDiv.textContent = message || ''; + resultDiv.className = `mt-2 text-${safeVariants.has(variant) ? variant : 'muted'}`; +} + function formatRu(value) { const formattedValue = formatNumber(value); return formattedValue === 'Not available' ? formattedValue : `${formattedValue} RU/s`; @@ -1341,100 +1452,1269 @@ function updateCosmosThroughputStatusPanel(status) { renderCosmosContainerMetrics(currentCosmosContainers); renderCosmosContainerPolicyModal(currentCosmosContainers); - const databasePortalManaged = isCosmosThroughputPortalManaged(throughput); - const globalScaleButtonsDisabled = status?.capacity_scope === 'container' || !throughput.is_scalable || databasePortalManaged; - const globalScaleUpButtonDisabled = globalScaleButtonsDisabled || isCosmosScaleUpBlockedBySimpleChatLimit(throughput); - const globalConvertButtonDisabled = status?.capacity_scope === 'container' || throughput.mode !== 'manual' || !throughput.is_scalable || databasePortalManaged; - document.getElementById('cosmos-throughput-convert-autoscale-btn')?.toggleAttribute('disabled', globalConvertButtonDisabled); - document.getElementById('cosmos-throughput-scale-up-btn')?.toggleAttribute('disabled', globalScaleUpButtonDisabled); - document.getElementById('cosmos-throughput-scale-down-btn')?.toggleAttribute('disabled', globalScaleButtonsDisabled); + const databasePortalManaged = isCosmosThroughputPortalManaged(throughput); + const globalScaleButtonsDisabled = status?.capacity_scope === 'container' || !throughput.is_scalable || databasePortalManaged; + const globalScaleUpButtonDisabled = globalScaleButtonsDisabled || isCosmosScaleUpBlockedBySimpleChatLimit(throughput); + const globalConvertButtonDisabled = status?.capacity_scope === 'container' || throughput.mode !== 'manual' || !throughput.is_scalable || databasePortalManaged; + document.getElementById('cosmos-throughput-convert-autoscale-btn')?.toggleAttribute('disabled', globalConvertButtonDisabled); + document.getElementById('cosmos-throughput-scale-up-btn')?.toggleAttribute('disabled', globalScaleUpButtonDisabled); + document.getElementById('cosmos-throughput-scale-down-btn')?.toggleAttribute('disabled', globalScaleButtonsDisabled); +} + +function hasContainerLevelCosmosMetrics(status) { + return (status?.containers || []).some(container => ( + container?.normalized_ru_percent !== null + && container?.normalized_ru_percent !== undefined + ) || ( + container?.request_units !== null + && container?.request_units !== undefined + )); +} + +function hasPortalManagedCosmosThroughput(status) { + return Boolean(status?.throughput?.portal_managed_scaling_required) || (status?.containers || []).some(container => ( + isCosmosThroughputPortalManaged(container) + )); +} + +function getCachedCosmosThroughputStatus() { + const cachedStatus = window.cosmosThroughputCachedStatus; + if (!cachedStatus || typeof cachedStatus !== 'object' || Array.isArray(cachedStatus)) { + return null; + } + + const hasStatusData = Boolean( + cachedStatus.last_checked_at + || cachedStatus.capacity_scope + || (Array.isArray(cachedStatus.containers) && cachedStatus.containers.length > 0) + ); + return hasStatusData ? cachedStatus : null; +} + +function initializeCosmosThroughputStatusView() { + const cachedStatus = getCachedCosmosThroughputStatus(); + const automationEnabled = isFieldChecked('cosmos_throughput_autoscale_enabled'); + + if (cachedStatus) { + updateCosmosThroughputStatusPanel(cachedStatus); + const refreshText = automationEnabled + ? 'Background automation refreshes this saved view on the Metrics Window cadence while enabled.' + : 'Automation is currently disabled; use Refresh to update this saved view.'; + setCosmosThroughputMessage(`Showing last saved Cosmos throughput status. ${refreshText}`, 'info'); + return; + } + + if (automationEnabled) { + setCosmosThroughputMessage('No saved Cosmos throughput status is available yet. Loading the first status check now...', 'info'); + loadCosmosThroughputStatus(); + } +} + +async function loadCosmosThroughputStatus(event = null) { + const triggerButton = event?.currentTarget || document.getElementById('cosmos-throughput-refresh-btn'); + if (triggerButton) { + setButtonBusy(triggerButton, true, 'Loading...'); + } + setCosmosThroughputMessage('Loading Cosmos throughput status...', 'info'); + + try { + const response = await fetch('/api/admin/settings/cosmos-throughput/status', { + method: 'GET', + headers: { 'Accept': 'application/json' }, + credentials: 'same-origin' + }); + const data = await response.json(); + if (!response.ok) { + throw new Error(data.error || 'Failed to load Cosmos throughput status.'); + } + + updateCosmosThroughputStatusPanel(data); + if (!data.configured) { + setCosmosThroughputMessage(data.error || 'Cosmos throughput management needs subscription, resource group, account, and database settings.', 'warning'); + } else if (data.throughput_error) { + setCosmosThroughputMessage(`Cosmos database throughput could not be read. ${data.throughput_error}`, 'danger'); + } else if (data.capacity_scope === 'container' && data.metrics?.normalized_ru_percent !== null && data.metrics?.normalized_ru_percent !== undefined && !hasContainerLevelCosmosMetrics(data)) { + setCosmosThroughputMessage('Azure Monitor returned aggregate RU utilization, but not per-container metric dimensions for this window. Container autoscale waits for per-container utilization before scaling individual containers; Refresh again after a few minutes.', 'warning'); + } else if (hasPortalManagedCosmosThroughput(data)) { + setCosmosThroughputMessage('One or more Cosmos throughput targets are above 10,000 RU/s. SimpleChat will monitor utilization and request units only; use the Azure portal for capacity changes, which can take 4 to 6 hours.', 'warning'); + } else if (data.capacity_scope === 'container') { + setCosmosThroughputMessage('Container-targeted throughput is active. Dedicated-throughput containers can be monitored and scaled individually; containers sharing database throughput remain view-only.', 'info'); + } else if (data.metric_error) { + setCosmosThroughputMessage('Throughput loaded, but Azure Monitor metrics are unavailable. Check the app identity permissions and Cosmos metrics availability.', 'warning'); + } else { + setCosmosThroughputMessage('Cosmos throughput status loaded.', 'success'); + } + } catch (error) { + setCosmosThroughputMessage(error.message || 'Failed to load Cosmos throughput status.', 'danger'); + } finally { + if (triggerButton) { + setButtonBusy(triggerButton, false); + } + } +} + +function buildCosmosThroughputAccessPayload() { + return { + cosmos_throughput_autoscale_enabled: isFieldChecked('cosmos_throughput_autoscale_enabled'), + cosmos_throughput_auto_scale_up_enabled: isFieldChecked('cosmos_throughput_auto_scale_up_enabled'), + cosmos_throughput_auto_scale_down_enabled: isFieldChecked('cosmos_throughput_auto_scale_down_enabled'), + cosmos_throughput_subscription_id: getFieldValue('cosmos_throughput_subscription_id'), + cosmos_throughput_resource_group: getFieldValue('cosmos_throughput_resource_group'), + cosmos_throughput_account_name: getFieldValue('cosmos_throughput_account_name'), + cosmos_throughput_database_name: getFieldValue('cosmos_throughput_database_name'), + cosmos_throughput_metrics_window_minutes: getNumericFieldValue('cosmos_throughput_metrics_window_minutes', 5), + cosmos_throughput_scale_up_threshold_percent: getNumericFieldValue('cosmos_throughput_scale_up_threshold_percent', 90), + cosmos_throughput_scale_down_threshold_percent: getNumericFieldValue('cosmos_throughput_scale_down_threshold_percent', 70), + cosmos_throughput_scale_up_step_ru: getNumericFieldValue('cosmos_throughput_scale_up_step_ru', 1000), + cosmos_throughput_scale_down_step_ru: getNumericFieldValue('cosmos_throughput_scale_down_step_ru', 1000), + cosmos_throughput_scale_up_cooldown_minutes: getNumericFieldValue('cosmos_throughput_scale_up_cooldown_minutes', 5), + cosmos_throughput_scale_down_cooldown_minutes: getNumericFieldValue('cosmos_throughput_scale_down_cooldown_minutes', 20), + cosmos_throughput_min_ru: getNumericFieldValue('cosmos_throughput_min_ru', 1000), + cosmos_throughput_max_ru: getNumericFieldValue('cosmos_throughput_max_ru', COSMOS_THROUGHPUT_SIMPLECHAT_MAX_RU), + cosmos_throughput_ignore_min_limit: isFieldChecked('cosmos_throughput_ignore_min_limit'), + cosmos_throughput_ignore_max_limit: isFieldChecked('cosmos_throughput_ignore_max_limit'), + cosmos_throughput_convert_manual_to_autoscale_enabled: isFieldChecked('cosmos_throughput_convert_manual_to_autoscale_enabled'), + cosmos_throughput_enforce_container_defaults: isFieldChecked('cosmos_throughput_enforce_container_defaults'), + cosmos_throughput_container_policies: collectCosmosContainerPolicies() + }; +} + +async function validateCosmosThroughputAccess(triggerButton = null) { + const button = triggerButton || document.getElementById('cosmos-throughput-validate-access-btn'); + if (button) { + setButtonBusy(button, true, 'Validating...'); + } + setCosmosThroughputMessage('Validating Cosmos throughput configuration and access...', 'info'); + + try { + const response = await fetch('/api/admin/settings/cosmos-throughput/validate-access', { + method: 'POST', + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json' + }, + credentials: 'same-origin', + body: JSON.stringify(buildCosmosThroughputAccessPayload()) + }); + const data = await response.json(); + if (!response.ok) { + throw new Error(data.error || 'Failed to validate Cosmos throughput access.'); + } + + if (data.status?.configured) { + updateCosmosThroughputStatusPanel(data.status); + } + setCosmosThroughputValidationResult(data); + } catch (error) { + setCosmosThroughputMessage(error.message || 'Failed to validate Cosmos throughput access.', 'danger'); + } finally { + if (button) { + setButtonBusy(button, false); + } + } +} + +async function manuallyScaleCosmosThroughput(direction, containerName = '', triggerButton = null) { + const button = triggerButton || document.getElementById(`cosmos-throughput-scale-${direction}-btn`); + if (button) { + setButtonBusy(button, true, direction === 'up' ? 'Scaling up...' : 'Scaling down...'); + } + const targetText = containerName ? ` for ${containerName}` : ''; + setCosmosThroughputMessage(direction === 'up' ? `Submitting scale-up request${targetText}...` : `Submitting scale-down request${targetText}...`, 'info'); + + try { + const response = await fetch('/api/admin/settings/cosmos-throughput/scale', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'same-origin', + body: JSON.stringify({ direction, container_name: containerName }) + }); + const data = await response.json(); + if (!response.ok) { + throw new Error(data.error || 'Cosmos throughput scale request failed.'); + } + + const scaledTarget = data.container_name ? ` for ${data.container_name}` : ''; + setCosmosThroughputMessage(`Cosmos throughput${scaledTarget} changed from ${formatRu(data.from_ru)} to ${formatRu(data.to_ru)}.`, 'success'); + await loadCosmosThroughputStatus(); + } catch (error) { + setCosmosThroughputMessage(error.message || 'Cosmos throughput scale request failed.', 'danger'); + } finally { + if (button) { + setButtonBusy(button, false, direction === 'up' ? 'Scale Up' : 'Scale Down'); + } + } +} + +async function convertCosmosThroughputToAutoscale(containerName = '', triggerButton = null) { + const targetText = containerName ? ` for ${containerName}` : ''; + if (triggerButton) { + setButtonBusy(triggerButton, true, 'Converting...'); + } + setCosmosThroughputMessage(`Converting manual Cosmos throughput${targetText} to native autoscale...`, 'info'); + + try { + const response = await fetch('/api/admin/settings/cosmos-throughput/convert-autoscale', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'same-origin', + body: JSON.stringify({ container_name: containerName }) + }); + const data = await response.json(); + if (!response.ok) { + throw new Error(data.error || 'Cosmos throughput mode conversion failed.'); + } + + const convertedTarget = data.container_name ? ` for ${data.container_name}` : ''; + setCosmosThroughputMessage(`Cosmos throughput${convertedTarget} converted from manual ${formatRu(data.from_ru)} to autoscale max ${formatRu(data.to_ru)}.`, 'success'); + await loadCosmosThroughputStatus(); + } catch (error) { + setCosmosThroughputMessage(error.message || 'Cosmos throughput mode conversion failed.', 'danger'); + } finally { + if (triggerButton) { + setButtonBusy(triggerButton, false); + } + } +} + +function renderRedisMonitoringStatus(statusPayload) { + const configuration = statusPayload?.configuration || {}; + const runtime = statusPayload?.runtime || {}; + const health = statusPayload?.health || {}; + const memory = statusPayload?.memory || {}; + const clients = statusPayload?.clients || {}; + const stats = statusPayload?.stats || {}; + const keyspace = statusPayload?.keyspace || {}; + const server = statusPayload?.server || {}; + const daiCache = statusPayload?.dai_cache || {}; + + let configurationText = 'Disabled'; + let configurationVariant = 'secondary'; + if (configuration.enabled && configuration.configured) { + configurationText = 'Enabled'; + configurationVariant = 'success'; + } else if (configuration.enabled) { + configurationText = 'Needs host'; + configurationVariant = 'warning'; + } + + setRedisMonitoringBadge('redis-monitoring-config-status', configurationText, configurationVariant); + setRedisMonitoringBadge( + 'redis-monitoring-health-status', + formatRedisStatus(health.status), + getRedisMonitoringStatusVariant(health.status) + ); + setRedisMonitoringBadge( + 'redis-monitoring-app-cache-status', + formatRedisRuntime(runtime.app_cache_using_redis), + runtime.app_cache_using_redis ? 'success' : 'secondary' + ); + setRedisMonitoringBadge( + 'redis-monitoring-session-status', + formatRedisRuntime(runtime.session_using_redis), + runtime.session_using_redis ? 'success' : 'secondary' + ); + + setElementText('redis-monitoring-ping-latency', formatRedisMetric(health.ping_latency_ms, 'ms')); + setElementText('redis-monitoring-memory-usage', formatRedisMemoryUsage(memory)); + setElementText( + 'redis-monitoring-memory-policy', + memory.maxmemory_policy ? `Policy: ${memory.maxmemory_policy}` : 'Policy: Not available' + ); + setElementText('redis-monitoring-connected-clients', formatNumber(clients.connected_clients)); + setElementText('redis-monitoring-ops-per-sec', formatNumber(stats.instantaneous_ops_per_sec)); + setElementText('redis-monitoring-hit-rate', formatRedisPercent(stats.keyspace_hit_rate_percent)); + setElementText('redis-monitoring-key-count', formatNumber(keyspace.total_keys)); + setElementText('redis-monitoring-dai-version-markers', formatNumber(daiCache.version_marker_count)); + setElementText( + 'redis-monitoring-dai-version-marker-expiry', + `${formatNumber(daiCache.version_marker_no_expiry_count)} no-expiry marker(s)` + ); + setElementText('redis-monitoring-dai-payload-keys', formatNumber(daiCache.payload_key_count)); + setElementText( + 'redis-monitoring-dai-version-ttl-policy', + `Version TTL: ${formatRedisExplorerTtl(daiCache.version_marker_ttl_seconds)}` + ); + setElementText('redis-monitoring-expired-keys', formatNumber(stats.expired_keys)); + setElementText('redis-monitoring-evicted-keys', formatNumber(stats.evicted_keys)); + setElementText('redis-monitoring-fragmentation', formatNumber(memory.mem_fragmentation_ratio)); + setElementText('redis-monitoring-errors', formatNumber(stats.total_error_replies)); + setElementText('redis-monitoring-rejected-connections', formatNumber(stats.rejected_connections)); + setElementText('redis-monitoring-version', server.redis_version || 'Not available'); + setElementText('redis-monitoring-source', formatRedisStatus(runtime.monitoring_source)); + setElementText('redis-monitoring-checked-at', statusPayload?.checked_at || 'Not available'); + setElementText('redis-monitoring-last-error', health.last_error || 'None'); +} + +async function loadRedisMonitoringStatus(event = null, options = {}) { + const showLoading = options.showLoading !== false; + const triggerButton = event?.currentTarget || (showLoading ? document.getElementById('redis-monitoring-refresh-btn') : null); + if (triggerButton) { + setButtonBusy(triggerButton, true, 'Loading...'); + } + if (showLoading) { + setRedisMonitoringMessage('Loading Redis monitoring status...', 'info'); + } + + try { + const response = await fetch('/api/admin/settings/redis-monitoring/status', { + method: 'GET', + headers: { 'Accept': 'application/json' }, + credentials: 'same-origin' + }); + const data = await response.json(); + if (!response.ok) { + throw new Error(data.error || 'Failed to load Redis monitoring status.'); + } + + renderRedisMonitoringStatus(data); + if (data.health?.status === 'healthy') { + setRedisMonitoringMessage('Redis monitoring status loaded.', 'success'); + } else if (data.health?.last_error) { + setRedisMonitoringMessage(data.health.last_error, data.health.status === 'error' ? 'danger' : 'warning'); + } else if (showLoading) { + setRedisMonitoringMessage('Redis monitoring status loaded.', 'info'); + } + } catch (error) { + setRedisMonitoringMessage(error.message || 'Failed to load Redis monitoring status.', 'danger'); + } finally { + if (triggerButton) { + setButtonBusy(triggerButton, false); + } + } +} + +function setRedisExplorerMessage(message, variant = 'info') { + const messageElement = document.getElementById('redis-explorer-message'); + if (!messageElement) { + return; + } + + messageElement.textContent = message || ''; + messageElement.className = `alert alert-${variant} small`; + messageElement.classList.toggle('d-none', !message); +} + +function formatRedisExplorerTtl(ttlSeconds) { + const numericValue = Number(ttlSeconds); + if (Number.isNaN(numericValue)) { + return 'Not available'; + } + if (numericValue === -2) { + return 'Expired or missing'; + } + if (numericValue === -1) { + return 'No expiry'; + } + return `${numericValue.toLocaleString()} sec`; +} + +function formatRedisExplorerMemory(bytes) { + const numericValue = Number(bytes); + if (Number.isNaN(numericValue) || numericValue < 0) { + return 'Not available'; + } + if (numericValue >= 1024 * 1024) { + return `${(numericValue / (1024 * 1024)).toLocaleString(undefined, { maximumFractionDigits: 2 })} MB`; + } + if (numericValue >= 1024) { + return `${(numericValue / 1024).toLocaleString(undefined, { maximumFractionDigits: 2 })} KB`; + } + return `${numericValue.toLocaleString()} bytes`; +} + + +function formatRedisExplorerResolutionLabel(resolution) { + if (!resolution) { + return ''; + } + if (resolution.resolved && resolution.entity_type) { + const entityType = formatRedisStatus(resolution.entity_type); + const entityName = resolution.entity_name || resolution.entity_id || 'Unknown'; + return `${resolution.label || 'SimpleChat entity'}: ${entityType} - ${entityName}`; + } + return resolution.label || ''; +} + + +function formatRedisExplorerResolutionEntity(resolution) { + if (!resolution) { + return 'Not resolved'; + } + if (!resolution.resolved) { + return formatRedisStatus(resolution.resolution_status || 'unresolved'); + } + const parts = []; + if (resolution.entity_type) { + parts.push(`Entity: ${formatRedisStatus(resolution.entity_type)}`); + } + if (resolution.entity_name) { + parts.push(`Name: ${resolution.entity_name}`); + } + if (resolution.entity_status) { + parts.push(`Status: ${formatRedisStatus(resolution.entity_status)}`); + } + if (Number.isFinite(Number(resolution.row_count))) { + parts.push(`DAI rows: ${formatNumber(resolution.row_count)}`); + } + return parts.join(' | ') || 'Resolved'; +} + + +function renderRedisExplorerResolution(resolution) { + const resolutionCard = document.getElementById('redis-explorer-resolution-card'); + if (!resolutionCard) { + return; + } + + resolutionCard.classList.toggle('d-none', !resolution); + if (!resolution) { + return; + } + + setElementText('redis-explorer-resolution-kind', resolution.label || formatRedisStatus(resolution.kind)); + setElementText('redis-explorer-resolution-entity', formatRedisExplorerResolutionEntity(resolution)); + setElementText('redis-explorer-resolution-scope', resolution.scope_key || resolution.cache_hash || resolution.scope_hash || 'No reversible scope key available'); + setElementText('redis-explorer-resolution-note', resolution.note || 'Resolved from Redis key classification.'); +} + + +function setRedisExplorerPreviewVisible(isVisible) { + document.getElementById('redis-explorer-preview-empty')?.classList.toggle('d-none', isVisible); + document.getElementById('redis-explorer-preview-panel')?.classList.toggle('d-none', !isVisible); +} + +function resetRedisExplorerPreview() { + redisExplorerState.selectedKey = ''; + setRedisExplorerPreviewVisible(false); + setElementText('redis-explorer-preview-key', 'Not loaded'); + setElementText('redis-explorer-preview-type', 'Not loaded'); + setElementText('redis-explorer-preview-ttl', 'Not loaded'); + setElementText('redis-explorer-preview-memory', 'Not loaded'); + setElementText('redis-explorer-preview-redacted', 'Not loaded'); + setElementText('redis-explorer-preview-content', 'Not loaded'); + renderRedisExplorerResolution(null); +} + +function getRedisExplorerScopeLabel() { + const filter = (redisExplorerState.filter || '').trim(); + return filter ? `Filter: "${filter}"` : 'Browsing all keys'; +} + +function renderRedisExplorerKeys(payload) { + const keyList = document.getElementById('redis-explorer-key-list'); + if (!keyList) { + return; + } + + const keys = Array.isArray(payload?.keys) ? payload.keys : []; + keyList.replaceChildren(); + const pageNumber = redisExplorerState.cursorHistory.length + 1; + setElementText( + 'redis-explorer-key-count', + `${getRedisExplorerScopeLabel()} | Page ${formatNumber(pageNumber)} | ${formatNumber(keys.length)} key(s)${payload?.has_more ? ' / more available' : ''}` + ); + + if (keys.length === 0) { + const emptyState = document.createElement('div'); + emptyState.className = 'list-group-item text-muted small'; + emptyState.textContent = 'No Redis keys matched this page and filter.'; + keyList.appendChild(emptyState); + } else { + keys.forEach(item => { + const keyButton = document.createElement('button'); + keyButton.type = 'button'; + keyButton.className = 'list-group-item list-group-item-action'; + keyButton.setAttribute('role', 'option'); + + const keyName = document.createElement('div'); + keyName.className = 'fw-semibold text-break'; + keyName.textContent = item.key || '(empty key)'; + + const metadata = document.createElement('div'); + metadata.className = 'small text-muted'; + metadata.textContent = [ + `Type: ${formatRedisStatus(item.type || 'unknown')}`, + `TTL: ${formatRedisExplorerTtl(item.ttl_seconds)}`, + item.preview_restricted ? 'Preview: restricted' : 'Preview: sanitized' + ].join(' | '); + + keyButton.appendChild(keyName); + keyButton.appendChild(metadata); + const resolutionLabel = formatRedisExplorerResolutionLabel(item.resolution); + if (resolutionLabel) { + const resolutionMetadata = document.createElement('div'); + resolutionMetadata.className = 'small text-primary'; + resolutionMetadata.textContent = resolutionLabel; + keyButton.appendChild(resolutionMetadata); + } + keyButton.addEventListener('click', () => loadRedisExplorerValue(item.key, keyButton)); + keyList.appendChild(keyButton); + }); + } + keyList.scrollTop = 0; + + redisExplorerState.nextCursor = String(payload?.next_cursor || '0'); + document.getElementById('redis-explorer-prev-btn')?.toggleAttribute( + 'disabled', + redisExplorerState.cursorHistory.length === 0 + ); + document.getElementById('redis-explorer-next-btn')?.toggleAttribute( + 'disabled', + !payload?.has_more + ); +} + +function getRedisExplorerRequestState(options = {}) { + const reset = Boolean(options.reset); + const filterInput = document.getElementById('redis-explorer-filter'); + const pageSizeSelect = document.getElementById('redis-explorer-page-size'); + if (reset) { + redisExplorerState.cursor = '0'; + redisExplorerState.nextCursor = '0'; + redisExplorerState.cursorHistory = []; + } + redisExplorerState.filter = filterInput?.value || ''; + redisExplorerState.pageSize = Number(pageSizeSelect?.value || 25); + return { + cursor: redisExplorerState.cursor, + filter: redisExplorerState.filter, + pageSize: redisExplorerState.pageSize + }; +} + +async function loadRedisExplorerKeys(options = {}) { + const triggerButton = options.triggerButton || null; + if (triggerButton) { + setButtonBusy(triggerButton, true, 'Loading...'); + } + setRedisExplorerMessage('Loading Redis keys...', 'info'); + resetRedisExplorerPreview(); + + try { + const requestState = getRedisExplorerRequestState(options); + const query = new URLSearchParams({ + cursor: requestState.cursor, + page_size: String(requestState.pageSize), + filter: requestState.filter + }); + const response = await fetch(`/api/admin/settings/redis-explorer/keys?${query.toString()}`, { + method: 'GET', + headers: { 'Accept': 'application/json' }, + credentials: 'same-origin' + }); + const data = await response.json(); + if (!response.ok || !data.success) { + throw new Error(data.error || data.last_error || 'Failed to load Redis keys.'); + } + + renderRedisExplorerKeys(data); + redisExplorerState.loaded = true; + setRedisExplorerMessage(`${getRedisExplorerScopeLabel()} loaded. Select a key to view sanitized preview content.`, 'success'); + } catch (error) { + renderRedisExplorerKeys({ keys: [], has_more: false, next_cursor: '0' }); + setRedisExplorerMessage(error.message || 'Failed to load Redis keys.', 'danger'); + } finally { + if (triggerButton) { + setButtonBusy(triggerButton, false); + } + } +} + +async function loadRedisExplorerValue(key, triggerButton = null) { + if (!key) { + return; + } + if (triggerButton) { + setButtonBusy(triggerButton, true, 'Loading...'); + } + setRedisExplorerMessage('Loading sanitized Redis key preview...', 'info'); + + try { + const response = await fetch('/api/admin/settings/redis-explorer/value', { + method: 'POST', + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json' + }, + credentials: 'same-origin', + body: JSON.stringify({ key }) + }); + const data = await response.json(); + if (!response.ok || !data.success) { + throw new Error(data.error || data.preview || 'Failed to load Redis key preview.'); + } + + redisExplorerState.selectedKey = key; + setRedisExplorerPreviewVisible(true); + setElementText('redis-explorer-preview-key', data.key || key); + setElementText('redis-explorer-preview-type', formatRedisStatus(data.type || 'unknown')); + setElementText('redis-explorer-preview-ttl', formatRedisExplorerTtl(data.ttl_seconds)); + setElementText('redis-explorer-preview-memory', formatRedisExplorerMemory(data.memory_usage_bytes)); + setElementText( + 'redis-explorer-preview-redacted', + data.preview_restricted ? 'Restricted' : data.redacted ? 'Redacted' : 'Sanitized' + ); + renderRedisExplorerResolution(data.resolution); + setElementText('redis-explorer-preview-content', data.preview || 'No preview available.'); + const previewContent = document.getElementById('redis-explorer-preview-content'); + if (previewContent) { + previewContent.scrollTop = 0; + } + setRedisExplorerMessage( + data.truncated ? 'Preview loaded and truncated to the safe display limit.' : 'Sanitized preview loaded.', + data.preview_restricted || data.redacted || data.truncated ? 'warning' : 'success' + ); + } catch (error) { + setRedisExplorerMessage(error.message || 'Failed to load Redis key preview.', 'danger'); + } finally { + if (triggerButton) { + setButtonBusy(triggerButton, false); + } + } +} + +function setupRedisExplorerControls() { + const modalElement = document.getElementById('redisExplorerModal'); + if (!modalElement) { + return; + } + + modalElement.addEventListener('shown.bs.modal', () => { + if (!redisExplorerState.loaded) { + loadRedisExplorerKeys({ reset: true }); + } + }); + document.getElementById('redis-explorer-search-btn')?.addEventListener('click', event => { + loadRedisExplorerKeys({ reset: true, triggerButton: event.currentTarget }); + }); + document.getElementById('redis-explorer-browse-all-btn')?.addEventListener('click', event => { + const filterInput = document.getElementById('redis-explorer-filter'); + if (filterInput) { + filterInput.value = ''; + } + loadRedisExplorerKeys({ reset: true, triggerButton: event.currentTarget }); + }); + document.getElementById('redis-explorer-refresh-btn')?.addEventListener('click', event => { + loadRedisExplorerKeys({ triggerButton: event.currentTarget }); + }); + document.getElementById('redis-explorer-filter')?.addEventListener('keydown', event => { + if (event.key === 'Enter') { + event.preventDefault(); + loadRedisExplorerKeys({ reset: true }); + } + }); + document.getElementById('redis-explorer-page-size')?.addEventListener('change', () => { + loadRedisExplorerKeys({ reset: true }); + }); + document.getElementById('redis-explorer-next-btn')?.addEventListener('click', event => { + redisExplorerState.cursorHistory.push(redisExplorerState.cursor); + redisExplorerState.cursor = redisExplorerState.nextCursor || '0'; + loadRedisExplorerKeys({ triggerButton: event.currentTarget }); + }); + document.getElementById('redis-explorer-prev-btn')?.addEventListener('click', event => { + redisExplorerState.cursor = redisExplorerState.cursorHistory.pop() || '0'; + loadRedisExplorerKeys({ triggerButton: event.currentTarget }); + }); +} + +function setupRedisMonitoringControls() { + const section = document.getElementById('redis-monitoring-section'); + if (!section) { + return; + } + + document.getElementById('redis-monitoring-refresh-btn')?.addEventListener('click', loadRedisMonitoringStatus); + setupRedisExplorerControls(); + loadRedisMonitoringStatus(null, { showLoading: false }); +} + +function setDocumentAccessIndexMessage(message, variant = 'info') { + const messageElement = document.getElementById('document-access-index-message'); + if (!messageElement) { + return; + } + + messageElement.textContent = message || ''; + messageElement.className = `alert alert-${variant} small mb-3`; + messageElement.classList.toggle('d-none', !message); +} + +function formatDocumentAccessIndexStatus(value) { + const normalizedValue = String(value || '').trim(); + if (!normalizedValue) { + return 'Not loaded'; + } + + return normalizedValue + .replace(/_/g, ' ') + .replace(/\b\w/g, character => character.toUpperCase()); +} + +function getDocumentAccessIndexStatusVariant(value) { + const normalizedValue = String(value || '').trim().toLowerCase(); + if (['succeeded', 'skipped_completed', 'completed', 'dry_run_completed', 'reconciled', 'matched', 'aligned'].includes(normalizedValue)) { + return 'success'; + } + if (['running', 'in_progress'].includes(normalizedValue)) { + return 'primary'; + } + if (['succeeded_with_errors', 'completed_with_errors', 'reconciled_with_errors', 'mismatch', 'missing_expected_indexes'].includes(normalizedValue)) { + return 'warning'; + } + if (['failed', 'error'].includes(normalizedValue)) { + return 'danger'; + } + return 'secondary'; +} + +function setDocumentAccessIndexBadge(elementId, value, variant = 'secondary') { + const badge = document.getElementById(elementId); + if (!badge) { + return; + } + + const safeVariants = new Set(['primary', 'secondary', 'success', 'danger', 'warning', 'info']); + badge.textContent = value || 'Not loaded'; + badge.className = `badge text-bg-${safeVariants.has(variant) ? variant : 'secondary'}`; +} + +function formatDocumentAccessIndexBoolean(value, enabledText = 'Enabled', disabledText = 'Disabled') { + return value ? enabledText : disabledText; +} + +function formatDocumentAccessIndexList(items) { + if (!Array.isArray(items) || items.length === 0) { + return 'None'; + } + + return items.map(item => String(item || '').trim()).filter(Boolean).join(', ') || 'None'; +} + +function formatDocumentAccessIndexMetric(value, unit) { + if (value === null || value === undefined || value === '') { + return 'Not available'; + } + const numericValue = Number(value); + if (Number.isNaN(numericValue)) { + return 'Not available'; + } + return `${numericValue.toLocaleString(undefined, { maximumFractionDigits: 3 })} ${unit}`; +} + +function formatDocumentAccessIndexPercent(value) { + if (value === null || value === undefined || value === '') { + return 'Not available'; + } + const numericValue = Number(value); + if (Number.isNaN(numericValue)) { + return 'Not available'; + } + return `${numericValue.toLocaleString(undefined, { maximumFractionDigits: 2 })}%`; +} + +function formatDocumentAccessIndexMetricPair(sourceValue, projectionValue, unit) { + const sourceMetric = formatDocumentAccessIndexMetric(sourceValue, unit); + const projectionMetric = formatDocumentAccessIndexMetric(projectionValue, unit); + if (sourceMetric === 'Not available' || projectionMetric === 'Not available') { + return 'Not available'; + } + return `${sourceMetric} / ${projectionMetric}`; +} + +function formatDocumentAccessIndexSavings(value, unit, positiveLabel, negativeLabel) { + if (value === null || value === undefined || value === '') { + return 'Not available'; + } + const numericValue = Number(value); + if (Number.isNaN(numericValue)) { + return 'Not available'; + } + if (Math.abs(numericValue) < 0.001) { + return `0 ${unit} difference`; + } + const label = numericValue > 0 ? positiveLabel : negativeLabel; + return `${formatDocumentAccessIndexMetric(Math.abs(numericValue), unit)} ${label}`; +} + +function getDocumentAccessIndexRollingWindow(shadowValidation, windowKey) { + const windows = shadowValidation?.rolling_metrics?.windows || {}; + return windows[windowKey] || {}; +} + +function getDocumentAccessIndexReadWindow(readMetrics, windowKey) { + const windows = readMetrics?.windows || {}; + return windows[windowKey] || {}; +} + +function getDocumentAccessIndexCacheWindow(cacheMetrics, windowKey) { + const windows = cacheMetrics?.windows || {}; + return windows[windowKey] || {}; +} + +function formatConversationCacheOperationCounts(operationCounts) { + const entries = Object.entries(operationCounts || {}) + .filter(([, count]) => Number(count || 0) > 0) + .sort(([left], [right]) => left.localeCompare(right)); + if (entries.length === 0) { + return 'No samples'; + } + + return entries + .map(([operation, count]) => `${formatDocumentAccessIndexStatus(operation)}: ${formatNumber(count)}`) + .join(', '); +} + +function formatDocumentAccessIndexSampleSummary(windowMetrics) { + if (!windowMetrics || Number(windowMetrics.sample_count || 0) === 0) { + return 'No samples'; + } + + const sampleCount = formatNumber(windowMetrics.sample_count || 0); + const matchedCount = formatNumber(windowMetrics.matched_count || 0); + const mismatchCount = formatNumber(windowMetrics.mismatch_count || 0); + const errorCount = formatNumber(windowMetrics.error_count || 0); + const comparableCount = formatNumber(windowMetrics.comparable_sample_count || 0); + return `${sampleCount} samples (${comparableCount} comparable, ${matchedCount} matched, ${mismatchCount} mismatch, ${errorCount} error)`; +} + +function formatDocumentAccessIndexReadSample(sample) { + if (!sample) { + return 'No samples'; + } + const status = formatDocumentAccessIndexStatus(sample.status || 'unknown'); + const operation = formatDocumentAccessIndexStatus(sample.operation || 'read'); + const scope = sample.source_scope ? ` / ${sample.source_scope}` : ''; + return `${status} (${operation}${scope})`; +} + +function formatDocumentAccessIndexCacheEvent(sample) { + if (!sample) { + return 'No cache events'; + } + const eventType = formatDocumentAccessIndexStatus(sample.event_type || 'unknown'); + const operation = formatDocumentAccessIndexStatus(sample.operation || 'cache'); + const reason = sample.reason ? ` / ${formatDocumentAccessIndexStatus(sample.reason)}` : ''; + return `${eventType} (${operation}${reason})`; +} + +function formatDocumentAccessIndexLatencyWindow(windowMetrics) { + const averageLatency = formatDocumentAccessIndexMetric(windowMetrics?.elapsed_ms_avg, 'ms'); + const p95Latency = formatDocumentAccessIndexMetric(windowMetrics?.elapsed_ms_p95, 'ms'); + if (averageLatency === 'Not available' && p95Latency === 'Not available') { + return 'Not available'; + } + return `${averageLatency} / ${p95Latency}`; +} + +function renderConversationCacheStatus(statusPayload) { + const conversationCache = statusPayload?.conversation_cache || {}; + const settings = conversationCache?.settings || {}; + const metrics = conversationCache?.metrics || {}; + const cache15m = getDocumentAccessIndexCacheWindow(metrics, '15m'); + const cacheEnabled = settings.enabled !== false; + const ttlSeconds = Number(settings.ttl_seconds ?? 120); + const runtimeStatus = cacheEnabled + ? (ttlSeconds > 0 ? `Enabled / ${formatDocumentAccessIndexMetric(ttlSeconds, 'sec')}` : 'Enabled / Writes disabled') + : 'Disabled'; + + setDocumentAccessIndexBadge( + 'conversation-cache-runtime-status', + runtimeStatus, + cacheEnabled ? 'success' : 'secondary' + ); + setElementText( + 'conversation-cache-15m-hit-rate', + formatDocumentAccessIndexPercent(cache15m.hit_rate_percent) + ); + setElementText( + 'conversation-cache-15m-hits-misses', + `${formatNumber(cache15m.hit_count || 0)} / ${formatNumber(cache15m.miss_count || 0)}` + ); + setElementText( + 'conversation-cache-15m-bypasses-errors', + `${formatNumber(cache15m.bypass_count || 0)} / ${formatNumber(cache15m.error_count || 0)}` + ); + setElementText( + 'conversation-cache-15m-writes-invalidations', + `${formatNumber(cache15m.write_count || 0)} / ${formatNumber(cache15m.invalidation_count || 0)}` + ); + setElementText( + 'conversation-cache-15m-operation-counts', + formatConversationCacheOperationCounts(cache15m.operation_counts) + ); + setElementText( + 'conversation-cache-last-event', + formatDocumentAccessIndexCacheEvent(metrics.last_event) + ); + setElementText( + 'conversation-cache-last-invalidation', + formatDocumentAccessIndexCacheEvent(metrics.last_invalidation) + ); +} + +function setCosmosMaintenanceMessage(message, variant = 'info') { + const messageElement = document.getElementById('cosmos-maintenance-message'); + if (!messageElement) { + return; + } + + messageElement.textContent = message || ''; + messageElement.className = `alert alert-${variant} small mb-3`; + messageElement.classList.toggle('d-none', !message); +} + +function getCosmosIndexingPolicyStatusText(indexingPolicy) { + if (!indexingPolicy || Object.keys(indexingPolicy).length === 0) { + return 'not_loaded'; + } + if (Number(indexingPolicy.failed_container_count || 0) > 0) { + return 'failed'; + } + if (Number(indexingPolicy.containers_missing_expected_indexes || 0) > 0) { + return 'missing_expected_indexes'; + } + return 'aligned'; +} + +function formatStaleCacheCleanupCategories(categories) { + if (!Array.isArray(categories) || categories.length === 0) { + return 'No categories'; + } + + return categories + .map(category => { + const name = formatDocumentAccessIndexStatus(category.category || 'unknown'); + const candidates = formatNumber(category.candidate_count || 0); + const deleted = formatNumber(category.deleted_count || 0); + const failed = formatNumber(category.failed_count || 0); + return `${name}: ${candidates} candidates / ${deleted} deleted / ${failed} failed`; + }) + .join('; '); +} + +function renderCosmosMaintenanceStatus(statusPayload) { + const indexingPolicy = statusPayload?.cosmos_indexing_policies || {}; + const cleanup = statusPayload?.stale_cache_cleanup || {}; + const indexingStatus = getCosmosIndexingPolicyStatusText(indexingPolicy); + const cleanupStatus = cleanup.status || 'not_run'; + + setDocumentAccessIndexBadge( + 'cosmos-indexing-policy-status', + formatDocumentAccessIndexStatus(indexingStatus), + getDocumentAccessIndexStatusVariant(indexingStatus) + ); + setElementText('cosmos-indexing-policy-mode', formatDocumentAccessIndexStatus(indexingPolicy.mode || 'not_loaded')); + setElementText('cosmos-indexing-policy-container-count', formatNumber(indexingPolicy.container_count || 0)); + setElementText( + 'cosmos-indexing-policy-missing-count', + formatNumber(indexingPolicy.containers_missing_expected_indexes || 0) + ); + setElementText('cosmos-indexing-policy-updated-count', formatNumber(indexingPolicy.updated_container_count || 0)); + setElementText('cosmos-indexing-policy-failed-count', formatNumber(indexingPolicy.failed_container_count || 0)); + setElementText('cosmos-indexing-policy-last-evaluated', indexingPolicy.evaluated_at || 'Not loaded'); + + setDocumentAccessIndexBadge( + 'stale-cache-cleanup-status', + formatDocumentAccessIndexStatus(cleanupStatus), + getDocumentAccessIndexStatusVariant(cleanupStatus) + ); + setElementText('stale-cache-cleanup-mode', formatDocumentAccessIndexStatus(cleanup.mode || 'not_run')); + setElementText('stale-cache-cleanup-candidates', formatNumber(cleanup.candidate_count || 0)); + setElementText('stale-cache-cleanup-deleted', formatNumber(cleanup.deleted_count || 0)); + setElementText('stale-cache-cleanup-failed', formatNumber(cleanup.failed_count || 0)); + setElementText('stale-cache-cleanup-more-candidates', cleanup.has_more_candidates ? 'Yes' : 'No'); + setElementText('stale-cache-cleanup-categories', formatStaleCacheCleanupCategories(cleanup.categories)); + setElementText('stale-cache-cleanup-last-evaluated', cleanup.evaluated_at || 'Not run yet'); +} + +function getStaleCacheCleanupStatusFromRunResult(result) { + const cleanupStep = Array.isArray(result?.steps) + ? result.steps.find(step => step?.name === 'stale_cache_document_cleanup') + : null; + return cleanupStep?.results || null; } -function hasContainerLevelCosmosMetrics(status) { - return (status?.containers || []).some(container => ( - container?.normalized_ru_percent !== null - && container?.normalized_ru_percent !== undefined - ) || ( - container?.request_units !== null - && container?.request_units !== undefined - )); +function getCosmosIndexingPolicyStatusFromRunResult(result) { + const indexingStep = Array.isArray(result?.steps) + ? result.steps.find(step => step?.name === 'cosmos_indexing_policy_maintenance') + : null; + return indexingStep?.results || null; } -function hasPortalManagedCosmosThroughput(status) { - return Boolean(status?.throughput?.portal_managed_scaling_required) || (status?.containers || []).some(container => ( - isCosmosThroughputPortalManaged(container) - )); +function getNormalizedDocumentAccessIndexStatus(status) { + return String(status || '').trim().toLowerCase(); } -function getCachedCosmosThroughputStatus() { - const cachedStatus = window.cosmosThroughputCachedStatus; - if (!cachedStatus || typeof cachedStatus !== 'object' || Array.isArray(cachedStatus)) { - return null; - } +function isDocumentAccessIndexBackfillRunning(status) { + return getNormalizedDocumentAccessIndexStatus(status) === 'running'; +} - const hasStatusData = Boolean( - cachedStatus.last_checked_at - || cachedStatus.capacity_scope - || (Array.isArray(cachedStatus.containers) && cachedStatus.containers.length > 0) - ); - return hasStatusData ? cachedStatus : null; +function isDocumentAccessIndexBackfillInProgress(status) { + return getNormalizedDocumentAccessIndexStatus(status) === 'in_progress'; } -function initializeCosmosThroughputStatusView() { - const cachedStatus = getCachedCosmosThroughputStatus(); - const automationEnabled = isFieldChecked('cosmos_throughput_autoscale_enabled'); +function stopDocumentAccessIndexPolling() { + if (documentAccessIndexStatusPollId) { + window.clearInterval(documentAccessIndexStatusPollId); + documentAccessIndexStatusPollId = null; + } +} - if (cachedStatus) { - updateCosmosThroughputStatusPanel(cachedStatus); - const refreshText = automationEnabled - ? 'Background automation refreshes this saved view on the Metrics Window cadence while enabled.' - : 'Automation is currently disabled; use Refresh to update this saved view.'; - setCosmosThroughputMessage(`Showing last saved Cosmos throughput status. ${refreshText}`, 'info'); +function startDocumentAccessIndexPolling() { + if (documentAccessIndexStatusPollId) { return; } - if (automationEnabled) { - setCosmosThroughputMessage('No saved Cosmos throughput status is available yet. Loading the first status check now...', 'info'); - loadCosmosThroughputStatus(); + documentAccessIndexStatusPollId = window.setInterval(() => { + loadDocumentAccessIndexStatus(null, { showLoading: false }); + }, DOCUMENT_ACCESS_INDEX_STATUS_POLL_INTERVAL_MS); +} + +function getDocumentAccessIndexBackfillStatusFromRunResult(result) { + const backfillStep = Array.isArray(result?.steps) + ? result.steps.find(step => step?.name === 'document_access_index_backfill') + : null; + return backfillStep?.results?.current_status || null; +} + +async function fetchAppMaintenanceStatus(errorMessage = 'Failed to load app maintenance status.') { + const response = await fetch('/api/admin/settings/app-maintenance/status', { + method: 'GET', + headers: { 'Accept': 'application/json' }, + credentials: 'same-origin' + }); + const data = await response.json(); + if (!response.ok) { + throw new Error(data.error || errorMessage); + } + return data; +} + +function renderDocumentAccessIndexStatus(statusPayload) { + const backfillStatus = statusPayload?.document_access_index_backfill || statusPayload; + const state = backfillStatus?.state || {}; + const settings = backfillStatus?.settings || {}; + const shadowValidation = backfillStatus?.shadow_validation || {}; + const maintenance = backfillStatus?.maintenance || {}; + const readMetrics = backfillStatus?.read_metrics || {}; + const cacheMetrics = backfillStatus?.cache_metrics || {}; + const statusText = String(state.status || 'not_started'); + + setDocumentAccessIndexBadge( + 'document-access-index-container-status', + formatDocumentAccessIndexBoolean(settings.container_enabled, 'Enabled', 'Disabled'), + settings.container_enabled ? 'success' : 'secondary' + ); + setDocumentAccessIndexBadge( + 'document-access-index-write-through-status', + formatDocumentAccessIndexBoolean(settings.write_through_enabled, 'Enabled', 'Disabled'), + settings.write_through_enabled ? 'success' : 'secondary' + ); + setDocumentAccessIndexBadge( + 'document-access-index-read-status', + settings.reads_enabled ? 'Enabled' : 'Disabled', + settings.reads_enabled ? 'success' : 'secondary' + ); + setDocumentAccessIndexBadge( + 'document-access-index-cache-status', + settings.cache_enabled ? `Enabled / ${formatDocumentAccessIndexMetric(settings.cache_ttl_seconds, 'sec')}` : 'Disabled', + settings.cache_enabled ? 'success' : 'secondary' + ); + setDocumentAccessIndexBadge( + 'document-access-index-shadow-status', + settings.shadow_validation_enabled ? 'Enabled' : 'Disabled', + settings.shadow_validation_enabled ? 'warning' : 'secondary' + ); + setDocumentAccessIndexBadge( + 'document-access-index-backfill-status', + formatDocumentAccessIndexStatus(statusText), + getDocumentAccessIndexStatusVariant(statusText) + ); + setDocumentAccessIndexBadge( + 'document-access-index-maintenance-mode', + maintenance.auto_maintenance_enabled ? 'Automatic' : 'Disabled', + maintenance.auto_maintenance_enabled ? 'success' : 'secondary' + ); + + setElementText('document-access-index-repair-count', formatNumber(backfillStatus?.repair_required_count)); + setElementText( + 'document-access-index-maintenance-next-action', + formatDocumentAccessIndexStatus(maintenance.next_action || 'monitor') + ); + setElementText( + 'document-access-index-maintenance-more-work', + maintenance.has_more_work ? 'Yes' : 'No' + ); + setElementText( + 'document-access-index-maintenance-active-interval', + formatDocumentAccessIndexMetric(maintenance.active_interval_seconds, 'sec') + ); + const read15m = getDocumentAccessIndexReadWindow(readMetrics, '15m'); + const cache15m = getDocumentAccessIndexCacheWindow(cacheMetrics, '15m'); + setElementText('document-access-index-read-15m-attempts', formatNumber(read15m.sample_count || 0)); + setElementText( + 'document-access-index-cache-15m-hit-rate', + formatDocumentAccessIndexPercent(cache15m.hit_rate_percent) + ); + setElementText( + 'document-access-index-cache-15m-hits-misses', + `${formatNumber(cache15m.hit_count || 0)} / ${formatNumber(cache15m.miss_count || 0)}` + ); + setElementText( + 'document-access-index-cache-15m-bypasses-errors', + `${formatNumber(cache15m.bypass_count || 0)} / ${formatNumber(cache15m.error_count || 0)}` + ); + setElementText( + 'document-access-index-cache-15m-invalidations', + formatNumber(cache15m.invalidation_count || 0) + ); + setElementText('document-access-index-read-15m-served', formatNumber(read15m.served_from_index_count || 0)); + setElementText('document-access-index-read-15m-fallbacks', formatNumber(read15m.source_fallback_count || 0)); + setElementText( + 'document-access-index-read-15m-fallback-rate', + formatDocumentAccessIndexPercent(read15m.fallback_rate_percent) + ); + setElementText( + 'document-access-index-read-15m-ru', + formatDocumentAccessIndexMetric(read15m.request_charge, 'RU') + ); + setElementText( + 'document-access-index-read-15m-latency', + formatDocumentAccessIndexLatencyWindow(read15m) + ); + setElementText( + 'document-access-index-read-last-fallback', + formatDocumentAccessIndexReadSample(readMetrics.last_fallback_sample) + ); + setElementText( + 'document-access-index-read-last-sample', + formatDocumentAccessIndexReadSample(readMetrics.last_sample) + ); + setElementText( + 'document-access-index-cache-last-event', + formatDocumentAccessIndexCacheEvent(cacheMetrics.last_event) + ); + setDocumentAccessIndexBadge( + 'document-access-index-shadow-last-status', + formatDocumentAccessIndexStatus(shadowValidation.status || 'not_run'), + getDocumentAccessIndexStatusVariant(shadowValidation.status || 'not_run') + ); + setElementText( + 'document-access-index-shadow-mismatches', + `${formatNumber(shadowValidation.missing_count || 0)} missing / ${formatNumber(shadowValidation.extra_count || 0)} extra` + ); + setElementText( + 'document-access-index-shadow-ru-comparison', + formatDocumentAccessIndexMetricPair(shadowValidation.source_query_ru, shadowValidation.validation_index_ru, 'RU') + ); + setElementText( + 'document-access-index-shadow-validation-ru', + formatDocumentAccessIndexMetric(shadowValidation.validation_index_ru, 'RU') + ); + setElementText( + 'document-access-index-shadow-candidate-ru', + formatDocumentAccessIndexMetric(shadowValidation.candidate_read_ru, 'RU') + ); + setElementText( + 'document-access-index-shadow-wave5-ru-savings', + formatDocumentAccessIndexSavings(shadowValidation.estimated_wave5_ru_savings, 'RU', 'less', 'more') + ); + setElementText( + 'document-access-index-shadow-ms-comparison', + formatDocumentAccessIndexMetricPair(shadowValidation.source_query_ms, shadowValidation.candidate_read_ms, 'ms') + ); + setElementText( + 'document-access-index-shadow-ms-savings', + formatDocumentAccessIndexSavings(shadowValidation.estimated_wave5_ms_savings, 'ms', 'faster', 'slower') + ); + const rolling5m = getDocumentAccessIndexRollingWindow(shadowValidation, '5m'); + const rolling15m = getDocumentAccessIndexRollingWindow(shadowValidation, '15m'); + setElementText( + 'document-access-index-rolling-5m-ru-comparison', + formatDocumentAccessIndexMetricPair(rolling5m.source_query_ru, rolling5m.candidate_read_ru, 'RU') + ); + setElementText( + 'document-access-index-rolling-5m-wave5-ru-savings', + formatDocumentAccessIndexSavings(rolling5m.estimated_wave5_ru_savings, 'RU', 'less', 'more') + ); + setElementText( + 'document-access-index-rolling-15m-ru-comparison', + formatDocumentAccessIndexMetricPair(rolling15m.source_query_ru, rolling15m.candidate_read_ru, 'RU') + ); + setElementText( + 'document-access-index-rolling-15m-wave5-ru-savings', + formatDocumentAccessIndexSavings(rolling15m.estimated_wave5_ru_savings, 'RU', 'less', 'more') + ); + setElementText( + 'document-access-index-rolling-15m-validation-overhead', + formatDocumentAccessIndexMetric(rolling15m.validation_index_ru, 'RU') + ); + setElementText( + 'document-access-index-rolling-15m-samples', + formatDocumentAccessIndexSampleSummary(rolling15m) + ); + setElementText('document-access-index-current-scope', state.current_source_scope || 'None'); + setElementText('document-access-index-completed-scopes', formatDocumentAccessIndexList(state.completed_source_scopes)); + setElementText('document-access-index-total-processed', formatNumber(state.total_documents_processed || 0)); + setElementText('document-access-index-total-failed', formatNumber(state.total_documents_failed || 0)); + setElementText('document-access-index-total-upserted', formatNumber(state.total_rows_upserted || 0)); + setElementText('document-access-index-total-deleted', formatNumber(state.total_rows_deleted || 0)); + setElementText('document-access-index-last-completed', state.last_completed_at || 'Not completed yet'); + setElementText('document-access-index-last-error', state.last_error || 'None'); + + const runButton = document.getElementById('document-access-index-run-batch-btn'); + const resetButton = document.getElementById('document-access-index-reset-btn'); + if (runButton) { + runButton.disabled = settings.container_enabled === false; + } + if (resetButton) { + resetButton.disabled = settings.container_enabled === false; + } + + if (isDocumentAccessIndexBackfillRunning(statusText)) { + startDocumentAccessIndexPolling(); + } else { + stopDocumentAccessIndexPolling(); } } -async function loadCosmosThroughputStatus(event = null) { - const triggerButton = event?.currentTarget || document.getElementById('cosmos-throughput-refresh-btn'); +async function loadDocumentAccessIndexStatus(event = null, options = {}) { + const showLoading = options.showLoading !== false; + const triggerButton = event?.currentTarget || (showLoading ? document.getElementById('document-access-index-refresh-btn') : null); + const isConversationCacheTrigger = triggerButton?.id === 'conversation-cache-refresh-btn'; if (triggerButton) { setButtonBusy(triggerButton, true, 'Loading...'); } - setCosmosThroughputMessage('Loading Cosmos throughput status...', 'info'); + if (showLoading) { + setDocumentAccessIndexMessage( + isConversationCacheTrigger ? 'Loading conversation cache metrics...' : 'Loading document access index status...', + 'info' + ); + } try { - const response = await fetch('/api/admin/settings/cosmos-throughput/status', { - method: 'GET', - headers: { 'Accept': 'application/json' }, - credentials: 'same-origin' - }); - const data = await response.json(); - if (!response.ok) { - throw new Error(data.error || 'Failed to load Cosmos throughput status.'); - } - - updateCosmosThroughputStatusPanel(data); - if (!data.configured) { - setCosmosThroughputMessage(data.error || 'Cosmos throughput management needs subscription, resource group, account, and database settings.', 'warning'); - } else if (data.throughput_error) { - setCosmosThroughputMessage(`Cosmos database throughput could not be read. ${data.throughput_error}`, 'danger'); - } else if (data.capacity_scope === 'container' && data.metrics?.normalized_ru_percent !== null && data.metrics?.normalized_ru_percent !== undefined && !hasContainerLevelCosmosMetrics(data)) { - setCosmosThroughputMessage('Azure Monitor returned aggregate RU utilization, but not per-container metric dimensions for this window. Container autoscale waits for per-container utilization before scaling individual containers; Refresh again after a few minutes.', 'warning'); - } else if (hasPortalManagedCosmosThroughput(data)) { - setCosmosThroughputMessage('One or more Cosmos throughput targets are above 10,000 RU/s. SimpleChat will monitor utilization and request units only; use the Azure portal for capacity changes, which can take 4 to 6 hours.', 'warning'); - } else if (data.capacity_scope === 'container') { - setCosmosThroughputMessage('Container-targeted throughput is active. Dedicated-throughput containers can be monitored and scaled individually; containers sharing database throughput remain view-only.', 'info'); - } else if (data.metric_error) { - setCosmosThroughputMessage('Throughput loaded, but Azure Monitor metrics are unavailable. Check the app identity permissions and Cosmos metrics availability.', 'warning'); - } else { - setCosmosThroughputMessage('Cosmos throughput status loaded.', 'success'); + const data = await fetchAppMaintenanceStatus('Failed to load document access index status.'); + + renderDocumentAccessIndexStatus(data); + renderConversationCacheStatus(data); + renderCosmosMaintenanceStatus(data); + if (showLoading) { + setDocumentAccessIndexMessage( + isConversationCacheTrigger ? 'Conversation cache metrics loaded.' : 'Document access index status loaded.', + 'success' + ); } } catch (error) { - setCosmosThroughputMessage(error.message || 'Failed to load Cosmos throughput status.', 'danger'); + setDocumentAccessIndexMessage( + error.message || ( + isConversationCacheTrigger + ? 'Failed to load conversation cache metrics.' + : 'Failed to load document access index status.' + ), + 'danger' + ); + stopDocumentAccessIndexPolling(); } finally { if (triggerButton) { setButtonBusy(triggerButton, false); @@ -1442,123 +2722,206 @@ async function loadCosmosThroughputStatus(event = null) { } } -function buildCosmosThroughputAccessPayload() { - return { - cosmos_throughput_autoscale_enabled: isFieldChecked('cosmos_throughput_autoscale_enabled'), - cosmos_throughput_auto_scale_up_enabled: isFieldChecked('cosmos_throughput_auto_scale_up_enabled'), - cosmos_throughput_auto_scale_down_enabled: isFieldChecked('cosmos_throughput_auto_scale_down_enabled'), - cosmos_throughput_subscription_id: getFieldValue('cosmos_throughput_subscription_id'), - cosmos_throughput_resource_group: getFieldValue('cosmos_throughput_resource_group'), - cosmos_throughput_account_name: getFieldValue('cosmos_throughput_account_name'), - cosmos_throughput_database_name: getFieldValue('cosmos_throughput_database_name'), - cosmos_throughput_metrics_window_minutes: getNumericFieldValue('cosmos_throughput_metrics_window_minutes', 5), - cosmos_throughput_scale_up_threshold_percent: getNumericFieldValue('cosmos_throughput_scale_up_threshold_percent', 90), - cosmos_throughput_scale_down_threshold_percent: getNumericFieldValue('cosmos_throughput_scale_down_threshold_percent', 70), - cosmos_throughput_scale_up_step_ru: getNumericFieldValue('cosmos_throughput_scale_up_step_ru', 1000), - cosmos_throughput_scale_down_step_ru: getNumericFieldValue('cosmos_throughput_scale_down_step_ru', 1000), - cosmos_throughput_scale_up_cooldown_minutes: getNumericFieldValue('cosmos_throughput_scale_up_cooldown_minutes', 5), - cosmos_throughput_scale_down_cooldown_minutes: getNumericFieldValue('cosmos_throughput_scale_down_cooldown_minutes', 20), - cosmos_throughput_min_ru: getNumericFieldValue('cosmos_throughput_min_ru', 1000), - cosmos_throughput_max_ru: getNumericFieldValue('cosmos_throughput_max_ru', COSMOS_THROUGHPUT_SIMPLECHAT_MAX_RU), - cosmos_throughput_ignore_min_limit: isFieldChecked('cosmos_throughput_ignore_min_limit'), - cosmos_throughput_ignore_max_limit: isFieldChecked('cosmos_throughput_ignore_max_limit'), - cosmos_throughput_convert_manual_to_autoscale_enabled: isFieldChecked('cosmos_throughput_convert_manual_to_autoscale_enabled'), - cosmos_throughput_enforce_container_defaults: isFieldChecked('cosmos_throughput_enforce_container_defaults'), - cosmos_throughput_container_policies: collectCosmosContainerPolicies() - }; +async function loadCosmosMaintenanceStatus(event = null, options = {}) { + const showLoading = options.showLoading !== false; + const triggerButton = event?.currentTarget || (showLoading ? document.getElementById('cosmos-maintenance-refresh-btn') : null); + if (triggerButton) { + setButtonBusy(triggerButton, true, 'Loading...'); + } + if (showLoading) { + setCosmosMaintenanceMessage('Loading Cosmos maintenance status...', 'info'); + } + + try { + const data = await fetchAppMaintenanceStatus('Failed to load Cosmos maintenance status.'); + renderDocumentAccessIndexStatus(data); + renderConversationCacheStatus(data); + renderCosmosMaintenanceStatus(data); + if (showLoading) { + setCosmosMaintenanceMessage('Cosmos maintenance status loaded.', 'success'); + } + } catch (error) { + setCosmosMaintenanceMessage(error.message || 'Failed to load Cosmos maintenance status.', 'danger'); + } finally { + if (triggerButton) { + setButtonBusy(triggerButton, false); + } + } } -async function validateCosmosThroughputAccess(triggerButton = null) { - const button = triggerButton || document.getElementById('cosmos-throughput-validate-access-btn'); - if (button) { - setButtonBusy(button, true, 'Validating...'); +async function runStaleCacheCleanup(options = {}) { + const applyChanges = Boolean(options.applyChanges); + const triggerButton = options.triggerButton || document.getElementById( + applyChanges ? 'stale-cache-cleanup-apply-confirm-btn' : 'stale-cache-cleanup-dry-run-btn' + ); + if (triggerButton) { + setButtonBusy(triggerButton, true, applyChanges ? 'Deleting...' : 'Scanning...'); } - setCosmosThroughputMessage('Validating Cosmos throughput configuration and access...', 'info'); + setCosmosMaintenanceMessage( + applyChanges ? 'Deleting one bounded batch of stale cache documents...' : 'Scanning for stale cache cleanup candidates...', + 'info' + ); try { - const response = await fetch('/api/admin/settings/cosmos-throughput/validate-access', { + const response = await fetch('/api/admin/settings/app-maintenance/run', { method: 'POST', headers: { 'Accept': 'application/json', 'Content-Type': 'application/json' }, credentials: 'same-origin', - body: JSON.stringify(buildCosmosThroughputAccessPayload()) + body: JSON.stringify({ + apply_cosmos_indexing_policies: false, + run_document_access_index_backfill: false, + run_stale_cache_cleanup: true, + apply_stale_cache_cleanup: applyChanges + }) }); const data = await response.json(); - if (!response.ok) { - throw new Error(data.error || 'Failed to validate Cosmos throughput access.'); + const cleanupResult = getStaleCacheCleanupStatusFromRunResult(data); + if (cleanupResult) { + renderCosmosMaintenanceStatus({ stale_cache_cleanup: cleanupResult }); + } + if (!response.ok || !data.success) { + throw new Error(data.error || 'Stale cache cleanup failed.'); } - if (data.status?.configured) { - updateCosmosThroughputStatusPanel(data.status); + const candidates = formatNumber(cleanupResult?.candidate_count || 0); + const deleted = formatNumber(cleanupResult?.deleted_count || 0); + const message = applyChanges + ? `Stale cache cleanup deleted ${deleted} document(s) from ${candidates} candidate(s).` + : `Stale cache cleanup dry run found ${candidates} candidate document(s).`; + setCosmosMaintenanceMessage(message, cleanupResult?.has_more_candidates ? 'warning' : 'success'); + showToast(message, cleanupResult?.has_more_candidates ? 'warning' : 'success'); + + const modalElement = document.getElementById('staleCacheCleanupApplyModal'); + if (applyChanges && modalElement) { + bootstrap.Modal.getInstance(modalElement)?.hide(); } - setCosmosThroughputValidationResult(data); + await loadCosmosMaintenanceStatus(null, { showLoading: false }); } catch (error) { - setCosmosThroughputMessage(error.message || 'Failed to validate Cosmos throughput access.', 'danger'); + setCosmosMaintenanceMessage(error.message || 'Stale cache cleanup failed.', 'danger'); + showToast(error.message || 'Stale cache cleanup failed.', 'danger'); } finally { - if (button) { - setButtonBusy(button, false); + if (triggerButton) { + setButtonBusy(triggerButton, false); } } } -async function manuallyScaleCosmosThroughput(direction, containerName = '', triggerButton = null) { - const button = triggerButton || document.getElementById(`cosmos-throughput-scale-${direction}-btn`); - if (button) { - setButtonBusy(button, true, direction === 'up' ? 'Scaling up...' : 'Scaling down...'); +async function runCosmosIndexingPolicyApply(options = {}) { + const triggerButton = options.triggerButton || document.getElementById('cosmos-indexing-policy-apply-confirm-btn'); + if (triggerButton) { + setButtonBusy(triggerButton, true, 'Applying...'); } - const targetText = containerName ? ` for ${containerName}` : ''; - setCosmosThroughputMessage(direction === 'up' ? `Submitting scale-up request${targetText}...` : `Submitting scale-down request${targetText}...`, 'info'); + setCosmosMaintenanceMessage( + 'Submitting missing Cosmos composite index updates. Index transformation may continue asynchronously in Azure Cosmos DB.', + 'info' + ); try { - const response = await fetch('/api/admin/settings/cosmos-throughput/scale', { + const response = await fetch('/api/admin/settings/app-maintenance/run', { method: 'POST', - headers: { 'Content-Type': 'application/json' }, + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json' + }, credentials: 'same-origin', - body: JSON.stringify({ direction, container_name: containerName }) + body: JSON.stringify({ + apply_cosmos_indexing_policies: true, + run_document_access_index_backfill: false, + run_stale_cache_cleanup: false + }) }); const data = await response.json(); - if (!response.ok) { - throw new Error(data.error || 'Cosmos throughput scale request failed.'); + const indexingResult = getCosmosIndexingPolicyStatusFromRunResult(data); + if (indexingResult) { + renderCosmosMaintenanceStatus({ cosmos_indexing_policies: indexingResult }); + } + if (!response.ok || !data.success) { + throw new Error(data.error || 'Cosmos indexing policy update failed.'); } - const scaledTarget = data.container_name ? ` for ${data.container_name}` : ''; - setCosmosThroughputMessage(`Cosmos throughput${scaledTarget} changed from ${formatRu(data.from_ru)} to ${formatRu(data.to_ru)}.`, 'success'); - await loadCosmosThroughputStatus(); + const updated = formatNumber(indexingResult?.updated_container_count || 0); + const missing = formatNumber(indexingResult?.containers_missing_expected_indexes || 0); + const message = Number(indexingResult?.updated_container_count || 0) > 0 + ? `Cosmos indexing policy update submitted for ${updated} container(s). Refresh later to monitor transformation status.` + : `Cosmos indexing policies are already aligned. Missing expected indexes: ${missing}.`; + setCosmosMaintenanceMessage(message, 'success'); + showToast(message, 'success'); + + const modalElement = document.getElementById('cosmosIndexingPolicyApplyModal'); + if (modalElement) { + bootstrap.Modal.getInstance(modalElement)?.hide(); + } + await loadCosmosMaintenanceStatus(null, { showLoading: false }); } catch (error) { - setCosmosThroughputMessage(error.message || 'Cosmos throughput scale request failed.', 'danger'); + setCosmosMaintenanceMessage(error.message || 'Cosmos indexing policy update failed.', 'danger'); + showToast(error.message || 'Cosmos indexing policy update failed.', 'danger'); } finally { - if (button) { - setButtonBusy(button, false, direction === 'up' ? 'Scale Up' : 'Scale Down'); + if (triggerButton) { + setButtonBusy(triggerButton, false); } } } -async function convertCosmosThroughputToAutoscale(containerName = '', triggerButton = null) { - const targetText = containerName ? ` for ${containerName}` : ''; +async function runDocumentAccessIndexBackfillBatch(options = {}) { + const reset = Boolean(options.reset); + const triggerButton = options.triggerButton || document.getElementById(reset ? 'document-access-index-reset-confirm-btn' : 'document-access-index-run-batch-btn'); if (triggerButton) { - setButtonBusy(triggerButton, true, 'Converting...'); + setButtonBusy(triggerButton, true, reset ? 'Resetting...' : 'Running...'); } - setCosmosThroughputMessage(`Converting manual Cosmos throughput${targetText} to native autoscale...`, 'info'); + setDocumentAccessIndexMessage(reset ? 'Resetting backfill checkpoint and running one batch...' : 'Running one document access backfill batch...', 'info'); try { - const response = await fetch('/api/admin/settings/cosmos-throughput/convert-autoscale', { + const response = await fetch('/api/admin/settings/app-maintenance/run', { method: 'POST', - headers: { 'Content-Type': 'application/json' }, + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json' + }, credentials: 'same-origin', - body: JSON.stringify({ container_name: containerName }) + body: JSON.stringify({ + apply_cosmos_indexing_policies: false, + run_document_access_index_backfill: true, + reset_document_access_index_backfill: reset + }) }); const data = await response.json(); - if (!response.ok) { - throw new Error(data.error || 'Cosmos throughput mode conversion failed.'); + const currentStatus = getDocumentAccessIndexBackfillStatusFromRunResult(data); + if (currentStatus) { + renderDocumentAccessIndexStatus(currentStatus); + } + if (!response.ok || !data.success) { + throw new Error(data.error || 'Document access index backfill batch failed.'); + } + + const status = currentStatus?.state?.status || 'completed'; + const statusVariant = getDocumentAccessIndexStatusVariant(status); + if (isDocumentAccessIndexBackfillRunning(status)) { + setDocumentAccessIndexMessage('Backfill batch is still running. Status will refresh automatically.', 'info'); + } else if (isDocumentAccessIndexBackfillInProgress(status)) { + setDocumentAccessIndexMessage('Backfill batch completed and more documents remain. Run another batch or leave scheduled backfill enabled to continue during maintenance.', 'info'); + } else if (statusVariant === 'danger' || statusVariant === 'warning') { + setDocumentAccessIndexMessage('Backfill batch completed with errors. Review the repair backlog and last error details.', 'warning'); + } else { + setDocumentAccessIndexMessage('Document access index backfill batch completed.', 'success'); } + showToast( + statusVariant === 'danger' || statusVariant === 'warning' + ? 'Document access index backfill batch completed with errors.' + : 'Document access index backfill batch completed.', + statusVariant === 'danger' ? 'danger' : statusVariant === 'warning' ? 'warning' : 'success' + ); - const convertedTarget = data.container_name ? ` for ${data.container_name}` : ''; - setCosmosThroughputMessage(`Cosmos throughput${convertedTarget} converted from manual ${formatRu(data.from_ru)} to autoscale max ${formatRu(data.to_ru)}.`, 'success'); - await loadCosmosThroughputStatus(); + const modalElement = document.getElementById('documentAccessIndexResetModal'); + if (reset && modalElement) { + bootstrap.Modal.getInstance(modalElement)?.hide(); + } } catch (error) { - setCosmosThroughputMessage(error.message || 'Cosmos throughput mode conversion failed.', 'danger'); + setDocumentAccessIndexMessage(error.message || 'Document access index backfill batch failed.', 'danger'); + showToast(error.message || 'Document access index backfill batch failed.', 'danger'); + stopDocumentAccessIndexPolling(); } finally { if (triggerButton) { setButtonBusy(triggerButton, false); @@ -1566,6 +2929,41 @@ async function convertCosmosThroughputToAutoscale(containerName = '', triggerBut } } +function setupDocumentAccessIndexControls() { + const section = document.getElementById('document-access-index-section'); + if (!section) { + return; + } + + document.getElementById('document-access-index-refresh-btn')?.addEventListener('click', loadDocumentAccessIndexStatus); + document.getElementById('conversation-cache-refresh-btn')?.addEventListener('click', loadDocumentAccessIndexStatus); + document.getElementById('document-access-index-run-batch-btn')?.addEventListener('click', event => { + runDocumentAccessIndexBackfillBatch({ triggerButton: event.currentTarget }); + }); + document.getElementById('document-access-index-reset-confirm-btn')?.addEventListener('click', event => { + runDocumentAccessIndexBackfillBatch({ reset: true, triggerButton: event.currentTarget }); + }); + loadDocumentAccessIndexStatus(null, { showLoading: false }); +} + +function setupCosmosMaintenanceControls() { + const section = document.getElementById('cosmos-maintenance-section'); + if (!section) { + return; + } + + document.getElementById('cosmos-maintenance-refresh-btn')?.addEventListener('click', loadCosmosMaintenanceStatus); + document.getElementById('cosmos-indexing-policy-apply-confirm-btn')?.addEventListener('click', event => { + runCosmosIndexingPolicyApply({ triggerButton: event.currentTarget }); + }); + document.getElementById('stale-cache-cleanup-dry-run-btn')?.addEventListener('click', event => { + runStaleCacheCleanup({ applyChanges: false, triggerButton: event.currentTarget }); + }); + document.getElementById('stale-cache-cleanup-apply-confirm-btn')?.addEventListener('click', event => { + runStaleCacheCleanup({ applyChanges: true, triggerButton: event.currentTarget }); + }); +} + function setupCosmosThroughputControls() { const section = document.getElementById('cosmos-throughput-section'); if (!section) { @@ -2943,6 +4341,9 @@ document.addEventListener('DOMContentLoaded', () => { // --- NEW: Chunk size controls --- setupChunkSizeControls(); + setupRedisMonitoringControls(); + setupCosmosMaintenanceControls(); + setupDocumentAccessIndexControls(); setupCosmosThroughputControls(); // --- Setup form change tracking --- @@ -4528,10 +5929,10 @@ function setupToggles() { const enableRedisCache = document.getElementById('enable_redis_cache'); const redisSettingsDiv = document.getElementById('redis_cache_settings'); if (enableRedisCache && redisSettingsDiv) { - // Set initial state - redisSettingsDiv.style.display = enableRedisCache.checked ? 'block' : 'none'; + updateRedisCanonicalCacheVisibility(enableRedisCache.checked); enableRedisCache.addEventListener('change', function () { - redisSettingsDiv.style.display = this.checked ? 'block' : 'none'; + updateRedisCanonicalCacheVisibility(this.checked); + markFormAsModified(); }); } @@ -5758,7 +7159,7 @@ function setupTestButtons() { if (testRedisBtn) { testRedisBtn.addEventListener('click', async () => { const resultDiv = document.getElementById('test_redis_result'); - resultDiv.innerHTML = 'Testing Redis...'; + setRedisTestResult(resultDiv, 'Testing Redis...', 'muted'); const payload = { test_type: 'redis', @@ -5775,12 +7176,13 @@ function setupTestButtons() { }); const data = await resp.json(); if (resp.ok) { - resultDiv.innerHTML = `${data.message}`; + setRedisTestResult(resultDiv, data.message || 'Redis connection successful.', 'success'); + loadRedisMonitoringStatus(null, { showLoading: false }); } else { - resultDiv.innerHTML = `${data.error || 'Error testing Redis'}`; + setRedisTestResult(resultDiv, data.error || 'Error testing Redis', 'danger'); } } catch (err) { - resultDiv.innerHTML = `Error: ${err.message}`; + setRedisTestResult(resultDiv, `Error: ${err.message}`, 'danger'); } }); } @@ -6334,7 +7736,7 @@ function updateRedisCanonicalCacheVisibility(isEnabled) { const redisSettingsDiv = document.getElementById('redis_cache_settings'); if (redisSettingsDiv) { - redisSettingsDiv.style.display = isEnabled ? 'block' : 'none'; + redisSettingsDiv.classList.toggle('d-none', !isEnabled); } } diff --git a/application/single_app/static/js/admin/admin_sidebar_nav.js b/application/single_app/static/js/admin/admin_sidebar_nav.js index 21bc0db7..0b9c9d60 100644 --- a/application/single_app/static/js/admin/admin_sidebar_nav.js +++ b/application/single_app/static/js/admin/admin_sidebar_nav.js @@ -1,3 +1,4 @@ +// admin_sidebar_nav.js // Admin Sidebar Navigation document.addEventListener('DOMContentLoaded', function() { // Only initialize if we're on admin settings page with sidebar nav @@ -201,6 +202,11 @@ function scrollToSection(sectionId) { 'file-processing-logs-section': 'file-processing-logs-section', // Scale tab sections 'redis-cache-section': 'redis-cache-section', + 'redis-monitoring-section': 'redis-monitoring-section', + 'document-access-index-section': 'document-access-index-section', + 'cosmos-maintenance-section': 'cosmos-maintenance-section', + 'cosmos-throughput-section': 'cosmos-throughput-section', + 'cosmos-throughput-metrics-table-section': 'cosmos-throughput-metrics-table-section', 'front-door-section': 'front-door-section', // Workspaces tab sections 'personal-workspaces-section': 'personal-workspaces-section', diff --git a/application/single_app/static/js/chat/chat-conversations.js b/application/single_app/static/js/chat/chat-conversations.js index 77dd90e6..c5e8027e 100644 --- a/application/single_app/static/js/chat/chat-conversations.js +++ b/application/single_app/static/js/chat/chat-conversations.js @@ -1698,7 +1698,7 @@ export async function selectConversation(conversationId) { } catch (error) { console.warn('Failed to reattach active stream for conversation:', error); } - markConversationRead(conversationId, { force: true, suppressErrorToast: true }).catch(error => { + markConversationRead(conversationId, { suppressErrorToast: true }).catch(error => { console.warn('Failed to clear unread state for conversation:', error); }); } diff --git a/application/single_app/static/js/chat/chat-streaming.js b/application/single_app/static/js/chat/chat-streaming.js index 24d6b89f..c08a1cf0 100644 --- a/application/single_app/static/js/chat/chat-streaming.js +++ b/application/single_app/static/js/chat/chat-streaming.js @@ -74,6 +74,21 @@ function getStreamingMessageElement(messageId) { return document.querySelector(`[data-message-id="${messageId}"]`); } +function isConversationCurrentlyActive(conversationId) { + const normalizedConversationId = String(conversationId || '').trim(); + return Boolean(normalizedConversationId) && window.currentConversationId === normalizedConversationId; +} + +function markStreamingConversationReadIfActive(conversationId, contextLabel) { + if (!isConversationCurrentlyActive(conversationId)) { + return; + } + + markConversationRead(conversationId, { force: true, suppressErrorToast: true }).catch(error => { + console.warn(`Failed to clear unread state after ${contextLabel}:`, error); + }); +} + function buildDefaultCancelEndpoint(conversationId) { const normalizedConversationId = String(conversationId || '').trim(); if (!normalizedConversationId) { @@ -1148,11 +1163,7 @@ function finalizeCancelledStreamingMessage(messageId, userMessageId, finalData, renderStoppedContent(messageElement, partialContent); } - if (finalData.conversation_id) { - markConversationRead(finalData.conversation_id, { force: true, suppressErrorToast: true }).catch(error => { - console.warn('Failed to clear unread state after stream cancellation:', error); - }); - } + markStreamingConversationReadIfActive(finalData.conversation_id, 'stream cancellation'); } function normalizeFallbackAgentIcon(iconPayload) { @@ -1252,11 +1263,7 @@ function finalizeStreamingMessage(messageId, userMessageId, finalData, fallbackA } if (existingFinalMessage) { - if (finalData.conversation_id) { - markConversationRead(finalData.conversation_id, { force: true, suppressErrorToast: true }).catch(error => { - console.warn('Failed to clear unread state after live streaming completion:', error); - }); - } + markStreamingConversationReadIfActive(finalData.conversation_id, 'live streaming completion'); return; } @@ -1307,7 +1314,7 @@ function finalizeStreamingMessage(messageId, userMessageId, finalData, fallbackA ); // Update conversation if needed - if (finalData.conversation_id && window.currentConversationId !== finalData.conversation_id) { + if (finalData.conversation_id && !window.currentConversationId) { window.currentConversationId = finalData.conversation_id; } @@ -1348,11 +1355,7 @@ function finalizeStreamingMessage(messageId, userMessageId, finalData, fallbackA window.chatMessages.loadMessages(finalData.conversation_id); } - if (finalData.conversation_id) { - markConversationRead(finalData.conversation_id, { force: true, suppressErrorToast: true }).catch(error => { - console.warn('Failed to clear unread state after live streaming completion:', error); - }); - } + markStreamingConversationReadIfActive(finalData.conversation_id, 'live streaming completion'); } export function cancelStreaming() { diff --git a/application/single_app/templates/_sidebar_nav.html b/application/single_app/templates/_sidebar_nav.html index aedab427..9e80ba11 100644 --- a/application/single_app/templates/_sidebar_nav.html +++ b/application/single_app/templates/_sidebar_nav.html @@ -579,6 +579,31 @@ Redis Cache +
  • + + Redis Metrics + +
    • +
  • + + DAI Metrics + +
    • +
  • + + Cosmos Maintenance + +
    • +
  • + + Cosmos DB Throughput + +
    • +
  • + + Cosmos Metrics + +
  • Azure Front Door diff --git a/application/single_app/templates/admin_settings.html b/application/single_app/templates/admin_settings.html index 59fd094b..d3c2a174 100644 --- a/application/single_app/templates/admin_settings.html +++ b/application/single_app/templates/admin_settings.html @@ -308,6 +308,64 @@ word-break: break-word; } + .redis-explorer-dialog { + max-width: min(1180px, calc(100vw - 2rem)); + } + + .redis-explorer-content { + height: min(88vh, 920px); + } + + .redis-explorer-body { + overflow-y: auto; + min-height: 0; + } + + .redis-explorer-alerts, + .redis-explorer-toolbar, + .redis-explorer-pagination { + flex: 0 0 auto; + } + + .redis-explorer-browser { + flex: 0 0 auto; + min-height: 0; + } + + .redis-explorer-column, + .redis-explorer-preview-card { + min-height: 0; + } + + .redis-explorer-key-list { + height: clamp(16rem, 36vh, 24rem); + overflow: auto; + overscroll-behavior: contain; + } + + .redis-explorer-preview-card { + min-height: clamp(18rem, 40vh, 28rem); + } + + .redis-explorer-preview-content { + flex: 1 1 auto; + min-height: 0; + max-height: clamp(10rem, 28vh, 18rem); + overflow: auto; + overscroll-behavior: contain; + } + + .redis-explorer-preview-panel:not(.d-none) { + display: flex; + flex: 1 1 auto; + min-height: 0; + flex-direction: column; + } + + .redis-explorer-preview-content { + white-space: pre-wrap; + } + @media (max-width: 768px) { .latest-feature-header { gap: 0.75rem; @@ -5233,6 +5291,7 @@
    Warnings
    + {% set enable_dai_debug = settings.enable_dai_debug | default(false) %}

    Configure Redis cache to improve enterprise scale and performance by caching session data. Enabling Redis allows you to horizontally scale your application across multiple instances without losing session data.

    @@ -5256,7 +5315,7 @@
    -
    +

    (example: simple-chat.redis.cache.windows.net)

    @@ -5293,6 +5352,1072 @@
    +
    +
    +
    +
    + Redis Monitoring +
    +

    + Monitor Redis availability, memory pressure, hit rate, evictions, and runtime cache usage before Redis-backed document list caching is enabled. +

    +
    +
    + + +
    +
    +
    +
    +
    +
    +
    Configuration
    + Not loaded +
    +
    +
    +
    +
    Health
    + Not loaded +
    +
    +
    +
    +
    App Cache Runtime
    + Not loaded +
    +
    +
    +
    +
    Session Runtime
    + Not loaded +
    +
    +
    +
    +
    +
    +
    Ping Latency
    +
    Not loaded
    +
    +
    +
    +
    +
    Memory Usage
    +
    Not loaded
    +
    Not loaded
    +
    +
    +
    +
    +
    Connected Clients
    +
    Not loaded
    +
    +
    +
    +
    +
    Ops/sec
    +
    Not loaded
    +
    +
    +
    +
    +
    Keyspace Hit Rate
    +
    Not loaded
    +
    +
    +
    +
    +
    Tracked Keys
    +
    Not loaded
    +
    +
    +
    +
    +
    DAI Version Markers
    +
    Not loaded
    +
    Not loaded
    +
    +
    +
    +
    +
    DAI Cache Payloads
    +
    Not loaded
    +
    Not loaded
    +
    +
    +
    +
    +
    Expired / Evicted Keys
    +
    + Not loaded + / + Not loaded +
    +
    +
    +
    +
    +
    Fragmentation Ratio
    +
    Not loaded
    +
    +
    +
    +
    +
    Error Replies
    +
    Not loaded
    +
    +
    +
    +
    +
    Rejected Connections
    +
    Not loaded
    +
    +
    +
    +
    +
    Redis Version
    +
    Not loaded
    +
    +
    +
    +
    +
    Monitoring Source
    +
    Not loaded
    +
    +
    +
    +
    +
    Last Checked
    +
    Not loaded
    +
    +
    +
    +
    +
    Last Error
    +
    Not loaded
    +
    +
    +
    +
    +
    + + + +
    +
    +
    +
    + Conversation Cache +
    +

    + Cache conversation list, feed, and advanced-search results per user. Redis is optional; cache misses and disabled cache paths continue using source Cosmos queries. +

    +
    + +
    +
    +
    +
    + + +
    +
    When enabled, list/feed/search payloads are cached by user and version. Disabling this bypasses cache reads and writes without requiring Redis.
    +
    +
    + + +
    Default 120 seconds. User-scoped version invalidation refreshes changed conversations; set to 0 to skip writing new entries.
    +
    +
    +
    +
    +
    +
    Runtime Status
    + Not loaded +
    +
    +
    +
    +
    15m Cache Hit Rate
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Cache Hits / Misses
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Cache Bypasses / Errors
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Writes / Invalidations
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Operation Mix
    +
    Not loaded
    +
    +
    +
    +
    +
    Last Cache Event
    +
    Not loaded
    +
    +
    +
    +
    +
    Last Invalidation
    +
    Not loaded
    +
    +
    +
    +
    + + Conversation cache metrics are lightweight in-process counters for the current app worker. Application Insights logs remain the durable fleet-wide source for cache warnings and fallback behavior. +
    +
    +
    +
    + +
    +
    +
    +
    + Cosmos Document Access Index +
    +

    + Monitor the document access projection used to remove expensive cross-partition document access queries. +

    +
    +
    + + {% if enable_dai_debug %} + + + {% endif %} +
    +
    + +
    + + Document access projection maintenance is automatic. The background scheduler repairs fail-open projection records first, then runs bounded backfill batches repeatedly while work remains. Production read metrics below show DAI-served reads, Redis cache hits, source fallbacks, RU, and latency without requiring shadow validation. + {% if enable_dai_debug %}Debug controls and shadow validation diagnostics are visible because enable_dai_debug is enabled in app settings.{% endif %} +
    + +
    + +
    +
    +
    +
    Container
    + Not loaded +
    +
    +
    +
    +
    Write-through
    + Not loaded +
    +
    +
    +
    +
    Read Path
    + Not loaded +
    +
    +
    +
    +
    Redis List Cache
    + Not loaded +
    +
    + {% if enable_dai_debug %} +
    +
    +
    Shadow Validation
    + Not loaded +
    +
    + {% endif %} +
    +
    +
    Auto Maintenance
    + Not loaded +
    +
    +
    +
    +
    Next Maintenance Action
    +
    Not loaded
    +
    +
    +
    +
    +
    More Work Pending
    +
    Not loaded
    +
    +
    +
    +
    +
    Active Loop Interval
    +
    Not loaded
    +
    +
    +
    + + {% if enable_dai_debug %} +
    +
    Automatic Maintenance and Diagnostics
    +
    +
    +
    + + +
    +
    Always on. New and changed documents synchronize into the access index and fail open to repair records if projection updates fail.
    +
    +
    +
    + + +
    +
    Always on. Maintenance keeps running bounded repair and backfill batches until DAI is healthy.
    +
    +
    +
    + + +
    +
    Compares source list results to projection rows and logs mismatches without changing reads.
    +
    +
    + + +
    Documents processed per manual or scheduled batch.
    +
    +
    + + +
    Fail-open repair records reconciled before each backfill batch.
    +
    +
    +
    + +
    +
    Default Read Path
    +
    +
    +
    + + +
    +
    Always on. DAI-backed document and tag list reads are the normal path; source-container fallback remains automatic when backfill is not ready, repairs are pending, or a DAI query fails.
    +
    +
    +
    + + +
    +
    Uses Redis read-through caching for DAI document, tag, and legacy-count reads. If Redis is unavailable, reads bypass cache and use DAI directly.
    +
    +
    + + +
    Default 900 seconds. Scope-version invalidation makes document changes visible immediately; TTL clears unreachable old entries.
    +
    +
    +
    + {% endif %} + +
    +
    +
    +
    Backfill State
    + Not loaded +
    +
    +
    +
    +
    Repair Backlog
    +
    Not loaded
    +
    +
    +
    +
    +
    15m DAI Read Attempts
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Redis Cache Hit Rate
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Cache Hits / Misses
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Cache Bypasses / Errors
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Cache Invalidations
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Served from DAI
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Source Fallbacks
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Fallback Rate
    +
    Not loaded
    +
    +
    +
    +
    +
    15m DAI Read RU
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Avg / P95 Latency
    +
    Not loaded
    +
    +
    +
    +
    +
    Last Fallback Reason
    +
    Not loaded
    +
    +
    +
    +
    +
    Last DAI Read Metric
    +
    Not loaded
    +
    +
    +
    +
    +
    Last Cache Event
    +
    Not loaded
    +
    +
    +
    +
    + + Production read metrics are lightweight in-process counters for the current app worker. Application Insights logs remain the durable fleet-wide source for fallback warnings and query failures. +
    +
    + {% if enable_dai_debug %} +
    +
    +
    Last Shadow Result
    + Not run +
    +
    +
    +
    +
    Shadow Mismatches
    +
    Not loaded
    +
    +
    +
    +
    +
    Source / Validation RU
    +
    Not loaded
    +
    +
    +
    +
    +
    Validation Index RU
    +
    Not loaded
    +
    +
    +
    +
    +
    Candidate Read RU
    +
    Not loaded
    +
    +
    +
    +
    +
    Estimated Wave 5 Savings
    +
    Not loaded
    +
    +
    +
    +
    +
    Source / Candidate Latency
    +
    Not loaded
    +
    +
    +
    +
    +
    Estimated Wave 5 Latency
    +
    Not loaded
    +
    +
    +
    +
    + + Rolling decision metrics aggregate shadow-validation samples over recent windows. Use these totals to compare source container RU with candidate access-index RU before enabling the future read path or Redis document access cache. +
    +
    +
    +
    +
    5m Source / Candidate RU
    +
    Not loaded
    +
    +
    +
    +
    +
    5m Estimated Wave 5 Savings
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Source / Candidate RU
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Estimated Wave 5 Savings
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Validation Overhead
    +
    Not loaded
    +
    +
    +
    +
    +
    15m Shadow Samples
    +
    Not loaded
    +
    +
    + {% endif %} +
    +
    +
    Current Scope
    +
    Not loaded
    +
    +
    +
    +
    +
    Completed Scopes
    +
    Not loaded
    +
    +
    +
    +
    +
    Total Documents Processed
    +
    Not loaded
    +
    +
    +
    +
    +
    Total Documents Failed
    +
    Not loaded
    +
    +
    +
    +
    +
    Rows Upserted
    +
    Not loaded
    +
    +
    +
    +
    +
    Rows Deleted
    +
    Not loaded
    +
    +
    +
    +
    +
    Last Batch Completed
    +
    Not loaded
    +
    +
    +
    +
    +
    Last Error
    +
    Not loaded
    +
    +
    +
    +
    + +
    +
    +
    +
    + Cosmos Maintenance +
    +

    + Review expected indexing policies and clean up stale operational cache documents without touching source configuration or user data. +

    +
    +
    + + + + +
    +
    + +
    + + Indexing maintenance only adds missing expected composite indexes and preserves existing policy paths. Composite indexes can increase write-index overhead while improving supported lookup and ordered-query speed. Stale cleanup is allowlisted to obsolete cache artifacts, supports dry-run review, and deletes at most one bounded batch per run. +
    +
    + +
    +
    +
    +
    Indexing Policy Status
    + Not loaded +
    +
    +
    +
    +
    Indexing Mode
    +
    Not loaded
    +
    +
    +
    +
    +
    Containers Checked
    +
    Not loaded
    +
    +
    +
    +
    +
    Missing Expected Indexes
    +
    Not loaded
    +
    +
    +
    +
    +
    Updated Containers
    +
    Not loaded
    +
    +
    +
    +
    +
    Indexing Failures
    +
    Not loaded
    +
    +
    +
    +
    +
    Last Indexing Evaluation
    +
    Not loaded
    +
    +
    +
    +
    +
    Stale Cleanup Status
    + Not loaded +
    +
    +
    +
    +
    Cleanup Mode
    +
    Not loaded
    +
    +
    +
    +
    +
    Cleanup Candidates
    +
    Not loaded
    +
    +
    +
    +
    +
    Deleted Docs
    +
    Not loaded
    +
    +
    +
    +
    +
    Cleanup Failures
    +
    Not loaded
    +
    +
    +
    +
    +
    More Candidates
    +
    Not loaded
    +
    +
    +
    +
    +
    Cleanup Categories
    +
    Not loaded
    +
    +
    +
    +
    +
    Last Cleanup Evaluation
    +
    Not loaded
    +
    +
    +
    +
    + + + + + + {% if enable_dai_debug %} +
    @@ -5590,7 +6715,7 @@
    Scale Down Policy
    -
    +
    diff --git a/docs/explanation/features/COSMOS_PERFORMANCE_OPTIMIZATION_PLAN.md b/docs/explanation/features/COSMOS_PERFORMANCE_OPTIMIZATION_PLAN.md index 0e6d0402..d15dae18 100644 --- a/docs/explanation/features/COSMOS_PERFORMANCE_OPTIMIZATION_PLAN.md +++ b/docs/explanation/features/COSMOS_PERFORMANCE_OPTIMIZATION_PLAN.md @@ -2,11 +2,11 @@ ## Header Information -**Status**: Proposal for technical review -**Documented against version**: **0.242.044** -**Implemented in version**: **TBD** -**Related config.py version update**: No version change is included with this proposal. Implementation should increment `VERSION` in `application/single_app/config.py` when code changes are made. -**Primary audience**: SimpleChat technical leadership and application developers +**Status**: Implemented through Phase 9 hardening docs, Cosmos indexing maintenance, stale cache cleanup maintenance, and Admin Settings maintenance visibility +**Documented against version**: **0.250.039** +**Implemented in version**: **0.250.005 - 0.250.039** +**Related config.py version update**: Phase 9 hardening docs and the Admin Settings guarded index-apply action are implemented in `application/single_app/config.py` version **0.250.039**. +**Primary audience**: SimpleChat technical leadership and application developers **Primary goal**: Reduce high-volume Azure Cosmos DB reads and repeated cross-partition queries while preserving the current application architecture and existing Azure AI Search indexing model. ## Review Summary @@ -15,6 +15,103 @@ This plan proposes a phased Cosmos performance program rather than a single larg The recommendation is to approve the full direction but implement it in phases, with feature flags and shadow validation before switching document list endpoints to the companion container. +### 2026-06-29 Technical Review Updates + +The repository review confirmed the main direction and added several implementation gates that should be treated as required before user-facing read paths change: + +1. **Baseline first**: Capture request charge, query metrics, index metrics, latency, result count, and source/result equivalence for the current hot paths before code changes. The Python SDK can expose `x-ms-request-charge`, query metrics, and index metrics after query iteration. +2. **Add a Phase 0 design gate**: Finalize projection row identifiers, shared-user/shared-group approval-state normalization, version-family/current-document semantics, and stale-index fallback rules before adding write-through hooks. +3. **Use continuation-token pagination where possible**: `OFFSET LIMIT` can be acceptable for shallow page-number compatibility, but deep paging should prefer SDK continuation tokens because offsets still require skipped work. +4. **Treat permission-reducing updates as security-sensitive**: Share removal, delete, retention delete, archive, and visibility changes must not leave stale `document_access_index` rows or Azure AI Search chunk visibility behind without repair tracking and safe read fallback. +5. **Keep Azure AI Search visibility synchronized**: The companion container optimizes Cosmos list screens only. It does not replace existing search-index visibility fields or source-of-truth authorization checks. +6. **Use shadow validation before reads**: Backfill plus write-through is not enough. Source-container results and `document_access_index` results must be compared by scope, sort, filter, and page before enabling reads. + +### 2026-07-01 Wave 4B Implementation Update + +Wave 4B adds shadow validation without switching any document list read paths. Personal, group, and public document list routes still return source-container results, then compare those authoritative current-document identities with `document_access_index` projection rows when `enable_document_access_index_shadow_validation` is enabled. The latest shadow validation result is persisted to the settings container and surfaced in Admin Settings > Scale > Cosmos Document Access Index. Read switchover remains disabled and reserved for Wave 5. + +### 2026-07-01 Wave 4B.1 Shadow Metrics Update + +Wave 4B.1 adds source-versus-projection query diagnostics to shadow validation. When shadow validation is enabled, document list routes capture the source query elapsed time and Cosmos request charge where the SDK provides it, then compare those values with the single-partition projection query. The admin dashboard displays source/index RU, estimated RU savings, source/index latency, and estimated latency savings. These values estimate future read-path benefits; shadow mode still runs the source query plus the projection query until Wave 5 enables controlled index reads. + +### 2026-07-01 Wave 4B2 Candidate Read Metrics Update + +Wave 4B2 keeps full parity validation in place but adds a separate candidate read query against `document_access_index`. The corrected candidate query is one single-partition query per scope key, returns all current rows for that scope, avoids Cosmos-side `ORDER BY`, `OFFSET`, `LIMIT`, and `TOP`, and projects only the fields needed by the document list UI plus `source_ts` for the current default sort contract. App Service remains responsible for sort/filter/page shaping until Wave 5 read switchover is deliberately implemented. Admin Settings separates **Validation Index RU** from **Candidate Read RU** and uses the candidate scope-read diagnostics for **Estimated Wave 5 Savings**. This prevents the full-scope parity query and Cosmos-side sort/page costs from overstating the expected cost of the future Wave 5 read path. + +### 2026-07-01 Wave 4B3 Rolling Decision Metrics Update + +Wave 4B3 keeps the latest shadow validation state in the settings container and adds a bounded rolling metric history to the same document. Admin Settings now shows 5-minute and 15-minute aggregate source-versus-candidate RU totals, estimated Wave 5 savings, validation overhead, and sample counts. These dashboard values are intended to help admins compare the current source-container document-list cost with the future access-index read path over a workflow window instead of only looking at the most recent validation call. The same aggregate structure will also help evaluate Wave 6 Redis document access caching, where a short TTL such as 15 minutes may make the access-index path beneficial for more deployments. + +### 2026-07-02 Wave 5A Read-Switch Canary Update + +Wave 5 is split into **5A canary** and **5B broad enablement**. Wave 5A implements `document_access_index`-backed list reads for personal, group, and public document-list routes behind `enable_document_access_index_reads`. This includes personal workspace lists, group workspace lists, internal public workspace lists, external public workspace lists, and chat document pickers that load personal, group, or visible public workspace documents. The switch stays off by default, is exposed in Admin Settings as a canary control, and only serves from the access index when the container is enabled, write-through is enabled, the relevant backfill scope is `succeeded`, and repair backlog count is zero. If the access-index query is unavailable or readiness checks fail, routes fall back to the existing source-container read path. When shadow validation remains enabled during canary testing, routes still run the source query for validation; admins should disable shadow validation when measuring the overhead-eliminated read path. + +Wave 5A uses the candidate-read shape established in Wave 4B2: one single-partition query per access `scope_key`, no Cosmos-side `ORDER BY`, `OFFSET`, `LIMIT`, or `TOP`, and projection-only fields needed by the list UI. Source containers remain authoritative, and DAI rows are shaped into source-like list documents before existing sort, paging, and response enrichment run. The Wave 5A projection schema is versioned, and reads require a matching backfill state schema version so older Wave 4 projection rows cannot be served after the read path is enabled. Existing deployments now run document access repair and backfill automatically during app maintenance after upgrading; manual batches remain available for support and troubleshooting. + +### 2026-07-02 Wave 5A2 Tag-List Read Optimization Update + +Wave 5A2 extends the same DAI read pattern to the remaining hot tag-list endpoints: personal tags, group tags, and public workspace tags. These endpoints now attempt a DAI tag-count read first when the readiness gates pass, count projected current owner-scope `tags` rows with one single-partition query per access `scope_key`, then merge existing tag definitions and safe colors from settings/group/public workspace records. Source-backed `get_workspace_tags(...)` remains the fallback whenever backfill is not ready, repair backlog is present, or the projection query fails. + +### 2026-07-03 Wave 5A2 Operations Dashboard and Auto-Maintenance Update + +Wave 5A2 hardens the DAI operations posture before broad enablement. The DAI container, write-through projection, and automatic repair/backfill maintenance are now treated as always-on requirements, including for deployments with older saved settings that previously disabled them. App maintenance repairs fail-open projection records first, runs one bounded backfill batch, and uses a short active maintenance interval while repair or backfill work remains so existing deployments transparently backfill and converge after upgrade. The Admin Settings DAI card now emphasizes operational health: automatic maintenance state, next action, active loop interval, repair backlog, production DAI read attempts, DAI-served reads, source fallbacks, fallback rate, DAI read RU, DAI latency, and last fallback reason. Shadow validation metrics remain available as optional parity diagnostics, but they are no longer the primary dashboard signal for DAI read health. + +### 2026-07-03 Wave 5A3 Redis Monitoring Baseline Update + +Wave 5A3 adds Redis monitoring before Wave 6 introduces Redis-backed DAI document-list caching. Admin Settings > Scale now surfaces sanitized Redis runtime and capacity signals from the active Redis client: configuration state, health, app-cache and session runtime usage, monitoring source, ping latency, Redis version, connected clients, memory usage, maxmemory policy, fragmentation ratio, ops/sec, keyspace hit rate, tracked keys, expired keys, evicted keys, error replies, rejected connections, last checked time, and last error summary. The monitoring endpoint does not return Redis keys, secret names, or host names; it reports whether Redis is disabled, missing configuration, unavailable at runtime, degraded, healthy, or errored so admins can establish a baseline before document-list caching changes traffic patterns. + +### 2026-07-03 Wave 5B Broad Default Read Enablement Update + +Wave 5B promotes DAI-backed document and tag list reads from canary to the default read path. App settings now create and migrate `enable_document_access_index_reads` to `True`, Admin Settings displays the read path as an always-on Wave 5B default, and the settings save path preserves that required state even if an older deployment had stored the prior canary value as disabled. The existing safety gates remain unchanged: DAI rows are used only when write-through is enabled, schema-v2 backfill has succeeded for the requested source scope, repair backlog is clear, and the DAI query succeeds. Otherwise, each optimized route automatically falls back to the source document containers and records source fallback metrics for the operations dashboard. Shadow validation remains optional and should stay disabled when measuring overhead-eliminated production reads. + +### 2026-07-04 Wave 6 Redis Document Access Cache Update + +Wave 6 adds Redis-only read-through caching on top of DAI document, tag, and legacy-count reads. The cache never writes Cosmos cache documents and never forces source-container fallback when Redis is unavailable; Redis misses, unavailable clients, or Redis errors simply bypass cache and run the DAI query directly. The default TTL is **900 seconds** with a supported range of **60-900 seconds**. Cache keys include the operation, source scope, schema version, normalized filters/access role, and per-scope Redis version tokens. DAI projection sync, delete, repair, and backfill paths bump affected scope-version tokens for personal, group, and public access scopes so document uploads, deletes, metadata updates, share/access changes, tag changes, and projection repairs make new list reads miss old cache entries immediately. Old versioned entries naturally expire by TTL. + +Admin Settings > Scale displays Redis DAI cache status and lightweight in-process cache metrics: cache enabled/TTL, 15-minute cache hit rate, hits/misses, bypasses/errors, invalidations, and last cache event. These metrics complement the Wave 5 production DAI read metrics and Wave 5A3 Redis runtime monitoring so admins can compare Redis cache hit behavior, Redis health, and DAI/source fallback rates before and after enabling broader Redis-dependent performance work. + +### 2026-07-06 Wave 6 Admin Metrics UI Cleanup + +DAI is now treated as an always-on production read path in the Admin Settings experience. The default Redis DAI cache TTL is **900 seconds**, relying on scope-version invalidation for immediate visibility after document, tag, share, repair, delete, and backfill changes. The Scale left navigation now links directly to **DAI Metrics**, **Cosmos Metrics**, and **Redis Metrics** so admins can jump to the relevant operational dashboard. + +Support-only controls and diagnostics are hidden by default behind the app setting `enable_dai_debug`, which defaults to `false` and is not exposed in the Admin Settings UI. When that setting is edited directly in Cosmos and set to `true`, the DAI card renders manual backfill/reset controls, DAI read/cache toggles, batch-size fields, shadow validation controls, and rolling shadow-decision diagnostics for troubleshooting. Default admins see only production DAI health, maintenance, cache, fallback, RU, latency, repair, and backfill metrics. + +### 2026-07-06 Phase 3 Low-Churn Cache Hardening + +Phase 3 completes and hardens the existing shared low-churn cache foundation for custom pages/navigation and chat bootstrap data. Shared cache helpers now record safe hit, miss, write, delete, and version-bump counters using hashed cache-key context, and app maintenance status includes these shared cache metrics alongside cache version document status. + +Custom pages cache invalidation now runs after app settings writes so feature enablement, menu configuration, and TTL changes do not depend on old cache entries expiring. Chat bootstrap cache invalidation coverage now includes group creation/deletion/status/member/role/ownership/model-endpoint changes and public workspace creation/deletion/status/member/role/ownership changes across both normal workspace routes, Control Center admin routes, and SimpleChat-native operations used by internal tools. Existing Redis-first/Cosmos-fallback shared cache behavior remains unchanged, and process memory remains a diagnostic/near-cache layer rather than the authoritative invalidation mechanism. + +### 2026-07-06 Phase 4 Conversation Cache Hardening + +Phase 4 completes and hardens the existing conversation list, feed, and advanced-search cache path. Conversation cache remains user-scoped and versioned, but version tokens and volatile payloads are Redis-only as of version `0.250.037`. When an app-cache Redis client is unavailable, routes bypass cache reads/writes and run the existing Cosmos source queries directly; when conversation caching is explicitly disabled, routes also bypass cache reads/writes and run the source path directly. + +The default app setting `enable_conversation_cache` is `true`, with `conversation_cache_ttl_seconds` controlling cache entry lifetime. Cache keys include user id, operation, user cache version, normalized request parameters, and group-access fingerprints for collaboration-visible feed/search results. Mutation coverage includes conversation create, title update, delete, bulk delete, pin/hide changes, mark-read, metadata/summary/context updates, scope-lock changes, chat completion/title initialization, collaboration participant/message/title/member changes, and notification read/dismiss actions. + +Version `0.250.034` adds DAI-style conversation cache metrics to Admin Settings > Scale. Reads, writes, disabled/TTL bypasses, and version invalidations emit lightweight in-process rolling samples over 5-minute, 15-minute, and 60-minute windows. The admin dashboard displays runtime state, 15-minute hit rate, hits/misses, bypasses/errors, writes/invalidations, operation mix, last cache event, and last invalidation without adding Cosmos reads to the conversation hot path. + +Version `0.250.035` tunes the mark-read flow exposed by those metrics. Normal conversation navigation no longer forces a mark-read request when the client has no unread state, and the backend mark-read endpoint only upserts the conversation and bumps the conversation cache version when unread fields actually changed. Notification clearing remains available, but already-read conversations no longer churn the conversation feed cache simply because a user switched conversations or reloaded the page. + +Version `0.250.036` preserves background chat-completion unread state after the mark-read tuning. Streaming finalization now only force-clears unread state when the completed conversation is still the active conversation, and it no longer switches the active conversation back to a completed background stream. This keeps the "chat finished while away" notification and green unread dot until the user opens that conversation. + +### 2026-07-06 Settings Container RU Suppression and Stale Cleanup Planning + +Version `0.250.037` removes the unexpected idle `settings` container RU pattern discovered during Phase 4 validation. Generic app settings writes no longer bump unrelated chat-bootstrap and custom-pages version documents, Cosmos throughput status refresh is read-only, no-op background autoscale checks no longer persist runtime status, volatile chat-bootstrap/conversation cache payloads do not fall back to Cosmos when Redis is unavailable, and DAI status uses short-lived in-process state caching while skipping disabled shadow-validation state reads. Post-deploy monitoring confirmed that `settings` dropped out of the top normalized RU consumers during idle monitoring. + +The remaining obsolete `settings` documents should be cleaned up through app maintenance rather than by ad hoc deletion. Cleanup must be allowlisted, idempotent, dry-run capable, and limited to stale operational cache artifacts that the new code no longer uses, such as old `conversation_cache_version:*` documents and volatile `shared_cache_entry:conversation_cache:*` entries. Cleanup must not delete `app_settings`, active cache-version docs that still coordinate low-churn caches, app-maintenance state, DAI state, repair/backfill state, or any source-of-truth configuration document. + +### 2026-07-06 Phase 5 Cosmos Maintenance Implementation + +Version `0.250.038` implements the Phase 5 maintenance surface. Expected Cosmos indexing policies remain centralized in `functions_cosmos_indexing.py`, app maintenance can compare or apply missing composite indexes non-destructively, and Admin Settings > Scale now displays indexing status, mode, missing expected indexes, updated containers, failures, and last evaluation time. + +The same maintenance surface now includes stale operational cache cleanup. Cleanup is logically separate from indexing updates, is skipped by default for background runs, and can be triggered from Admin Settings in dry-run or apply mode. Apply mode deletes only allowlisted stale cache artifacts from the settings container: retired `conversation_cache_version:*` documents, obsolete Redis-only `shared_cache_entry:conversation_cache:*` and `shared_cache_entry:chat_bootstrap:*` payloads, and expired shared cache entries. It reports candidate, deleted, skipped, failed, category, and more-candidate counts without touching app settings, active cache-version documents, DAI state, app maintenance state, source documents, or user data. + +### 2026-07-06 Phase 9 Hardening and Admin Index Apply + +Version `0.250.039` completes the Phase 9 runbook pass and exposes a guarded Admin Settings action for applying missing Cosmos composite indexes. Admin Settings > Scale > Cosmos Maintenance now keeps the default status view read-only, then requires an explicit confirmation modal before posting `apply_cosmos_indexing_policies=true` to the existing app-maintenance run endpoint. The modal clarifies that the update is additive and non-destructive, preserves existing indexing policy paths, can improve supported lookup and ordered-query speed, and may add write-index overhead plus asynchronous Cosmos index transformation work. + +Phase 9 also documents support runbooks for rebuilding caches, rebuilding `document_access_index`, cleaning stale operational cache documents, interpreting shadow validation diffs, interpreting DAI fallback/cache metrics, and deciding whether broad source fallback should remain after DAI/cache telemetry is stable. + ## Executive Summary The current application already improved app settings and user UI settings access by introducing request-scoped caching, Redis-backed caching, Cosmos-coordinated worker-local near-caching, shared version checks, and targeted invalidation. This proposal extends that pattern to other high-utilization areas of the application: @@ -364,7 +461,7 @@ Conversation search also loads candidate conversations and runs a cross-partitio ### Proposed Cache Strategy -Use per-user versioned cache keys. Redis is preferred, but the same key model can be stored in Cosmos cache documents when Redis is unavailable: +Use per-user versioned cache keys. Redis is the runtime backend for conversation cache version tokens and volatile payloads; when Redis is unavailable, the application bypasses cache and uses source Cosmos queries: ```text conversation_cache_version:{user_id} = 42 @@ -373,9 +470,9 @@ conversation_search:{user_id}:v42:{filters_hash} conversation_classifications:{user_id}:v42 ``` -When conversation state changes, increment `conversation_cache_version:{user_id}` in the shared backend. Old keys become stale immediately because reads use the latest version. Worker memory can keep a short-lived near-cache of the latest version and payload, but it must re-check the shared version before trusting local data beyond the local version-read TTL. +When conversation state changes, increment `conversation_cache_version:{user_id}` in Redis. Old keys become stale immediately because reads use the latest version. If Redis is unavailable, no Cosmos cache-version fallback is used; cache key construction returns no key and the request falls through to the source query behavior. -Example Cosmos-backed conversation cache entry: +Retired Cosmos-backed conversation cache entry shape cleaned up by Phase 5 maintenance: ```json { @@ -734,6 +831,12 @@ Parameters: ] ``` +Pagination implementation note: + +- Prefer SDK continuation tokens for high-volume or deep paging. Continuation tokens avoid repeatedly scanning and discarding skipped results. +- If the current UI must preserve page-number semantics, use `OFFSET LIMIT` only for shallow pages and capture RU/latency separately for deeper pages. +- Keep `TOP` values as literal integers if `TOP` is used in Cosmos SQL; do not parameterize `TOP`. + Count current documents for a group: ```sql @@ -818,6 +921,40 @@ Option B, store projection entries on the source document: Recommendation: start with deterministic row ids and add `document_access_index_entries` only if cleanup becomes too scattered. +Recommended deterministic row id shape: + +```text +{scope_key}:source:{source_container}:doc:{document_id} +``` + +The implementation should generate row ids from normalized projection inputs, not from display names or mutable metadata. `scope_key`, `source_container`, and `document_id` are enough for one current list row per source document per access scope. If future requirements need multiple rows per scope and document, add a stable suffix such as `:access:{access_type}`. + +### Shared Access Normalization + +The current source document arrays can contain simple ids and comma-suffixed approval-state values. The projection must normalize those source values before writing index rows: + +```json +{ + "scope_key": "user:99115fd2-1234-45a8-9184-000000000020", + "scope_type": "user", + "scope_id": "99115fd2-1234-45a8-9184-000000000020", + "access_type": "shared_user", + "share_status": "approved" +} +``` + +Projection helpers should centralize parsing so list queries, cleanup, backfill, and write-through paths all interpret sharing state the same way. This also avoids carrying source-array delimiter behavior into new query code. + +### Version Family Semantics + +The `document_access_index` should represent listable current documents, not every historical revision: + +- Only the current source document in a revision family should have active index rows. +- New version creation should update existing rows to the new current document/version metadata and ensure the old version no longer appears in list results. +- Deleting a non-current revision should not change index rows unless it changes current-version resolution. +- If deleting the current revision promotes another revision, the promotion step must update index rows in the same maintenance/write-through flow. +- Direct document open and download must continue to validate access against source-of-truth documents, not only against the index row. + ### Consistency Model The source containers remain authoritative. The Document Access Index is eventually consistent within the application write flow. @@ -825,9 +962,11 @@ The source containers remain authoritative. The Document Access Index is eventua For user-facing behavior: - Normal writes should update source and Document Access Index rows together. -- If index update fails, log the issue and enqueue or mark for maintenance repair. +- If a permission-grant projection fails, the operation may proceed only if a repair marker is recorded and the affected scope falls back to the source query until repaired. +- If a permission-reducing projection fails, such as unshare, delete, archive, or retention delete, the affected scope must be marked unsafe for index reads until repair completes. - The admin maintenance job can rebuild the Document Access Index from source documents. - Direct document open should still validate access against source-of-truth data if there is any doubt. +- Azure AI Search chunk visibility and sharing fields must continue to be updated by existing document workflows. The access index is a Cosmos list read model, not a replacement for search authorization metadata. ### Expected RU Reduction @@ -938,6 +1077,17 @@ The buttons should start jobs and return immediately. The UI should poll job sta }, "processed": 0, "errors": [] + }, + "cleanup_stale_cache_docs": { + "status": "pending", + "dry_run": true, + "candidate_count": 0, + "deleted_count": 0, + "skipped_count": 0, + "checkpoint": { + "continuation_token": null + }, + "errors": [] } }, "summary": { @@ -946,7 +1096,9 @@ The buttons should start jobs and return immediately. The UI should poll job sta "indexing_policies_submitted": 2, "access_index_rows_upserted": 0, "access_index_rows_deleted": 0, - "cache_versions_initialized": 4 + "cache_versions_initialized": 4, + "stale_cache_docs_deleted": 0, + "stale_cache_docs_skipped": 0 } } ``` @@ -1005,7 +1157,29 @@ document_access_index_projection_version This should be optional because conversation cache can lazy warm. Manual rebuild may be useful after cache flushes or support events. -#### Step 6: Backfill Document Access Index Container +#### Step 6: Clean Stale Operational Cache Documents + +This step removes obsolete cache artifacts from the `settings` container after the code version that stopped using them is confirmed live. + +- Run in dry-run mode by default and report candidate counts by document prefix/type. +- Delete only allowlisted stale artifacts: + - `conversation_cache_version:*` + - `shared_cache_entry:conversation_cache:*` + - expired volatile shared cache entries whose namespace is no longer Cosmos-backed. +- Preserve source-of-truth and active coordination docs: + - `app_settings` + - `app_settings_cache_version` + - `governance_cache_version` + - `custom_pages_cache_version` + - `chat_bootstrap_global_cache_version` + - `document_access_index_projection_version` + - `app_maintenance_state` + - DAI backfill, repair backlog, and shadow-validation state docs + - background-task lock docs +- Process deletes in bounded batches and record deleted counts, skipped counts, last continuation token, and failures. +- Surface cleanup status and failures in Admin Settings maintenance status. + +#### Step 7: Backfill Document Access Index Container For each source document container: @@ -1031,6 +1205,7 @@ POST /api/admin/maintenance/run-app-maintenance POST /api/admin/maintenance/rebuild-document-access-index POST /api/admin/maintenance/rebuild-custom-pages-cache POST /api/admin/maintenance/clear-app-caches +POST /api/admin/maintenance/cleanup-stale-cache-docs ``` All endpoints must use: @@ -1072,21 +1247,20 @@ Recommended rollout settings: ```python { - "enable_document_access_index_container": False, - "enable_document_access_index_reads": False, + "enable_document_access_index_container": True, + "enable_document_access_index_reads": True, "enable_document_access_index_write_through": True, - "enable_startup_document_access_index_backfill": False + "enable_startup_document_access_index_backfill": True } ``` Suggested phased rollout: -1. Deploy container, maintenance jobs, cache settings, and write-through projection disabled for reads. -2. Run backfill manually or through startup maintenance. -3. Enable write-through projection. -4. Compare source list results and Document Access Index list results in shadow mode. -5. Enable Document Access Index reads for admin/test users. -6. Enable Document Access Index reads globally. +1. Deploy container, maintenance jobs, cache settings, write-through projection, automatic repair/backfill, and default DAI reads together. +2. Allow app maintenance to backfill and repair automatically; use manual batches only for support-driven retries. +3. Let DAI read attempts fall back to source containers until schema-v2 backfill is complete and repairs are clear. +4. Enable shadow validation only when parity diagnostics are needed; disable it again when measuring production read savings. +5. Monitor DAI-served reads, source fallbacks, fallback rate, RU, latency, and repair/backfill state during and after rollout. ## Functional Test Plan @@ -1114,19 +1288,33 @@ Recommended tests: - Verify distributed lock prevents duplicate runs. - Verify status doc updates. - Verify failed step records error and job remains inspectable. + - Verify stale cache cleanup dry-run reports candidates without deleting. + - Verify stale cache cleanup apply mode deletes only allowlisted obsolete cache docs. + - Verify stale cache cleanup preserves app settings, active version docs, DAI state docs, and background-task locks. 5. **Document Access Index Projection Test** - Verify create document writes owner index row. - Verify new version updates existing index row rather than creating duplicate current rows. - Verify share with user creates user index row. + - Verify comma-suffixed or approval-state share values are normalized into `scope_id` and `share_status`. - Verify share with group creates group index row, not one row per member. - Verify unshare deletes corresponding index row. + - Verify permission-reducing changes mark affected scopes unsafe for index reads if projection cleanup fails. - Verify document delete removes all index rows. + - Verify source document search visibility updates continue to propagate to Azure AI Search chunks. 6. **Document Access Index Query Equivalence Test** - Build source-container result set and Document Access Index result set for a fixture user. - Verify visible document ids match. - Verify sorting, filtering, and pagination are equivalent. + - Verify multi-scope merges deduplicate documents visible through more than one scope. + - Verify stale or missing projection rows trigger source-query fallback while feature flags are not fully enabled. + +7. **Version Family Projection Test** + - Verify only the current revision appears in `document_access_index`. + - Verify new version creation updates index metadata to the new current version. + - Verify deleting a non-current revision leaves index rows unchanged. + - Verify deleting or archiving a current revision updates or removes index rows safely. ## Performance Validation Plan @@ -1141,6 +1329,9 @@ Before and after implementation, collect: - Maintenance job duration and throughput. - Document Access Index backfill throughput. - Query latency p50, p95, and p99 where available. +- Cosmos query metrics and index metrics for representative Python SDK queries by setting `populate_query_metrics=True` and `populate_index_metrics=True` during diagnostic runs. +- Continuation-token versus `OFFSET LIMIT` RU and latency for document list paging, especially beyond the first few pages. +- Source-query versus access-index equivalence counts and diff summaries during shadow mode. Add structured App Insights events for: @@ -1171,50 +1362,291 @@ document_access_index_projection_repair_needed ## Recommended Implementation Phases -### Phase 1: Cache Configuration and Maintenance Framework +### Phase 0: Baseline Metrics and Design Lock + +This phase should happen before code behavior changes. + +1. Capture current RU charge, query metrics, index metrics, latency, result count, and payload size for: + - Personal document lists. + - Group document lists. + - Public workspace document lists. + - Conversation list and conversation search. + - Custom page navigation injection. + - Chat bootstrap loaders. +2. Finalize `document_access_index` projection rules: + - Deterministic row id format. + - Shared user/group parsing and approval-state normalization. + - Current-version and revision-family semantics. + - Permission-reducing failure behavior. + - Source-query fallback criteria. +3. Build representative fixtures for owner, shared user, shared group, public workspace, multi-version, archived, deleted, and overlapping multi-scope visibility. + +Exit criteria: + +- Baseline metrics are recorded. +- Projection rules are documented. +- Functional test fixture expectations are agreed. + +### Phase 1: Maintenance Framework and Admin Controls + +1. Reuse `background_tasks.py` distributed lock patterns for app maintenance jobs. +2. Store low-volume job state in the existing settings container, using `type` fields and stable ids. +3. Add a dedicated maintenance job/log container only if job history or logs become high-volume. +4. Add startup maintenance loop with: + - Idempotent step execution. + - Checkpoints. + - Lease protection. + - Safe no-op behavior when disabled. +5. Add admin-only manual maintenance endpoints and UI controls: + - Job status. + - Run selected job. + - Cancel or mark job cancelled where safe. + - Clear/rebuild caches. + - Validate containers and indexing policies. +6. Add route-policy and admin functional tests for new endpoints. + +Exit criteria: + +- Jobs can be started manually and by startup loop without duplicate execution across workers. +- Job state is visible to admins. +- Failed steps are inspectable and resumable. + +### Phase 2: Shared Cache Foundation + +1. Extract or extend existing app settings/governance/search cache patterns into shared helpers for: + - Redis-first cache access. + - Cosmos-backed cache documents when Redis is unavailable. + - Versioned cache keys. + - Short-lived worker-local near-cache after shared version checks. + - Cache hit/miss/invalidation telemetry. +2. Add advanced admin settings for cache enablement, TTLs, version-read TTL, and backend preference. +3. Add manual cache clear and version bump helpers. + +Exit criteria: + +- New cache consumers can use one common versioned cache contract. +- Worker-local memory is never the authoritative invalidation layer. + +### Phase 3: Low-Churn Cache Wins + +Implemented in version: **0.250.032** + +Implement the safest cache wins before higher-risk read-model changes. + +1. Custom pages cache: + - Cache catalog and role-aware navigation. + - Invalidate on custom page create/update/delete and manual cache clear. + - Rebuild via maintenance job. +2. Chat bootstrap cache: + - Cache scoped fragments for user, group, and global bootstrap data. + - Add invalidation hooks for agents, actions, prompts, groups, public workspaces, governance policy changes, and model endpoint changes. +3. Add functional tests and App Insights events for cache hits, misses, and invalidations. + +Exit criteria: + +- Repeated page loads avoid repeated Cosmos reads for unchanged low-churn data. +- All write paths that change cached data bump the correct shared version. + +### Phase 4: Conversation List and Search Cache + +Implemented in version: **0.250.033**; metrics dashboard updated in **0.250.034**; mark-read invalidation tuned in **0.250.035**; background unread-state preservation updated in **0.250.036** + +1. Add user-scoped conversation cache version docs or Redis version keys. +2. Add lazy warm for `/api/get_conversations`. +3. Cache normalized conversation search result signatures with query/filter hashes. +4. Invalidate on: + - Create. + - Rename/title update. + - Delete or bulk delete. + - Pin/unpin. + - Hide/unhide. + - Mark read/unread. + - Scope lock changes. + - Metadata, summary, classification, tag, or context updates. +5. Add manual rebuild/clear controls. +6. Add operational dashboard metrics for runtime status, hit rate, hits/misses, bypasses/errors, writes/invalidations, operation mix, and last invalidation. +7. Keep mark-read invalidation idempotent so normal navigation does not invalidate the feed cache when no unread state changed. + +Exit criteria: + +- Cache misses still use source queries safely. +- Cache invalidation is versioned and cross-worker safe. +- Conversation list/search RU reductions are measurable against Phase 0 baseline. + +### Phase 5: Cosmos Indexing Policy Maintenance + +Implemented in version: **0.250.038**; guarded admin apply action added in **0.250.039** + +1. Define expected indexing policies in one central module. +2. Compare current container policies to expected policies. +3. Apply non-destructive updates through maintenance jobs. +4. Record index transformation submission and current status where available. +5. Validate high-use source-container query metrics before and after policy updates. +6. Surface indexing-policy comparison, transformation progress, skipped containers, and failures in Admin Settings maintenance status. +7. Include the stale operational cache-document cleanup task in the same maintenance framework, but keep it logically separate from indexing-policy updates so a cleanup failure cannot block policy validation. + +Exit criteria: + +- Indexing policy changes can be re-run safely. +- Admins can see whether policy validation or update failed. +- Admins can explicitly apply missing composite indexes from the UI after acknowledging write-index overhead and asynchronous transformation behavior. +- Source-query RU improves or remains stable while companion-container work proceeds. +- Stale cache cleanup can run in dry-run and apply modes, reports exact candidate/deleted/skipped counts, and never deletes source-of-truth or active coordination documents. + +### Phase 6: Document Access Index Container and Write-Through + +1. Add `document_access_index` container partitioned by `/scope_key`. +2. Add feature flags: + - `enable_document_access_index_container`. + - `enable_document_access_index_write_through`. + - `enable_document_access_index_reads`. + - `enable_document_access_index_shadow_validation`. + - `enable_startup_document_access_index_backfill`. +3. Add projection builder helpers that normalize source documents into deterministic rows. +4. Add write-through hooks in source document workflows: + - Create. + - Metadata update. + - Processing status update. + - Share/unshare. + - Share approval/denial. + - New version/current-version changes. + - Archive/unarchive. + - Delete and retention delete. +5. Keep Azure AI Search visibility updates in the same source workflows. + +Exit criteria: + +- New writes produce expected projection rows while reads remain on source paths. +- Projection failure creates repair/fallback state instead of silent drift. + +### Phase 7: Backfill, Reconciliation, and Shadow Validation + +1. Add resumable, throttled backfill job with checkpoints per source container. +2. Add reconciliation job that detects: + - Missing rows. + - Orphan rows. + - Incorrect current-version rows. + - Incorrect sharing status rows. + - Unsafe scopes requiring source fallback. +3. Add shadow validation for document list endpoints: + - Query source path and access-index path. + - Compare ids, sort order, filter behavior, pagination, counts, and deduplication. + - Log diff summaries and RU deltas. +4. Keep source fallback active until shadow validation is clean and DAI maintenance is healthy. + +Exit criteria: + +- Backfill has completed or reached an accepted scope. +- Shadow validation shows equivalent results for representative users, groups, public workspaces, and edge-case fixtures. +- Repair jobs can fix detected drift. + +### Phase 8: Controlled Read Switchover + +1. Add read wrapper that chooses source query or access-index query based on feature flags, scope safety, and shadow validation state. +2. Enable access-index reads in this order: + - Admin/test users. + - Small internal cohort. + - Personal document list. + - Group document list. + - Public workspace document list. + - Global rollout. +3. Keep direct document open/download authorization against source-of-truth data. +4. Monitor RU, latency, error rate, stale fallback count, repair count, and user-reported discrepancies. + +Exit criteria: + +- Document list RU and latency improve against baseline. +- No unresolved projection drift or authorization regressions are observed. +- Source fallback remains available until confidence is high enough to remove or narrow it. + +### Phase 9: Hardening, Documentation, and Release Readiness + +Implemented in version: **0.250.039** + +1. Add or update functional tests for caches, maintenance jobs, indexing policy validation, projection write-through, backfill, reconciliation, shadow validation, and read switchover. +2. Add fix/feature documentation with the implementation version. +3. Update release notes if approved. +4. Add support runbook content for: + - Rebuilding caches. + - Rebuilding `document_access_index`. + - Cleaning stale operational cache documents from `settings`. + - Interpreting shadow validation diffs. + - Interpreting DAI source fallback metrics while repair/backfill catches up. + - Interpreting DAI Redis cache metrics and Redis-unavailable bypass behavior. +5. Review whether source fallback can remain as a broad safety path or should be limited to admin repair scenarios after DAI, Redis cache, and maintenance telemetry are stable. +6. Complete final docs and release readiness checks, including maintenance runbooks, support-safe cleanup guidance, and validation queries for idle RU baselines. + +#### Phase 9 Support Runbook + +Use Admin Settings > Scale as the first operational surface. The dashboard is intentionally split into DAI Metrics, Cosmos Maintenance, Cosmos Metrics, Redis Metrics, and Conversation Cache so support can isolate the subsystem before changing settings. + +##### Rebuilding low-churn and conversation caches + +1. Confirm Redis health in Redis Metrics. If Redis is unavailable, Redis-only cache paths bypass cache and use the safe source or DAI query path rather than writing volatile cache payloads to Cosmos. +2. For custom pages/navigation and chat bootstrap cache issues, make the source-of-truth change again or save the relevant app/custom-page/workspace setting to bump the low-churn cache version. +3. For conversation list/feed/search cache issues, use the normal user-visible mutation path that changes the affected conversation, such as title update, pin/hide, delete, mark-read when unread state actually changed, or metadata update. These paths bump Redis-only conversation version tokens. +4. Do not recreate retired `conversation_cache_version:*` or volatile shared-cache payload documents in `settings`. If old documents remain, use the stale cleanup dry run and apply action instead. + +##### Rebuilding `document_access_index` + +1. Check Cosmos Document Access Index status in Admin Settings > Scale. +2. Confirm DAI container, write-through, automatic maintenance, repair backlog, backfill state, and last error. +3. If DAI debug mode is enabled through the hidden `enable_dai_debug` setting, run one manual backfill batch or reset the checkpoint only when support needs to reprocess all source scopes. +4. Leave production source fallback enabled while repair or backfill is incomplete. Routes only serve DAI rows when readiness gates pass and the projection query succeeds. +5. After repair backlog clears and backfill state is succeeded for the relevant scopes, compare 15-minute DAI served reads, fallback count, fallback rate, DAI RU, and DAI latency. + +##### Applying expected Cosmos composite indexes + +1. Open Cosmos Maintenance and refresh status. +2. Review Missing Expected Indexes, Indexing Failures, and Last Indexing Evaluation. +3. Choose **Apply Missing Indexes** only when the team accepts the tradeoff: additional composite indexes improve supported lookup/ordered-query patterns but add write-index maintenance overhead and may start asynchronous index transformation. +4. Confirm the modal. This posts an apply-mode app-maintenance run. It only appends missing expected composite indexes and preserves current included paths, excluded paths, default indexes, TTL, conflict-resolution, analytical-storage TTL, and full-text policy settings. +5. Refresh status later to monitor updated container count, failures, and `indexTransformationProgress` where Cosmos returns it. + +##### Cleaning stale operational cache documents + +1. Run **Dry Run Cleanup** first. +2. Review candidate count, categories, failed count, and whether more candidates remain. +3. Use **Delete Stale Cache Docs** only after dry-run counts look reasonable. +4. Apply mode deletes one bounded batch of allowlisted operational artifacts only: retired conversation cache versions, obsolete Redis-only conversation/chat-bootstrap shared-cache payloads, and expired shared-cache entries. +5. Repeat dry-run/apply if `More Candidates` remains `Yes`. -- Add advanced cache settings defaults. -- Add admin UI fields for performance/cache TTLs. -- Add maintenance job container or job docs in settings container. -- Add background maintenance loop with distributed lock. -- Add manual admin maintenance endpoints and UI. -- Add job status polling. +##### Interpreting shadow validation diffs -### Phase 2: Low-Churn Cache Wins +Shadow validation is a diagnostic, not the normal production read path. If enabled for troubleshooting: -- Implement custom pages static cache. -- Implement chat bootstrap scoped cache fragments. -- Add invalidation to agent/action/prompt/group/public workspace/governance write paths. -- Add cache telemetry. +- `missing_count` means source documents expected by the current source path were not found in DAI projection rows. +- `extra_count` means DAI returned rows that the source path did not return. +- Source/index RU and latency compare the cost of the current source query and the validation/candidate DAI query. +- Rolling 5-minute and 15-minute samples are useful for workflow-level comparison but should not be mixed with Redis cache hit-rate conclusions. +- Disable shadow validation before measuring production DAI/cache RU because shadow mode intentionally adds validation work. -### Phase 3: Conversation Cache +##### Interpreting DAI fallback and Redis cache metrics + +- `DAI Read Attempts` counts optimized read attempts. +- `DAI Served Reads` means the access index satisfied the read. +- `Source Fallbacks` and `Fallback Rate` show how often routes used the source containers because readiness, repair, backfill, cache safety, or query execution did not allow DAI service. +- `Last Fallback Reason` is the fastest way to decide whether to wait for backfill/repair, investigate projection errors, or inspect Redis/cache safety. +- Redis DAI cache misses and bypasses are not failures by themselves. A Redis miss should run the DAI query; Redis unavailable bypass should not write volatile cache documents to Cosmos. +- A healthy deployment should trend toward low source fallback once backfill and repair converge. A high fallback rate with clear repair backlog is expected during rollout; a high fallback rate after convergence needs investigation. -- Add conversation cache helper. -- Add lazy warm to `/api/get_conversations`. -- Add version bump to conversation CRUD and metadata update paths. -- Add conversation search result cache with query hash. -- Add manual rebuild and clear controls. +##### Source fallback readiness decision -### Phase 4: Cosmos Indexing Policy Maintenance +Keep broad source fallback while any of the following are true: -- Define expected indexing policies for high-use containers. -- Add maintenance validation and apply step. -- Expose status in admin maintenance UI. +- DAI repair backlog is non-zero or unknown. +- Backfill is incomplete for active source scopes. +- Shadow validation is reporting unresolved missing or extra rows. +- Production fallback reasons show projection, readiness, Redis invalidation, or query errors. +- Support has not yet validated idle RU, 15-minute DAI served reads, cache hit rate, and direct source authorization behavior after the latest deployment. -### Phase 5: Document Access Index Companion Container - -- Add `document_access_index` container. -- Add projection builder helper. -- Add write-through updates for create/update/share/unshare/delete/version changes. -- Add backfill maintenance job. -- Add shadow comparison mode. -- Switch document list endpoints to Document Access Index reads behind feature flag. +Consider narrowing fallback only after DAI readiness remains stable across representative personal, group, public, and chat document-picker workflows; Redis cache bypass behavior is understood; and support has a tested rebuild path for projection drift. ## Open Decisions 1. Should maintenance jobs use a new `maintenance_jobs` container or store job docs in the existing `settings` container? - - New container is cleaner for querying job history. - - Existing settings container avoids adding one more container. + - Recommended: start with low-volume job state in the existing `settings` container to reuse current lock/version patterns. + - Add a dedicated `maintenance_jobs` or `maintenance_job_logs` container only if detailed job history becomes high-volume or needs independent retention. 2. Should Document Access Index projection be enabled for writes before reads? - Recommended: yes. This allows backfill and shadow validation before user-facing read switch. @@ -1228,6 +1660,15 @@ document_access_index_projection_repair_needed 5. Should custom pages cache use Redis by default when Redis is enabled? - Recommended: yes. When Redis is unavailable, use Cosmos-backed cache documents. Worker-local memory may only act as a short-lived near-cache after checking a shared version value. +6. Should document list paging use continuation tokens or page-number offsets? + - Recommended: use continuation tokens for high-volume list APIs and keep page-number offsets only where the current UI requires shallow random access. + +7. Should permission-reducing projection failures fail the source write? + - Recommended: avoid silent success. Either complete the source, search visibility, and projection updates together, or mark the affected scope unsafe for access-index reads and enqueue repair before returning success. + +8. Should the Document Access Index replace source authorization checks? + - Recommended: no. It is a list read model only. Direct open/download and sensitive operations must validate against source documents. + ## Approval Request Approval is requested to proceed with a phased implementation of: @@ -1240,4 +1681,4 @@ Approval is requested to proceed with a phased implementation of: 6. Cosmos indexing policy validation and update workflow. 7. Companion `document_access_index` container for fast document list, count, filter, and pagination workloads. -The companion container is the largest change, but it directly addresses the biggest Cosmos RU issue identified in document listing: list queries are currently shaped by user/group/workspace access patterns while the source containers are partitioned by document id. The companion container aligns partitioning with the UI read pattern while preserving the existing source-of-truth document containers. \ No newline at end of file +The companion container is the largest change, but it directly addresses the biggest Cosmos RU issue identified in document listing: list queries are currently shaped by user/group/workspace access patterns while the source containers are partitioned by document id. The companion container aligns partitioning with the UI read pattern while preserving the existing source-of-truth document containers. diff --git a/docs/explanation/features/REDIS_EXPLORER.md b/docs/explanation/features/REDIS_EXPLORER.md new file mode 100644 index 00000000..c73ab414 --- /dev/null +++ b/docs/explanation/features/REDIS_EXPLORER.md @@ -0,0 +1,93 @@ +# Redis Explorer + +## Header Information + +**Overview**: Redis Explorer is an admin-only troubleshooting tool in Admin Settings > Scale > Redis Monitoring. It lets administrators browse Redis keys with cursor pagination, inspect sanitized read-only value previews, and resolve Document Access Index (DAI) Redis version-marker hashes to safe SimpleChat scope metadata. + +**Implemented in version**: **0.250.040** + +**Latest UI refinement version**: **0.250.043** + +**Related config.py version update**: `application/single_app/config.py` version **0.250.043** + +**Dependencies**: + +- Redis app-cache or session Redis client available at runtime. +- Admin Settings access. +- Existing Redis monitoring configuration. +- Document Access Index projection containers for DAI marker resolution. + +## Technical Specifications + +### Architecture + +Redis Explorer extends the existing Redis monitoring surface: + +- Backend helper: `functions_redis_monitoring.py` +- Admin routes: `route_backend_settings.py` +- UI modal and controls: `templates/admin_settings.html` +- Browser behavior: `static/js/admin/admin_settings.js` + +The feature is read-only. It never writes, deletes, expires, or mutates Redis keys. +The modal body can scroll when the dialog contents exceed the viewport. The key list and value preview also use bounded independent scrolling so admins can browse key pages without losing the selected preview context. + +DAI version-marker keys use the `DAI_LIST_CACHE_VERSION:{hash}` pattern. The hash is one-way, so Redis Explorer resolves it by re-hashing known SimpleChat scope keys from DAI projection rows, group workspaces, public workspaces, and user settings. When a match is found, the preview shows the entity type, ID, optional workspace name/status, scope key, and DAI row count. It does not expose raw settings, Redis secrets, or user-settings payloads. + +### API Endpoints + +- `GET /api/admin/settings/redis-explorer/keys` + - Admin-only. + - Uses Redis `SCAN` with a cursor, page size, and optional substring filter. + - Returns key metadata including key name, type, TTL, memory usage when available, whether preview is restricted, and safe SimpleChat resolution metadata when available. + +- `POST /api/admin/settings/redis-explorer/value` + - Admin-only. + - Accepts a Redis key in the JSON request body. + - Returns sanitized metadata, safe SimpleChat resolution metadata when available, and a bounded preview. + +### Security and Privacy + +- Only admins can access the endpoints. +- Value previews are sanitized and bounded. +- Keys with names containing session, token, cookie, credential, password, secret, authorization, or CSRF indicators return a restricted preview message instead of content. +- JSON fields with sensitive names are redacted. +- DAI hash resolution returns only safe entity metadata and summary counts; it does not expose raw app settings, Redis secrets, or user-settings documents. +- Errors returned to the browser do not expose Redis host names, connection strings, or secret values. + +## Usage Instructions + +1. Open Admin Settings. +2. Go to Scale. +3. In Redis Monitoring, choose Redis Explorer. +4. Leave Key Filter blank and choose Browse All to page through Redis keys, or enter a case-sensitive key substring and choose Apply Filter. +5. Choose a page size. +6. Use Previous Page and Next Page to move through Redis `SCAN` cursor pages. +7. Select a key to view sanitized metadata, SimpleChat resolution details when available, and preview content. + +Redis key names and filters are case sensitive. Keys can use uppercase names, prefixes, or internal identifiers. For example, global app settings cache entries are stored as `APP_SETTINGS_CACHE` and `APP_SETTINGS_CACHE_VERSION`, not `app_settings`. + +For DAI cache keys, Redis Monitoring also shows: + +- DAI payload key count. +- DAI version-marker key count. +- No-expiry DAI version-marker count. +- The current derived DAI version-marker TTL policy. + +DAI version markers now receive a bounded TTL that is longer than the payload TTL. With the current 900-second DAI payload TTL, marker TTLs are refreshed to 3,600 seconds on reads, invalidations, and app-maintenance hygiene. Existing no-expiry markers are repaired through the maintenance pass instead of being deleted. + +## Testing and Validation + +Coverage includes: + +- Redis Explorer key pagination and metadata using a fake Redis client. +- Sanitized JSON value previews. +- Restricted previews for session-like keys. +- DAI version-marker hygiene and safe hash resolution metadata. +- Admin UI static contract checks for modal controls and endpoint wiring. +- Route policy tests for new admin endpoints. + +Known limitations: + +- Redis `SCAN` cursors are forward-oriented; Previous uses the browser's local cursor history for the current modal session. +- Binary Redis keys or values are decoded with replacement characters for safe browser display. +- Previews are intentionally bounded and may be truncated. diff --git a/docs/explanation/fixes/CHAT_COMPLETION_BACKGROUND_UNREAD_GUARD_FIX.md b/docs/explanation/fixes/CHAT_COMPLETION_BACKGROUND_UNREAD_GUARD_FIX.md new file mode 100644 index 00000000..2c2484f8 --- /dev/null +++ b/docs/explanation/fixes/CHAT_COMPLETION_BACKGROUND_UNREAD_GUARD_FIX.md @@ -0,0 +1,59 @@ +# Chat Completion Background Unread Guard Fix + +## Header Information + +**Issue description**: After conversation mark-read cache invalidation tuning, a chat response that finished while the user was viewing another conversation could lose its notification and green unread dot. + +**Root cause analysis**: Streaming finalization still force-called `markConversationRead()` for the completed conversation and updated `window.currentConversationId` to the finished stream's conversation id. That made a background completion look active to the client and immediately cleared the unread state that the backend had just set. + +**Fixed in version**: **0.250.036** + +**Related config.py version update**: `application/single_app/config.py` was updated to version **0.250.036**. + +## Technical Details + +### Files Modified + +- `application/single_app/static/js/chat/chat-streaming.js` +- `application/single_app/config.py` +- `functional_tests/test_chat_streaming_background_unread_guard.py` +- `functional_tests/test_chat_completion_notifications.py` +- `docs/explanation/features/COSMOS_PERFORMANCE_OPTIMIZATION_PLAN.md` + +### Code Changes Summary + +- Added an active-conversation guard around streaming finalization mark-read calls. +- Kept forced mark-read behavior for the currently visible conversation so active readers do not see stale unread dots. +- Prevented a completed background stream from switching `window.currentConversationId` away from the conversation the user is currently viewing. +- Added a static regression test that verifies streaming finalization cannot call `markConversationRead()` directly with `finalData.conversation_id`. + +### Testing Approach + +- JavaScript syntax validation for `chat-streaming.js`. +- Focused static functional test for the background unread guard. +- Existing chat completion notification contract updated to require the guarded streaming helper. + +### Impact Analysis + +- Active conversation streaming behavior remains unchanged for users watching the response complete. +- Background chat completions keep their backend-created notification and green unread dot until the user opens that conversation. +- Conversation cache remains optional and source/Cosmos-backed fallback behavior is unchanged. + +## Validation + +### Test Results + +- `node --check application\single_app\static\js\chat\chat-streaming.js` +- `venv\Scripts\python.exe functional_tests\test_chat_streaming_background_unread_guard.py` +- `venv\Scripts\python.exe -m py_compile functional_tests\test_chat_completion_notifications.py functional_tests\test_chat_streaming_background_unread_guard.py` +- `git diff --check` + +### Before/After Comparison + +- **Before**: Finalizing a background stream force-cleared unread state and could remove the notification/dot before the user returned. +- **After**: Only the active visible conversation is force-cleared. Background completions stay unread until opened. + +### User Experience Improvements + +- Restores the expected "chat finished while I am away" notification behavior. +- Restores the green unread dot for conversations that complete in the background. diff --git a/docs/explanation/fixes/CONVERSATION_CACHE_INVALIDATION_AUTHORIZATION_FIX.md b/docs/explanation/fixes/CONVERSATION_CACHE_INVALIDATION_AUTHORIZATION_FIX.md new file mode 100644 index 00000000..bba650a7 --- /dev/null +++ b/docs/explanation/fixes/CONVERSATION_CACHE_INVALIDATION_AUTHORIZATION_FIX.md @@ -0,0 +1,43 @@ +# Conversation Cache Invalidation Authorization Fix + +Fixed/Implemented in version: **0.250.046** + +## Issue Description + +PR readiness validation flagged a direct personal conversation read in the message-mutation cache invalidation path. The read used a request-derived `conversation_id` before proving that the current caller owned the exact personal conversation. + +## Root Cause Analysis + +`_invalidate_conversation_cache_after_message_mutation(...)` loaded the conversation directly from `cosmos_conversations_container`. Although the helper only invalidated cache state, the direct read crossed an authorization boundary and bypassed the existing personal conversation ownership helper. + +## Technical Details + +Files modified: + +* `application/single_app/route_backend_conversations.py` +* `functional_tests/test_chat_completion_notifications.py` +* `functional_tests/test_cosmos_wave3a_indexing_maintenance.py` +* `functional_tests/test_cosmos_wave3b_document_access_index.py` +* `functional_tests/test_cosmos_wave4a1_admin_document_access_ui.py` +* `functional_tests/test_cosmos_wave4a_document_access_backfill.py` +* `functional_tests/test_cosmos_wave4b_document_access_shadow_validation.py` +* `functional_tests/test_cosmos_wave5a_document_access_read_switch.py` + +Code changes summary: + +* Routed the cache invalidation conversation load through `_authorize_personal_conversation_read(user_id, conversation_id)`. +* Updated DAI functional test fake `config` modules to include the current document access index container imports. +* Updated the chat completion notification functional test to import app modules through a fake Cosmos client boundary so local regression tests do not reach live Cosmos configuration. + +## Validation + +Validation performed: + +* `python scripts/check_broken_access_control.py --full-file application/single_app/route_backend_conversations.py` +* `python functional_tests/test_chat_completion_notifications.py` +* DAI functional tests covering indexing maintenance, write-through, admin UI, backfill, shadow validation, and read switch behavior. + +Impact: + +* Preserves cache invalidation behavior for authorized personal conversation mutations. +* Fails closed for unauthorized or missing conversations while still falling back to user-scoped cache version invalidation. \ No newline at end of file diff --git a/docs/explanation/fixes/COSMOSCLIENT_IMPORT_BINDING_CODEQL_FIX.md b/docs/explanation/fixes/COSMOSCLIENT_IMPORT_BINDING_CODEQL_FIX.md new file mode 100644 index 00000000..107d7392 --- /dev/null +++ b/docs/explanation/fixes/COSMOSCLIENT_IMPORT_BINDING_CODEQL_FIX.md @@ -0,0 +1,41 @@ +# CosmosClient Import Binding CodeQL Fix + +Fixed/Implemented in version: **0.250.047** + +## Issue Description + +CodeQL reported that direct imports of `CosmosClient` would not observe later changes to `azure.cosmos.CosmosClient`. This can happen when tests or diagnostics patch the Azure SDK module attribute before importing app modules. + +## Root Cause Analysis + +Python `from azure.cosmos import CosmosClient` binds the current class object to a local module name. If a test later replaces `azure.cosmos.CosmosClient`, that local binding still points to the original class. The affected SimpleChat modules included direct `CosmosClient` imports in Cosmos initialization and plugin paths. + +## Technical Details + +Files modified: + +* `application/single_app/config.py` +* `application/single_app/functions_data_management.py` +* `application/single_app/route_backend_plugins.py` +* `application/single_app/semantic_kernel_plugins/cosmos_query_plugin.py` +* `functional_tests/test_cosmos_query_plugin.py` + +Code changes summary: + +* Replaced direct `CosmosClient` imports with `import azure.cosmos as azure_cosmos`. +* Updated client creation to call `azure_cosmos.CosmosClient(...)` at runtime. +* Updated the Cosmos query plugin functional test to patch `module.azure_cosmos.CosmosClient` and to import app modules through a fake Cosmos client boundary. + +## Validation + +Validation performed: + +* Python syntax compilation for affected app and test files. +* `functional_tests/test_cosmos_query_plugin.py`. +* Search verification that direct `from azure.cosmos import CosmosClient` imports were removed from affected app files. + +Impact: + +* Runtime behavior is unchanged for normal app execution. +* CodeQL can observe module-qualified SDK lookups. +* Tests that patch `azure.cosmos.CosmosClient` now affect app modules consistently. \ No newline at end of file diff --git a/docs/explanation/fixes/SETTINGS_CONTAINER_RU_WRITE_SUPPRESSION_FIX.md b/docs/explanation/fixes/SETTINGS_CONTAINER_RU_WRITE_SUPPRESSION_FIX.md new file mode 100644 index 00000000..5af18d7b --- /dev/null +++ b/docs/explanation/fixes/SETTINGS_CONTAINER_RU_WRITE_SUPPRESSION_FIX.md @@ -0,0 +1,55 @@ +# Settings Container RU Write Suppression Fix + +## Issue Description + +The Cosmos DB `settings` container was consuming excessive RU with only a single active user. Investigation found several high-churn runtime paths writing or reading operational cache/state documents in `settings` when they should have used Redis, short-lived in-process state, or read-only behavior. + +Fixed in version: **0.250.037** + +## Root Cause + +- Generic app settings writes invalidated unrelated chat-bootstrap and custom-pages cache-version documents. +- The admin Cosmos throughput status refresh persisted live status back into app settings on every GET. +- Background Cosmos autoscale evaluations returned runtime settings updates even when no scale action occurred. +- Conversation cache versions and volatile payloads could fall back to the global `settings` container instead of bypassing cache when Redis was unavailable. +- DAI admin status reads could repeatedly point-read state documents, including shadow-validation state while shadow validation was disabled. + +## Technical Details + +### Files Modified + +- `application/single_app/functions_settings.py` +- `application/single_app/route_backend_settings.py` +- `application/single_app/functions_cosmos_throughput.py` +- `application/single_app/functions_shared_cache.py` +- `application/single_app/functions_chat_bootstrap_cache.py` +- `application/single_app/functions_conversation_cache.py` +- `application/single_app/route_backend_conversations.py` +- `application/single_app/functions_document_access_index.py` +- `application/single_app/config.py` + +### Code Changes Summary + +- Scoped app settings cache refresh to the app-settings cache only. +- Made Cosmos throughput status GET read-only. +- Limited autoscale runtime settings updates to actual scale actions or errors. +- Added `allow_cosmos_fallback=False` support to shared cache helpers. +- Disabled Cosmos fallback for chat bootstrap and conversation volatile cache payloads. +- Moved conversation cache versioning to Redis-only behavior; Redis-unavailable paths bypass cache and continue source Cosmos queries. +- Added short-TTL in-process caching for DAI backfill, repair backlog, and shadow-validation state reads. +- Skipped shadow-validation state reads in DAI status when shadow validation is disabled. + +## Validation + +- `python functional_tests\test_cosmos_wave2a_chat_bootstrap_cache.py` +- `python functional_tests\test_cosmos_wave2b_conversation_cache.py` +- `python functional_tests\test_cosmos_phase3_shared_cache_metrics.py` +- `python functional_tests\test_cosmos_wave4a_document_access_backfill.py` +- `python functional_tests\test_cosmos_throughput_refresh_logging.py` +- `.\venv\Scripts\python.exe functional_tests\test_cosmos_wave2a_custom_pages_cache.py` +- `python -m compileall -f` on touched application and functional test files + +## Before and After + +- **Before:** routine settings writes, status refreshes, and volatile cache activity could inflate `settings` writes and reads. +- **After:** only intentional app settings changes, actual autoscale actions, and required DAI state operations touch `settings`; Redis-unavailable conversation caching falls back to normal source-query behavior without creating cache/version documents. diff --git a/docs/explanation/fixes/index.md b/docs/explanation/fixes/index.md index 2902d5e3..782474a3 100644 --- a/docs/explanation/fixes/index.md +++ b/docs/explanation/fixes/index.md @@ -7,6 +7,10 @@ category: Version History --- - [Azure OpenAI Model Discovery Identity Fix](v0.250.001/AZURE_OPENAI_MODEL_DISCOVERY_IDENTITY_FIX.md) +- [CosmosClient Import Binding CodeQL Fix](COSMOSCLIENT_IMPORT_BINDING_CODEQL_FIX.md) +- [Conversation Cache Invalidation Authorization Fix](CONVERSATION_CACHE_INVALIDATION_AUTHORIZATION_FIX.md) +- [Chat Completion Background Unread Guard Fix](CHAT_COMPLETION_BACKGROUND_UNREAD_GUARD_FIX.md) +- [Settings Container RU Write Suppression Fix](SETTINGS_CONTAINER_RU_WRITE_SUPPRESSION_FIX.md) - [Tabular SK Python 3.13 Kernel Parameter Fix](v0.242.068/TABULAR_SK_PY313_KERNEL_PARAMETER_FIX.md) - [Chat Model Icon Avatar Fix](v0.242.071/CHAT_MODEL_ICON_AVATAR_FIX.md) - [Python 3.12 CI and XSS Guardrail Fix](PYTHON_312_CI_AND_XSS_GUARDRAIL_FIX.md) diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 3033ddde..d3cef187 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,183 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.250.047)** + +#### Bug Fixes + +* **CosmosClient Import Binding CodeQL Cleanup** + * Replaced direct `CosmosClient` imports with module-qualified `azure_cosmos.CosmosClient` lookups so tests and diagnostics that patch `azure.cosmos.CosmosClient` are observed consistently. + * Updated the Cosmos query plugin functional test to patch the module-qualified SDK client and avoid live Cosmos connections during app-module imports. + * (Ref: `config.py`, `functions_data_management.py`, `route_backend_plugins.py`, `cosmos_query_plugin.py`, `test_cosmos_query_plugin.py`) + +### **(v0.250.046)** + +#### Bug Fixes + +* **Conversation Cache Invalidation Authorization** + * Route-level message mutation cache invalidation now loads personal conversations through the existing ownership authorization helper instead of directly reading a request-derived conversation id. + * Updated PR-readiness functional test fixtures to match the current document access index config imports and avoid live Cosmos connections during notification regression tests. + * (Ref: `route_backend_conversations.py`, `test_chat_completion_notifications.py`, DAI functional test fixtures) + +### **(v0.250.044)** + +#### New Features + +* **Development Requirements Overlay** + * Added a root `requirements-dev.txt` for test and development-only Python dependencies that are not already included in the base app requirements. + * The overlay includes pytest, pytest Playwright integration, and Azure Playwright management support so local validation can be installed without duplicating the production dependency set. + * (Ref: `requirements-dev.txt`, testing dependencies, local development setup) + +### **(v0.250.043)** + +#### New Features + +* **Redis Explorer SimpleChat Resolution** + * Redis Explorer now resolves `DAI_LIST_CACHE_VERSION:{hash}` keys to safe SimpleChat scope metadata when possible, including user/group/public workspace identity, workspace name/status, DAI row counts, and source/access-role summaries. + * Added DAI cache hygiene counters to Redis Monitoring for payload keys, version markers, no-expiry markers, and the active version-marker TTL policy. + * (Ref: `functions_redis_monitoring.py`, `functions_document_access_index.py`, `admin_settings.html`, `admin_settings.js`, `REDIS_EXPLORER.md`) + +#### Bug Fixes + +* **DAI Redis Version Marker TTL Hygiene** + * DAI Redis version marker keys now receive a bounded TTL that outlives DAI payload cache entries instead of remaining in Redis indefinitely. + * Marker TTLs refresh on DAI cache reads, cache invalidations, and app-maintenance hygiene; existing no-expiry markers are repaired in place rather than deleted. + * With the current 900-second DAI payload TTL, version markers are refreshed to 3,600 seconds. + * (Ref: DAI Redis cache version markers, `refresh_document_access_cache_version_marker_ttls`, app maintenance) + +### **(v0.250.041)** + +#### User Interface Enhancements + +* **Redis Explorer Browse-All Layout** + * Improved Redis Explorer with a fixed-height modal where the key list and sanitized preview pane scroll independently, keeping pagination controls and selected preview context easier to use. + * Added an explicit Browse All action, clearer Apply Filter and Previous Page/Next Page controls, page/scope status text, and guidance for app settings cache key names such as `APP_SETTINGS_CACHE`. + * (Ref: `REDIS_EXPLORER.md`, `admin_settings.html`, `admin_settings.js`, Redis Explorer) + +### **(v0.250.040)** + +#### New Features + +* **Redis Explorer** + * Added an admin-only Redis Explorer in Admin Settings > Scale > Redis Monitoring for read-only, cursor-paginated Redis key browsing with substring filtering and page-size controls. + * Admins can select a key to view sanitized metadata and bounded preview content; session, token, cookie, credential, password, secret, authorization, and CSRF-like keys return restricted previews. + * JSON previews redact sensitive fields and all browser rendering uses text-safe DOM updates. + * (Ref: `REDIS_EXPLORER.md`, `functions_redis_monitoring.py`, `route_backend_settings.py`, `admin_settings.html`, `admin_settings.js`) + +### **(v0.250.039)** + +#### New Features + +* **Phase 9 Cosmos Performance Runbooks** + * Added support guidance for rebuilding caches, rebuilding the Document Access Index, cleaning stale operational cache documents, applying expected Cosmos composite indexes, interpreting shadow validation diffs, and interpreting DAI fallback/cache metrics. + * Documented when broad source fallback should remain available while DAI repair, backfill, Redis cache, and production fallback telemetry stabilize. + * (Ref: `COSMOS_PERFORMANCE_OPTIMIZATION_PLAN.md`, Cosmos maintenance, DAI runbooks) + +#### User Interface Enhancements + +* **Guarded Cosmos Index Apply Action** + * Added an Admin Settings > Scale > Cosmos Maintenance action for applying missing expected Cosmos composite indexes through the existing app-maintenance endpoint. + * The confirmation modal explains that updates are additive and preserve existing indexing policy paths, but can add write-index overhead and trigger asynchronous Cosmos index transformation. + * (Ref: `admin_settings.html`, `admin_settings.js`, `functions_cosmos_indexing.py`, app maintenance) + +### **(v0.250.037)** + +#### Bug Fixes + +* **Settings Container RU Write Suppression** + * Reduced idle Cosmos DB `settings` container RU consumption by stopping routine status refreshes, no-op autoscale checks, and generic settings writes from creating unrelated cache/version churn. + * Conversation cache versioning and volatile chat bootstrap/conversation payloads now require Redis; when Redis is unavailable, the app bypasses those caches and falls back to source Cosmos query behavior instead of writing cache artifacts to `settings`. + * Document Access Index status now uses short-lived in-process state caching and skips shadow-validation state reads when shadow validation is disabled. + * Post-deploy validation showed `settings` dropped out of the top normalized RU consumers during idle monitoring. + * (Ref: `functions_settings.py`, `functions_cosmos_throughput.py`, `functions_shared_cache.py`, `functions_conversation_cache.py`, `functions_document_access_index.py`, `SETTINGS_CONTAINER_RU_WRITE_SUPPRESSION_FIX.md`) + +### **(v0.250.035)** + +#### Bug Fixes + +* **Conversation Cache Mark-Read Invalidation Tuning** + * Normal conversation switching now skips mark-read requests when the client has no unread assistant-response state, reducing unnecessary conversation feed cache invalidations during page reloads and navigation. + * The mark-read API now only upserts the conversation and bumps the user-scoped conversation cache version when unread conversation fields actually changed; notification clearing remains intact. + * (Ref: conversation cache invalidation, mark-read flow, `chat-conversations.js`, `route_backend_conversations.py`, `test_conversations_read_ownership_authorization.py`) + +### **(v0.250.034)** + +#### New Features + +* **Conversation Cache Metrics Dashboard** + * Added DAI-style rolling metrics for conversation list, feed, and advanced-search cache activity, including 15-minute hit rate, hits/misses, bypasses/errors, writes/invalidations, operation mix, last cache event, and last invalidation. + * Exposed normalized conversation cache settings and metrics through app maintenance status without adding Cosmos reads to the conversation hot path. + * Removed the Phase 4 badge from the Conversation Cache card now that the feature is part of the operational dashboard. + * (Ref: conversation cache metrics, `functions_conversation_cache.py`, `functions_app_maintenance.py`, `admin_settings.html`, `admin_settings.js`) + +### **(v0.250.033)** + +#### New Features + +* **Phase 4 Conversation Cache Hardening** + * Hardened conversation list, feed, and advanced-search caching behind an explicit `enable_conversation_cache` setting. + * Added Admin Settings > Scale controls for enabling conversation cache and adjusting the conversation cache TTL without requiring direct Cosmos settings edits. + * Redis remains optional: enabled cache paths use the shared Redis-first/Cosmos-fallback helper, while disabled cache paths bypass cache reads and writes and continue using source Cosmos queries. + * Conversation cache entries remain user-scoped and versioned, with collaboration-aware feed/search fingerprints based on the user's group-access state. + * (Ref: conversation cache, optional Redis fallback, `functions_conversation_cache.py`, `route_backend_conversations.py`, `admin_settings.html`) + +#### Bug Fixes + +* **Conversation Cache Invalidation Coverage** + * Added cache invalidation after metadata reads normalize and persist legacy/missing `chat_type` values so cached list/search payloads do not remain stale. + * Updated conversation cache and ownership authorization regressions to cover disabled-cache bypass, source fallback, and route wiring. + * (Ref: conversation metadata normalization, cache invalidation, `test_cosmos_wave2b_conversation_cache.py`, `test_conversations_read_ownership_authorization.py`) + +### **(v0.250.032)** + +#### New Features + +* **Phase 3 Low-Churn Cache Hardening** + * Hardened the shared low-churn cache foundation for custom pages/navigation and chat bootstrap data. + * Added safe shared-cache diagnostics for hit, miss, write, delete, and version-bump activity using hashed cache-key context, and exposed those metrics through app maintenance status. + * Expanded chat bootstrap cache invalidation coverage across group and public workspace metadata, membership, role, ownership, status, model endpoint, Control Center, and SimpleChat operation mutations. + * (Ref: shared cache metrics, chat bootstrap cache, custom pages cache, `functions_shared_cache.py`, `functions_app_maintenance.py`) + +#### Bug Fixes + +* **Low-Churn Cache Invalidation Coverage** + * Fixed stale chat bootstrap labels after group rename/update paths by invalidating the global chat bootstrap cache after successful group metadata writes. + * Custom pages/navigation cache now invalidates after app settings writes so menu and feature-setting changes do not wait for TTL expiry. + * (Ref: group metadata updates, custom page settings invalidation, `route_backend_groups.py`, `functions_settings.py`) + +* **Public Workspace Document Manager Approval** + * Fixed document-manager request approval so the manager addition and pending-request cleanup are persisted in a single workspace update instead of risking a stale overwrite. + * The approval and rejection flows now invalidate chat bootstrap cache state for public workspace visibility updates. + * (Ref: public workspace document managers, `functions_public_workspaces.py`, `test_cosmos_wave2a_chat_bootstrap_cache.py`) + +### **(v0.250.030)** + +#### New Features + +* **Redis Document Access Index Cache** + * Added Redis read-through caching for DAI-backed document list, tag list, and legacy-count reads with scope-version invalidation and bounded TTL controls. + * Admin Settings now shows Redis DAI cache health, hit/miss/bypass/error metrics, invalidations, and the latest cache event alongside DAI read and maintenance status. + * (Ref: DAI Redis cache, `functions_document_access_index.py`, `admin_settings.html`, `admin_settings.js`) + +#### Bug Fixes + +* **DAI Cache Invalidation and Repair Safety** + * Hardened DAI cache invalidation so access changes fail closed when Redis invalidation cannot be proven safe, including revoked-share, delete, partial projection failure, and untracked repair-state scenarios. + * Preserved historical revoked scopes through repair and blocked DAI/cache reads while repair safety is unknown. + * (Ref: DAI cache invalidation, projection repair backlog, `test_cosmos_wave6_document_access_cache.py`) + +* **DAI List Parity Improvements** + * DAI document-list reads now include pending shared documents needed by approval UI, preserve generated-artifact requester identity for public workspace actions, collapse legacy revisions like source reads, and match source exact/case-sensitive classification and array filters. + * DAI tag-list reads now project file names for legacy rows so distinct legacy documents without revision-family metadata do not collapse into a single tag-count identity. + * External public DAI lists now use unfiltered DAI legacy counts so legacy-update prompts match source-backed behavior. + * (Ref: DAI list parity, public workspace documents, external public documents) + +* **Maintenance Failure Visibility** + * Manual app maintenance runs now surface failed step status instead of reporting full success, and Admin Settings keeps detailed DAI status visible when a run completes with errors. + * The DAI maintenance status now reports automatic maintenance only when the app-maintenance scheduler gates are enabled and startup maintenance can actually run. + * Optional shadow-validation source queries now fail open after successful DAI reads so diagnostics cannot break served list responses. + * (Ref: app maintenance, DAI shadow validation, `functions_app_maintenance.py`, document list routes) + ### **(v0.250.015)** #### Bug Fixes diff --git a/functional_tests/debug_containers.py b/functional_tests/debug_containers.py index 07c088aa..c056fc6c 100644 --- a/functional_tests/debug_containers.py +++ b/functional_tests/debug_containers.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 """ Debug script to check container structure and data. +Version: 0.250.047 + +CosmosClient module import cleanup updated in: 0.250.047 """ import os import sys -from azure.cosmos import CosmosClient +import azure.cosmos as azure_cosmos def debug_containers(): """Debug containers and data structure.""" @@ -14,7 +17,7 @@ def debug_containers(): endpoint = os.getenv('AZURE_COSMOS_ENDPOINT', '') key = os.getenv('AZURE_COSMOS_KEY', '') - client = CosmosClient(endpoint, key, consistency_level="Session") + client = azure_cosmos.CosmosClient(endpoint, key, consistency_level="Session") database = client.get_database_client("SimpleChat") diff --git a/functional_tests/debug_document_metrics.py b/functional_tests/debug_document_metrics.py index bca6a076..264c6fb6 100644 --- a/functional_tests/debug_document_metrics.py +++ b/functional_tests/debug_document_metrics.py @@ -1,14 +1,15 @@ #!/usr/bin/env python3 """ Debug document metrics for Control Center. -Version: 0.230.023 +Version: 0.250.047 Implemented in: 0.230.023 +CosmosClient module import cleanup updated in: 0.250.047 This script debugs document metrics to understand structure and implement improvements. """ import os -from azure.cosmos import CosmosClient +import azure.cosmos as azure_cosmos from datetime import datetime, timezone, timedelta def debug_document_metrics(): @@ -18,7 +19,7 @@ def debug_document_metrics(): endpoint = os.getenv('AZURE_COSMOS_ENDPOINT', '') key = os.getenv('AZURE_COSMOS_KEY', '') - client = CosmosClient(endpoint, key, consistency_level="Session") + client = azure_cosmos.CosmosClient(endpoint, key, consistency_level="Session") database = client.get_database_client("SimpleChat") # Get user documents container diff --git a/functional_tests/debug_message_count.py b/functional_tests/debug_message_count.py index a2765ad8..3bcab8e3 100644 --- a/functional_tests/debug_message_count.py +++ b/functional_tests/debug_message_count.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 """ Debug script for message count issue with specific user ID -Version: 0.230.022 +Version: 0.250.047 +Implemented in: 0.230.022 +CosmosClient module import cleanup updated in: 0.250.047 This script will test the actual queries against the database containers. """ import sys import os -from azure.cosmos import CosmosClient +import azure.cosmos as azure_cosmos def debug_message_count_for_user(): """Debug message count queries for specific user""" @@ -34,7 +36,7 @@ def debug_message_count_for_user(): endpoint = os.getenv('AZURE_COSMOS_ENDPOINT', '') key = os.getenv('AZURE_COSMOS_KEY', '') - client = CosmosClient(endpoint, key, consistency_level="Session") + client = azure_cosmos.CosmosClient(endpoint, key, consistency_level="Session") conversations = list(client.get_database_client("SimpleChat").get_container_client("conversations").query_items( query=conversations_query, diff --git a/functional_tests/debug_message_simple.py b/functional_tests/debug_message_simple.py index 5eaddb1f..5fbaf3bb 100644 --- a/functional_tests/debug_message_simple.py +++ b/functional_tests/debug_message_simple.py @@ -1,14 +1,15 @@ #!/usr/bin/env python3 """ Simple debug script for message counting issue. -Version: To be checked against config.py +Version: 0.250.047 Implemented in: 0.230.022 +CosmosClient module import cleanup updated in: 0.250.047 This script debugs why message counts show 0 for users with conversations. """ import os import sys -from azure.cosmos import CosmosClient +import azure.cosmos as azure_cosmos # Set up environment os.environ['ENVIRONMENT'] = 'development' @@ -21,7 +22,7 @@ def debug_message_count(): endpoint = os.getenv('AZURE_COSMOS_ENDPOINT', '') key = os.getenv('AZURE_COSMOS_KEY', '') - client = CosmosClient(endpoint, key, consistency_level="Session") + client = azure_cosmos.CosmosClient(endpoint, key, consistency_level="Session") database = client.get_database_client("SimpleChat") conversations_container = database.get_container_client("conversations") diff --git a/functional_tests/route_tests/test_route_blueprint_policy_inventory.py b/functional_tests/route_tests/test_route_blueprint_policy_inventory.py index c85ffcea..b236df2b 100644 --- a/functional_tests/route_tests/test_route_blueprint_policy_inventory.py +++ b/functional_tests/route_tests/test_route_blueprint_policy_inventory.py @@ -2,7 +2,7 @@ # test_route_blueprint_policy_inventory.py """ Functional test for route blueprint policy inventory. -Version: 0.250.004 +Version: 0.250.006 Implemented in: 0.242.069 This test ensures every SimpleChat route is assigned to a Blueprint-based diff --git a/functional_tests/test_chat_completion_notifications.py b/functional_tests/test_chat_completion_notifications.py index 2b6c01a2..70d89b3d 100644 --- a/functional_tests/test_chat_completion_notifications.py +++ b/functional_tests/test_chat_completion_notifications.py @@ -2,8 +2,10 @@ # test_chat_completion_notifications.py """ Functional test for chat completion notifications. -Version: 0.239.136 +Version: 0.250.047 Implemented in: 0.239.128 +Mark-read cache invalidation tuned in: 0.250.035 +Background streaming unread guard updated in: 0.250.036 This test ensures that personal chat completions create deep-link notifications, conversation unread state is normalized for list/detail responses, and the @@ -11,6 +13,7 @@ """ import copy +import importlib import os import re import sys @@ -26,6 +29,7 @@ class FakeConversationContainer: def __init__(self, items=None): self.items = {} + self.upsert_count = 0 for item in items or []: self.upsert_item(item) @@ -36,6 +40,7 @@ def read_item(self, item=None, partition_key=None, *args, **kwargs): return copy.deepcopy(self.items[item_id]) def upsert_item(self, item): + self.upsert_count += 1 self.items[item['id']] = copy.deepcopy(item) return copy.deepcopy(item) @@ -93,10 +98,47 @@ def query_items(self, query=None, parameters=None, partition_key=None, enable_cr return results +class FakeConfigCosmosDatabase: + """Minimal Cosmos database stand-in for importing config.py without live I/O.""" + + def __init__(self): + self.containers = {} + + def create_container_if_not_exists(self, id, **kwargs): + if id not in self.containers: + self.containers[id] = FakeConversationContainer() + return self.containers[id] + + +class FakeConfigCosmosClient: + """Minimal Cosmos client stand-in for config.py import-time container setup.""" + + def __init__(self, *args, **kwargs): + self.database = FakeConfigCosmosDatabase() + + def create_database_if_not_exists(self, *args, **kwargs): + return self.database + + +def import_app_module_without_live_cosmos(module_name): + """Import app modules without letting config.py connect to live Cosmos.""" + if module_name in sys.modules: + return sys.modules[module_name] + + import azure.cosmos as azure_cosmos + + original_cosmos_client = azure_cosmos.CosmosClient + azure_cosmos.CosmosClient = FakeConfigCosmosClient + try: + return importlib.import_module(module_name) + finally: + azure_cosmos.CosmosClient = original_cosmos_client + + def build_test_app(test_user_id, conversation_container, notification_container): """Register the conversation routes with fake auth/container dependencies.""" - import functions_notifications - import route_backend_conversations + functions_notifications = import_app_module_without_live_cosmos("functions_notifications") + route_backend_conversations = import_app_module_without_live_cosmos("route_backend_conversations") original_notification_container = functions_notifications.cosmos_notifications_container original_conversation_container = route_backend_conversations.cosmos_conversations_container @@ -105,6 +147,8 @@ def build_test_app(test_user_id, conversation_container, notification_container) original_swagger_route = route_backend_conversations.swagger_route original_get_auth_security = route_backend_conversations.get_auth_security original_get_current_user_id = route_backend_conversations.get_current_user_id + original_bump_conversation_cache_version = route_backend_conversations.bump_conversation_cache_version + cache_bumps = [] functions_notifications.cosmos_notifications_container = notification_container route_backend_conversations.cosmos_conversations_container = conversation_container @@ -113,6 +157,9 @@ def build_test_app(test_user_id, conversation_container, notification_container) route_backend_conversations.swagger_route = lambda **kwargs: (lambda func: func) route_backend_conversations.get_auth_security = lambda: {} route_backend_conversations.get_current_user_id = lambda: test_user_id + route_backend_conversations.bump_conversation_cache_version = ( + lambda user_id, reason=None: cache_bumps.append((user_id, reason)) or len(cache_bumps) + ) app = Flask(__name__) app.config['TESTING'] = True @@ -126,8 +173,9 @@ def restore(): route_backend_conversations.swagger_route = original_swagger_route route_backend_conversations.get_auth_security = original_get_auth_security route_backend_conversations.get_current_user_id = original_get_current_user_id + route_backend_conversations.bump_conversation_cache_version = original_bump_conversation_cache_version - return app, restore + return app, restore, cache_bumps def unwrap_response(result): @@ -146,7 +194,7 @@ def test_chat_response_notification_creation_and_deep_link(): """Verify helper-created notifications include the chat completion deep link and metadata.""" print("🔍 Testing chat completion notification creation...") - import functions_notifications + functions_notifications = import_app_module_without_live_cosmos("functions_notifications") fake_container = FakeNotificationContainer() original_container = functions_notifications.cosmos_notifications_container @@ -201,7 +249,7 @@ def test_conversation_routes_normalize_unread_fields(): 'is_hidden': False, } - app, restore = build_test_app( + app, restore, _cache_bumps = build_test_app( test_user_id, FakeConversationContainer([old_conversation]), FakeNotificationContainer(), @@ -246,7 +294,7 @@ def test_mark_read_endpoint_clears_unread_state_and_notification(): """Verify mark-read clears conversation unread state and marks matching notifications read.""" print("🔍 Testing conversation mark-read lifecycle...") - import functions_notifications + functions_notifications = import_app_module_without_live_cosmos("functions_notifications") test_user_id = 'test-user-mark-read' conversation_id = 'conversation-mark-read' @@ -281,7 +329,8 @@ def test_mark_read_endpoint_clears_unread_state_and_notification(): ) functions_notifications.cosmos_notifications_container = original_notification_container - app, restore = build_test_app(test_user_id, fake_conversations, fake_notifications) + app, restore, cache_bumps = build_test_app(test_user_id, fake_conversations, fake_notifications) + fake_conversations.upsert_count = 0 try: with app.test_request_context(f'/api/conversations/{conversation_id}/mark-read', method='POST'): @@ -297,6 +346,9 @@ def test_mark_read_endpoint_clears_unread_state_and_notification(): if not response_payload.get('success'): print(f"❌ Mark-read endpoint did not report success: {response_payload}") return False + if response_payload.get('conversation_state_changed') is not True: + print(f"❌ Mark-read endpoint did not report conversation state changed: {response_payload}") + return False updated_conversation = fake_conversations.read_item(conversation_id, conversation_id) if updated_conversation.get('has_unread_assistant_response'): @@ -312,6 +364,31 @@ def test_mark_read_endpoint_clears_unread_state_and_notification(): print(f"❌ Notification was not marked read: {stored_notification}") return False + if cache_bumps != [(test_user_id, 'conversation_marked_read')]: + print(f"❌ Mark-read did not invalidate conversation cache exactly once: {cache_bumps}") + return False + + first_upsert_count = fake_conversations.upsert_count + with app.test_request_context(f'/api/conversations/{conversation_id}/mark-read', method='POST'): + second_response, second_status_code = unwrap_response( + app.view_functions['mark_conversation_read_api'](conversation_id) + ) + + if second_status_code != 200: + print(f"❌ Unexpected idempotent mark-read status: {second_status_code}") + return False + + second_payload = second_response.get_json() + if second_payload.get('conversation_state_changed') is not False: + print(f"❌ Idempotent mark-read should not report changed state: {second_payload}") + return False + if len(cache_bumps) != 1: + print(f"❌ Idempotent mark-read should not invalidate conversation cache: {cache_bumps}") + return False + if fake_conversations.upsert_count != first_upsert_count: + print("❌ Idempotent mark-read should not upsert an already-read conversation") + return False + print("✅ Mark-read endpoint cleared conversation unread state and related notification") return True finally: @@ -332,8 +409,10 @@ def test_frontend_wires_unread_dot_and_mark_read_flow(): (main_list_js, 'fetch(`/api/conversations/${conversationId}/mark-read`, {'), (main_list_js, 'function createUnreadDotElement() {'), (main_list_js, 'setSidebarConversationUnreadState(conversationId, hasUnread);'), + (main_list_js, 'markConversationRead(conversationId, { suppressErrorToast: true })'), (sidebar_js, "conversation-unread-dot', 'sidebar-conversation-unread-dot"), - (streaming_js, "markConversationRead(finalData.conversation_id, { force: true, suppressErrorToast: true })"), + (streaming_js, "function markStreamingConversationReadIfActive(conversationId, contextLabel)"), + (streaming_js, "markStreamingConversationReadIfActive(finalData.conversation_id, 'live streaming completion')"), (chats_css, '.conversation-unread-dot {'), ] @@ -398,11 +477,11 @@ def test_version_updated_for_feature(): with open(config_file_path, 'r', encoding='utf-8') as handle: config_content = handle.read() - if 'VERSION = "0.239.136"' not in config_content: - print("❌ Version not updated to 0.239.136") + if 'VERSION = "0.250.047"' not in config_content: + print("❌ Version not updated to 0.250.047") return False - print("✅ Version properly updated to 0.239.136") + print("✅ Version properly updated to 0.250.047") return True diff --git a/functional_tests/test_chat_streaming_background_unread_guard.py b/functional_tests/test_chat_streaming_background_unread_guard.py new file mode 100644 index 00000000..902e6ee0 --- /dev/null +++ b/functional_tests/test_chat_streaming_background_unread_guard.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +""" +Functional test for background chat streaming unread-state guard. +Version: 0.250.036 +Implemented in: 0.250.036 + +This test ensures streaming finalization only clears unread assistant-response +state when the completed conversation is still the active conversation, so +background completions keep their notification and green unread dot. +""" + +import os +import re +import sys + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +STREAMING_JS = os.path.join( + ROOT_DIR, + "application", + "single_app", + "static", + "js", + "chat", + "chat-streaming.js", +) + + +def test_streaming_finalization_keeps_background_unread_state(): + """Validate streaming completion mark-read is active-conversation guarded.""" + print("Testing streaming background unread guard...") + + with open(STREAMING_JS, "r", encoding="utf-8") as handle: + source = handle.read() + + required_snippets = [ + "function isConversationCurrentlyActive(conversationId)", + "function markStreamingConversationReadIfActive(conversationId, contextLabel)", + "if (!isConversationCurrentlyActive(conversationId)) {", + "markConversationRead(conversationId, { force: true, suppressErrorToast: true })", + "markStreamingConversationReadIfActive(finalData.conversation_id, 'stream cancellation')", + "markStreamingConversationReadIfActive(finalData.conversation_id, 'live streaming completion')", + "if (finalData.conversation_id && !window.currentConversationId) {", + ] + for snippet in required_snippets: + if snippet not in source: + print(f"Missing required streaming guard snippet: {snippet}") + return False + + forbidden_patterns = [ + r"markConversationRead\(finalData\.conversation_id,\s*\{\s*force:\s*true", + r"if\s*\(\s*finalData\.conversation_id\s*&&\s*window\.currentConversationId\s*!==\s*finalData\.conversation_id\s*\)", + ] + for pattern in forbidden_patterns: + if re.search(pattern, source): + print(f"Found forbidden unguarded streaming pattern: {pattern}") + return False + + print("Streaming background unread guard test passed!") + return True + + +if __name__ == "__main__": + success = test_streaming_finalization_keeps_background_unread_state() + sys.exit(0 if success else 1) diff --git a/functional_tests/test_conversations_read_ownership_authorization.py b/functional_tests/test_conversations_read_ownership_authorization.py index e9d6a5f7..1351a09a 100644 --- a/functional_tests/test_conversations_read_ownership_authorization.py +++ b/functional_tests/test_conversations_read_ownership_authorization.py @@ -2,13 +2,14 @@ # test_conversations_read_ownership_authorization.py """ Functional test for personal conversation read authorization hardening. -Version: 0.241.032 -Implemented in: 0.241.011; 0.241.022; 0.241.032 +Version: 0.250.036 +Implemented in: 0.241.011; 0.241.022; 0.241.032; 0.250.033; 0.250.035 This test ensures authenticated users can only read messages and images from their own personal conversations, and that foreign conversation reads fail with 403 without querying the message container. It also validates blob-backed image -messages stream only after conversation authorization succeeds. +messages stream only after conversation authorization succeeds. It validates +that mark-read cache invalidation is idempotent for already-read conversations. """ import copy @@ -36,6 +37,7 @@ class FakeConversationContainer: def __init__(self, items=None): self.items = {} + self.upsert_count = 0 for item in items or []: self.items[item['id']] = copy.deepcopy(item) @@ -45,6 +47,11 @@ def read_item(self, item=None, partition_key=None, *args, **kwargs): raise DummyNotFoundError(item_id) return copy.deepcopy(self.items[item_id]) + def upsert_item(self, item): + self.upsert_count += 1 + self.items[item['id']] = copy.deepcopy(item) + return copy.deepcopy(item) + class FakeMessageContainer: """In-memory message container that tracks query attempts.""" @@ -133,12 +140,19 @@ def _install_route_import_stubs(): config_module.CLIENTS = {} stub_modules['config'] = config_module + appinsights_module = types.ModuleType('functions_appinsights') + appinsights_module.log_event = lambda *args, **kwargs: None + stub_modules['functions_appinsights'] = appinsights_module + collaboration_module = types.ModuleType('functions_collaboration') collaboration_module.assert_user_can_view_collaboration_conversation = lambda *args, **kwargs: None collaboration_module.assert_user_can_participate_in_collaboration_conversation = lambda *args, **kwargs: None collaboration_module.ensure_collaboration_source_conversation = lambda *args, **kwargs: None collaboration_module.get_collaboration_conversation = lambda *args, **kwargs: (_ for _ in ()).throw(DummyNotFoundError('missing')) + collaboration_module.list_group_collaboration_conversations_for_user = lambda *args, **kwargs: [] collaboration_module.list_collaboration_messages = lambda *args, **kwargs: [] + collaboration_module.list_personal_collaboration_conversations_for_user = lambda *args, **kwargs: [] + collaboration_module.serialize_collaboration_conversation = lambda item, *args, **kwargs: item stub_modules['functions_collaboration'] = collaboration_module auth_module = types.ModuleType('functions_authentication') @@ -157,9 +171,46 @@ def _install_route_import_stubs(): metadata_module.update_conversation_with_metadata = lambda *args, **kwargs: None stub_modules['functions_conversation_metadata'] = metadata_module + conversation_cache_module = types.ModuleType('functions_conversation_cache') + conversation_cache_module.build_conversation_cache_key = lambda *args, **kwargs: None + conversation_cache_module.bump_conversation_cache_version = lambda *args, **kwargs: None + conversation_cache_module.get_cached_conversation_payload = lambda *args, **kwargs: None + conversation_cache_module.get_conversation_cache_settings = lambda *args, **kwargs: { + 'enabled': False, + 'ttl_seconds': 0, + } + conversation_cache_module.invalidate_conversation_cache_for_item = lambda *args, **kwargs: None + conversation_cache_module.set_cached_conversation_payload = lambda *args, **kwargs: None + stub_modules['functions_conversation_cache'] = conversation_cache_module + + conversation_feed_module = types.ModuleType('functions_conversation_feed') + conversation_feed_module.CONVERSATION_FEED_SOURCE_COLLABORATION = 'collaboration' + conversation_feed_module.CONVERSATION_FEED_SOURCE_LEGACY = 'legacy' + conversation_feed_module.build_conversation_feed_page = lambda *args, **kwargs: {} + conversation_feed_module.decode_conversation_feed_cursor = lambda *args, **kwargs: None + conversation_feed_module.get_conversation_feed_source_offsets = lambda *args, **kwargs: {} + conversation_feed_module.is_conversation_feed_cursor_compatible = lambda *args, **kwargs: False + conversation_feed_module.normalize_conversation_feed_page_size = lambda value=None: int(value or 50) + conversation_feed_module.sort_conversation_feed_recent = lambda items: items + conversation_feed_module.tag_conversation_feed_source = lambda item, source: item + stub_modules['functions_conversation_feed'] = conversation_feed_module + + def normalize_conversation_unread_state(item): + item['has_unread_assistant_response'] = bool(item.get('has_unread_assistant_response', False)) + item['last_unread_assistant_message_id'] = item.get('last_unread_assistant_message_id') + item['last_unread_assistant_at'] = item.get('last_unread_assistant_at') + return item + + def clear_conversation_unread(item): + normalized_item = normalize_conversation_unread_state(item) + normalized_item['has_unread_assistant_response'] = False + normalized_item['last_unread_assistant_message_id'] = None + normalized_item['last_unread_assistant_at'] = None + return normalized_item + unread_module = types.ModuleType('functions_conversation_unread') - unread_module.clear_conversation_unread = lambda *args, **kwargs: None - unread_module.normalize_conversation_unread_state = lambda item: item + unread_module.clear_conversation_unread = clear_conversation_unread + unread_module.normalize_conversation_unread_state = normalize_conversation_unread_state stub_modules['functions_conversation_unread'] = unread_module notifications_module = types.ModuleType('functions_notifications') @@ -170,6 +221,15 @@ def _install_route_import_stubs(): debug_module.debug_print = lambda *args, **kwargs: None stub_modules['functions_debug'] = debug_module + documents_module = types.ModuleType('functions_documents') + documents_module.delete_chat_upload_workspace_documents_for_conversation = lambda *args, **kwargs: None + documents_module.serialize_chat_upload_workspace_documents_for_conversation = lambda *args, **kwargs: [] + stub_modules['functions_documents'] = documents_module + + group_module = types.ModuleType('functions_group') + group_module.get_user_groups = lambda *args, **kwargs: [] + stub_modules['functions_group'] = group_module + artifacts_module = types.ModuleType('functions_message_artifacts') artifacts_module.build_message_artifact_payload_map = lambda items: {} artifacts_module.filter_assistant_artifact_items = lambda items: items @@ -179,6 +239,7 @@ def _install_route_import_stubs(): simplechat_operations_module = types.ModuleType('functions_simplechat_operations') simplechat_operations_module.create_personal_conversation_for_current_user = lambda *args, **kwargs: {} simplechat_operations_module.delete_blob_backed_chat_message_files = lambda *args, **kwargs: None + simplechat_operations_module.derive_conversation_title_from_message = lambda *args, **kwargs: '' stub_modules['functions_simplechat_operations'] = simplechat_operations_module swagger_module = types.ModuleType('swagger_wrapper') @@ -197,6 +258,10 @@ def _install_route_import_stubs(): thoughts_module.delete_thoughts_for_conversation = lambda *args, **kwargs: None stub_modules['functions_thoughts'] = thoughts_module + utils_cache_module = types.ModuleType('utils_cache') + utils_cache_module.invalidate_personal_search_cache = lambda *args, **kwargs: None + stub_modules['utils_cache'] = utils_cache_module + for module_name, module in stub_modules.items(): sys.modules[module_name] = module @@ -215,6 +280,7 @@ def build_test_app(test_user_id, conversation_items, message_items, blob_items=N conversation_container = FakeConversationContainer(conversation_items) message_container = FakeMessageContainer(message_items) + cache_bumps = [] route_backend_conversations.cosmos_conversations_container = conversation_container route_backend_conversations.cosmos_messages_container = message_container @@ -226,12 +292,19 @@ def build_test_app(test_user_id, conversation_items, message_items, blob_items=N route_backend_conversations.debug_print = lambda *args, **kwargs: None route_backend_conversations.filter_assistant_artifact_items = lambda items: items route_backend_conversations.CosmosResourceNotFoundError = DummyNotFoundError + route_backend_conversations.bump_conversation_cache_version = ( + lambda user_id, reason=None: cache_bumps.append((user_id, reason)) or len(cache_bumps) + ) + route_backend_conversations.mark_chat_response_notifications_read_for_conversation = lambda *args, **kwargs: 0 route_backend_conversations.CLIENTS = { 'storage_account_office_docs_client': FakeBlobServiceClient(blob_items), } app = Flask(__name__) app.config['TESTING'] = True + app.config['conversation_container'] = conversation_container + app.config['conversation_cache_bumps'] = cache_bumps + app.config['route_backend_conversations'] = route_backend_conversations route_backend_conversations.register_route_backend_conversations(app) return app, message_container, (lambda: None) @@ -556,6 +629,69 @@ def test_missing_image_preserves_not_found_response(): restore() +def test_mark_read_only_invalidates_cache_when_unread_state_changes(): + """Verify mark-read does not churn conversation cache for already-read conversations.""" + print("🔍 Testing mark-read cache invalidation idempotency...") + + app, _message_container, restore = build_test_app( + 'user-owner', + [ + { + 'id': 'conversation-unread', + 'user_id': 'user-owner', + 'has_unread_assistant_response': True, + 'last_unread_assistant_message_id': 'assistant-message-1', + 'last_unread_assistant_at': '2026-07-06T00:00:00Z', + }, + { + 'id': 'conversation-read', + 'user_id': 'user-owner', + 'has_unread_assistant_response': False, + 'last_unread_assistant_message_id': None, + 'last_unread_assistant_at': None, + }, + ], + [], + ) + + try: + conversation_container = app.config['conversation_container'] + cache_bumps = app.config['conversation_cache_bumps'] + conversation_container.upsert_count = 0 + + with app.test_client() as client: + unread_response = client.post('/api/conversations/conversation-unread/mark-read') + read_response = client.post('/api/conversations/conversation-read/mark-read') + + unread_payload = unread_response.get_json() + read_payload = read_response.get_json() + if unread_response.status_code != 200 or unread_payload.get('conversation_state_changed') is not True: + print(f"❌ Expected unread mark-read to change state: {unread_response.status_code} {unread_payload}") + return False + + if read_response.status_code != 200 or read_payload.get('conversation_state_changed') is not False: + print(f"❌ Expected already-read mark-read to be idempotent: {read_response.status_code} {read_payload}") + return False + + updated_unread = conversation_container.read_item('conversation-unread', 'conversation-unread') + if updated_unread.get('has_unread_assistant_response'): + print(f"❌ Unread conversation was not cleared: {updated_unread}") + return False + + if conversation_container.upsert_count != 1: + print(f"❌ Expected only one conversation upsert, got {conversation_container.upsert_count}") + return False + + if cache_bumps != [('user-owner', 'conversation_marked_read')]: + print(f"❌ Expected only unread conversation to invalidate cache, got {cache_bumps}") + return False + + print("✅ Mark-read cache invalidation is idempotent for already-read conversations") + return True + finally: + restore() + + if __name__ == '__main__': tests = [ test_owner_can_read_messages, @@ -565,6 +701,7 @@ def test_missing_image_preserves_not_found_response(): test_owner_can_stream_blob_backed_image, test_foreign_image_return_forbidden_before_query, test_missing_image_preserves_not_found_response, + test_mark_read_only_invalidates_cache_when_unread_state_changes, ] print('đŸ§Ē Running conversation read ownership authorization tests...') diff --git a/functional_tests/test_cosmos_phase3_shared_cache_metrics.py b/functional_tests/test_cosmos_phase3_shared_cache_metrics.py new file mode 100644 index 00000000..dc3c10bb --- /dev/null +++ b/functional_tests/test_cosmos_phase3_shared_cache_metrics.py @@ -0,0 +1,234 @@ +# test_cosmos_phase3_shared_cache_metrics.py +#!/usr/bin/env python3 +""" +Functional test for Phase 3 shared cache metrics. +Version: 0.250.037 +Implemented in: 0.250.032 +Cosmos fallback bypass implemented in: 0.250.037 + +This test ensures shared cache consumers record safe hit, miss, write, +delete, and version-bump metrics without exposing raw cache keys. +""" + +import copy +import importlib +import os +import sys +import types + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SINGLE_APP_DIR = os.path.join(ROOT_DIR, "application", "single_app") +if SINGLE_APP_DIR not in sys.path: + sys.path.insert(0, SINGLE_APP_DIR) + + +class FakeCosmosError(Exception): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class FakeCosmosContainer: + def __init__(self): + self.items = {} + self.read_count = 0 + self.delete_count = 0 + self._etag_counter = 0 + + def _copy_with_new_etag(self, body): + self._etag_counter += 1 + item = copy.deepcopy(body) + item["_etag"] = f"etag-{self._etag_counter}" + return item + + def read_item(self, item, partition_key): + self.read_count += 1 + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + return copy.deepcopy(self.items[item]) + + def create_item(self, body): + item_id = body["id"] + if item_id in self.items: + raise FakeCosmosError(409, f"Duplicate item {item_id}") + self.items[item_id] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[item_id]) + + def upsert_item(self, body): + self.items[body["id"]] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[body["id"]]) + + def replace_item(self, item, body, etag=None, match_condition=None, **kwargs): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + if etag and self.items[item].get("_etag") != etag: + raise FakeCosmosError(412, f"ETag mismatch for item {item}") + self.items[item] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[item]) + + def delete_item(self, item, partition_key, **kwargs): + self.delete_count += 1 + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + del self.items[item] + + +class FakeRedisClient: + def __init__(self): + self.values = {} + + def get(self, key): + return self.values.get(key) + + def set(self, key, value): + self.values[key] = value + return True + + def setex(self, key, ttl_seconds, value): + self.values[key] = value + return True + + def delete(self, key): + self.values.pop(key, None) + return True + + +def _load_shared_cache_module(settings_container): + fake_config = types.ModuleType("config") + fake_config.cosmos_settings_container = settings_container + sys.modules["config"] = fake_config + + fake_appinsights = types.ModuleType("functions_appinsights") + fake_appinsights.log_event = lambda *args, **kwargs: None + sys.modules["functions_appinsights"] = fake_appinsights + + sys.modules.pop("functions_shared_cache", None) + return importlib.import_module("functions_shared_cache") + + +def test_shared_cache_records_safe_metrics_for_common_operations(): + """Shared cache operations should emit counters with hashed cache-key context only.""" + settings_container = FakeCosmosContainer() + shared_cache = _load_shared_cache_module(settings_container) + shared_cache.reset_shared_cache_metrics() + + assert shared_cache.get_shared_cache_entry("phase3", "sensitive-user-key") is None + assert shared_cache.set_shared_cache_entry("phase3", "sensitive-user-key", {"value": 1}) is True + assert shared_cache.get_shared_cache_entry("phase3", "sensitive-user-key") == {"value": 1} + assert shared_cache.delete_shared_cache_entry("phase3", "sensitive-user-key") is True + + metrics = shared_cache.get_shared_cache_metrics() + counts = metrics["counts"] + + assert counts["phase3:get:cosmos:miss"] == 1 + assert counts["phase3:set:cosmos:success"] == 1 + assert counts["phase3:get:cosmos:hit"] == 1 + assert counts["phase3:delete:cosmos:success"] == 1 + assert "cache_key_hash" in metrics["last_event"] + assert "sensitive-user-key" not in str(metrics) + + +def test_shared_cache_records_version_bump_metrics(): + """Version reads and bumps should be visible in shared cache diagnostics.""" + settings_container = FakeCosmosContainer() + shared_cache = _load_shared_cache_module(settings_container) + shared_cache.reset_shared_cache_metrics() + + version = shared_cache.bump_shared_cache_version( + "phase3_cache_version", + description="Phase 3 test cache version.", + ) + + metrics = shared_cache.get_shared_cache_metrics() + + assert version == 1 + assert metrics["counts"]["shared_cache_version:version_bump:cosmos:success"] == 1 + assert metrics["last_event"]["operation"] == "version_bump" + assert metrics["last_event"]["version"] == 1 + assert "phase3_cache_version" not in str(metrics) + + +def test_shared_cache_can_skip_cosmos_fallback_for_volatile_entries(): + """High-churn caches should be able to bypass Cosmos when Redis is unavailable.""" + settings_container = FakeCosmosContainer() + shared_cache = _load_shared_cache_module(settings_container) + shared_cache.reset_shared_cache_metrics() + + assert shared_cache.get_shared_cache_entry( + "volatile", + "conversation-list", + allow_cosmos_fallback=False, + ) is None + assert shared_cache.set_shared_cache_entry( + "volatile", + "conversation-list", + {"value": 1}, + allow_cosmos_fallback=False, + ) is False + assert shared_cache.delete_shared_cache_entry( + "volatile", + "conversation-list", + allow_cosmos_fallback=False, + ) is True + + metrics = shared_cache.get_shared_cache_metrics() + counts = metrics["counts"] + + assert counts["volatile:get:cosmos:skipped"] == 1 + assert counts["volatile:set:cosmos:skipped"] == 1 + assert counts["volatile:delete:cosmos:skipped"] == 1 + assert settings_container.items == {} + + +def test_shared_cache_skips_cosmos_after_redis_miss_when_fallback_disabled(): + """Redis-backed volatile cache misses should not point-read/delete Cosmos fallback entries.""" + settings_container = FakeCosmosContainer() + shared_cache = _load_shared_cache_module(settings_container) + shared_cache.reset_shared_cache_metrics() + redis_client = FakeRedisClient() + + assert shared_cache.get_shared_cache_entry( + "volatile", + "conversation-list", + redis_client=redis_client, + allow_cosmos_fallback=False, + ) is None + assert shared_cache.delete_shared_cache_entry( + "volatile", + "conversation-list", + redis_client=redis_client, + allow_cosmos_fallback=False, + ) is True + + metrics = shared_cache.get_shared_cache_metrics() + counts = metrics["counts"] + + assert counts["volatile:get:redis:miss"] == 1 + assert counts["volatile:get:cosmos:skipped"] == 1 + assert counts["volatile:delete:redis:success"] == 1 + assert counts["volatile:delete:cosmos:skipped"] == 1 + assert settings_container.read_count == 0 + assert settings_container.delete_count == 0 + assert settings_container.items == {} + + +if __name__ == "__main__": + tests = [ + test_shared_cache_records_safe_metrics_for_common_operations, + test_shared_cache_records_version_bump_metrics, + test_shared_cache_can_skip_cosmos_fallback_for_volatile_entries, + test_shared_cache_skips_cosmos_after_redis_miss_when_fallback_disabled, + ] + results = [] + for test in tests: + print(f"Running {test.__name__}...") + try: + test() + print("Test passed.") + results.append(True) + except Exception as exc: + print(f"Test failed: {exc}") + results.append(False) + + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_cosmos_query_plugin.py b/functional_tests/test_cosmos_query_plugin.py index 2d568c8c..3477cc4a 100644 --- a/functional_tests/test_cosmos_query_plugin.py +++ b/functional_tests/test_cosmos_query_plugin.py @@ -2,8 +2,9 @@ #!/usr/bin/env python3 """ Functional test for the Cosmos query plugin. -Version: 0.241.024 +Version: 0.250.047 Implemented in: 0.241.024 +CosmosClient module import cleanup updated in: 0.250.047 This test ensures that the Cosmos query plugin enforces read-only query rules, normalizes query parameters, supports account-key authentication, exposes @@ -93,13 +94,64 @@ def get_database_client(self, database_name): return FakeCosmosSdkDatabaseClient() +class FakeConfigCosmosContainer: + """Minimal Cosmos container stand-in for importing config.py without live I/O.""" + + def read_item(self, item, partition_key=None): + if item == "app_settings": + return {"id": "app_settings", "settings": {}} + raise KeyError(item) + + def upsert_item(self, item): + return item + + def query_items(self, *args, **kwargs): + return [] + + +class FakeConfigCosmosDatabase: + """Minimal Cosmos database stand-in for config.py import-time container setup.""" + + def __init__(self): + self.containers = {} + + def create_container_if_not_exists(self, id, **kwargs): + self.containers.setdefault(id, FakeConfigCosmosContainer()) + return self.containers[id] + + +class FakeConfigCosmosClient: + """Minimal Cosmos client stand-in for config.py import-time client setup.""" + + def __init__(self, *args, **kwargs): + self.database = FakeConfigCosmosDatabase() + + def create_database_if_not_exists(self, *args, **kwargs): + return self.database + + +def import_app_module_without_live_cosmos(module_name): + """Import app modules without letting config.py connect to live Cosmos.""" + if module_name in sys.modules: + return sys.modules[module_name] + + import azure.cosmos as azure_cosmos + + original_cosmos_client = azure_cosmos.CosmosClient + azure_cosmos.CosmosClient = FakeConfigCosmosClient + try: + return importlib.import_module(module_name) + finally: + azure_cosmos.CosmosClient = original_cosmos_client + + def get_cosmos_query_plugin_class(): - module = importlib.import_module("semantic_kernel_plugins.cosmos_query_plugin") + module = import_app_module_without_live_cosmos("semantic_kernel_plugins.cosmos_query_plugin") return module.CosmosQueryPlugin def get_discover_plugins_function(): - module = importlib.import_module("semantic_kernel_plugins.plugin_loader") + module = import_app_module_without_live_cosmos("semantic_kernel_plugins.plugin_loader") return module.discover_plugins @@ -177,10 +229,10 @@ def test_cosmos_query_key_auth_uses_account_key_credentials(): CosmosQueryPlugin = None try: - module = importlib.import_module("semantic_kernel_plugins.cosmos_query_plugin") + module = import_app_module_without_live_cosmos("semantic_kernel_plugins.cosmos_query_plugin") CosmosQueryPlugin = module.CosmosQueryPlugin - original_client = module.CosmosClient - module.CosmosClient = FakeCosmosSdkClient + original_client = module.azure_cosmos.CosmosClient + module.azure_cosmos.CosmosClient = FakeCosmosSdkClient FakeCosmosSdkClient.reset() CosmosQueryPlugin._client_cache = {} @@ -201,7 +253,7 @@ def test_cosmos_query_key_auth_uses_account_key_credentials(): return False finally: if module is not None and original_client is not None: - module.CosmosClient = original_client + module.azure_cosmos.CosmosClient = original_client if CosmosQueryPlugin is not None: CosmosQueryPlugin._client_cache = {} diff --git a/functional_tests/test_cosmos_throughput_refresh_logging.py b/functional_tests/test_cosmos_throughput_refresh_logging.py index c5b1e276..f548e222 100644 --- a/functional_tests/test_cosmos_throughput_refresh_logging.py +++ b/functional_tests/test_cosmos_throughput_refresh_logging.py @@ -1,13 +1,15 @@ -#!/usr/bin/env python3 # test_cosmos_throughput_refresh_logging.py +#!/usr/bin/env python3 """ Functional test for Cosmos throughput refresh logging. -Version: 0.241.149 +Version: 0.250.037 Implemented in: 0.241.149 +Read-only status refresh fixed in: 0.250.037 This test ensures that Admin Settings Cosmos throughput refreshes emit route-level and helper-level backend logs with a refresh correlation ID and -phase timing markers. +phase timing markers, while status refresh and no-op autoscale checks do not +persist runtime status into app settings. """ import os @@ -60,10 +62,35 @@ def test_refresh_helper_logs_backend_phases(): assert marker in source, f"Missing helper logging marker: {marker}" +def test_status_refresh_does_not_persist_runtime_settings(): + """The admin Cosmos throughput status GET should be read-only.""" + source = _read_file(ROUTE_FILE) + route_marker = "@bp.route('/api/admin/settings/cosmos-throughput/status', methods=['GET'])" + next_route_marker = "@bp.route('/api/admin/settings/cosmos-throughput/validate-access'" + route_start = source.index(route_marker) + route_end = source.index(next_route_marker, route_start) + route_source = source[route_start:route_end] + + assert "get_cosmos_throughput_status(" in route_source + assert "update_settings(" not in route_source + assert "build_runtime_update(" not in route_source + + +def test_noop_autoscale_checks_do_not_build_settings_update(): + """Background autoscale should only return settings_update after an actual scale action.""" + source = _read_file(HELPER_FILE) + + assert "settings_update = None" in source + assert "if scale_result:" in source + assert "settings_update = build_runtime_update(status, decision, scale_result, settings=settings)" in source + + if __name__ == "__main__": tests = [ test_refresh_route_logs_request_lifecycle, test_refresh_helper_logs_backend_phases, + test_status_refresh_does_not_persist_runtime_settings, + test_noop_autoscale_checks_do_not_build_settings_update, ] results = [] for test in tests: diff --git a/functional_tests/test_cosmos_wave1_app_maintenance.py b/functional_tests/test_cosmos_wave1_app_maintenance.py new file mode 100644 index 00000000..655ee07b --- /dev/null +++ b/functional_tests/test_cosmos_wave1_app_maintenance.py @@ -0,0 +1,404 @@ +# test_cosmos_wave1_app_maintenance.py +#!/usr/bin/env python3 +""" +Functional test for Cosmos Wave 1 app maintenance framework. +Version: 0.250.043 +Implemented in: 0.250.005 +Conversation cache metrics updated in: 0.250.034 +Stale cache cleanup maintenance updated in: 0.250.038 +DAI version marker TTL hygiene updated in: 0.250.043 + +This test ensures the maintenance runner initializes cache version documents +and records durable run state without requiring live Azure resources. +""" + +import copy +import importlib +import os +import sys +import types + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SINGLE_APP_DIR = os.path.join(ROOT_DIR, "application", "single_app") +if SINGLE_APP_DIR not in sys.path: + sys.path.insert(0, SINGLE_APP_DIR) + + +class FakeCosmosError(Exception): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class FakeCosmosContainer: + def __init__(self): + self.items = {} + self._etag_counter = 0 + self.container_properties = { + "id": "fake-container", + "_etag": "container-etag", + "indexingPolicy": { + "indexingMode": "consistent", + "automatic": True, + "includedPaths": [{"path": "/*"}], + "excludedPaths": [{"path": "/\"_etag\"/?"}], + "compositeIndexes": [], + }, + } + + def _copy_with_new_etag(self, body): + self._etag_counter += 1 + item = copy.deepcopy(body) + item["_etag"] = f"etag-{self._etag_counter}" + return item + + def read_item(self, item, partition_key): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + return copy.deepcopy(self.items[item]) + + def read(self): + return copy.deepcopy(self.container_properties) + + def create_item(self, body): + item_id = body["id"] + if item_id in self.items: + raise FakeCosmosError(409, f"Duplicate item {item_id}") + self.items[item_id] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[item_id]) + + def upsert_item(self, body): + self.items[body["id"]] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[body["id"]]) + + def replace_item(self, item, body, etag=None, match_condition=None, **kwargs): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + if etag and self.items[item].get("_etag") != etag: + raise FakeCosmosError(412, f"ETag mismatch for item {item}") + self.items[item] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[item]) + + def delete_item(self, item, partition_key, **kwargs): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + del self.items[item] + + def query_items(self, query, parameters=None, enable_cross_partition_query=False, **kwargs): + parameter_map = { + parameter["name"]: parameter["value"] + for parameter in list(parameters or []) + } + item_type = parameter_map.get("@type") + item_status = parameter_map.get("@status") + document_metadata_type = parameter_map.get("@document_metadata_type") + results = [] + for item in self.items.values(): + if item_type is not None and item.get("type") != item_type: + continue + if item_status is not None and item.get("status") != item_status: + continue + if document_metadata_type is not None and item.get("type") not in (None, document_metadata_type): + continue + results.append(copy.deepcopy(item)) + if "SELECT VALUE COUNT" in query.upper(): + return [len(results)] + return results + + +class FakeCosmosDatabase: + def replace_container(self, **kwargs): + return kwargs + + +def _load_maintenance_module(container, governance_container=None): + fake_config = types.ModuleType("config") + fake_config.VERSION = "0.250.043" + fake_config.cosmos_settings_container = container + fake_config.cosmos_governance_policies_container = governance_container or container + fake_config.cosmos_document_access_index_container = FakeCosmosContainer() + fake_config.cosmos_groups_container = FakeCosmosContainer() + fake_config.cosmos_public_workspaces_container = FakeCosmosContainer() + fake_config.cosmos_user_settings_container = FakeCosmosContainer() + fake_config.cosmos_database = FakeCosmosDatabase() + for name in [ + "cosmos_collaboration_messages", + "cosmos_conversations", + "cosmos_group_documents", + "cosmos_messages", + "cosmos_public_documents", + "cosmos_user_documents", + ]: + setattr(fake_config, f"{name}_container", container) + setattr(fake_config, f"{name}_container_name", name.replace("cosmos_", "")) + sys.modules["config"] = fake_config + + fake_appinsights = types.ModuleType("functions_appinsights") + fake_appinsights.log_event = lambda *args, **kwargs: None + fake_appinsights.debug_print = lambda *args, **kwargs: None + fake_appinsights.is_debug_enabled = lambda: False + sys.modules["functions_appinsights"] = fake_appinsights + + fake_app_settings_cache = types.ModuleType("app_settings_cache") + fake_app_settings_cache.get_app_cache_redis_client = lambda: None + sys.modules["app_settings_cache"] = fake_app_settings_cache + + fake_group = types.ModuleType("functions_group") + fake_group.find_group_by_id = lambda group_id: None + sys.modules["functions_group"] = fake_group + + fake_settings = types.ModuleType("functions_settings") + fake_settings.get_settings = lambda: {} + sys.modules["functions_settings"] = fake_settings + + sys.modules.pop("functions_shared_cache", None) + sys.modules.pop("functions_conversation_cache", None) + sys.modules.pop("functions_cosmos_stale_cleanup", None) + sys.modules.pop("functions_cosmos_indexing", None) + sys.modules.pop("functions_document_access_index", None) + sys.modules.pop("functions_app_maintenance", None) + return importlib.import_module("functions_app_maintenance") + + +def test_maintenance_initializes_cache_version_documents(): + """Maintenance should create all Wave 1 cache version documents.""" + container = FakeCosmosContainer() + governance_container = FakeCosmosContainer() + maintenance = _load_maintenance_module(container, governance_container=governance_container) + + result = maintenance.run_app_maintenance_once(triggered_by="test", requested_by="tester@example.com") + + assert result["success"] is True + assert container.items["app_maintenance_state"]["last_status"] == "succeeded" + backfill_step = next(step for step in result["steps"] if step["name"] == "document_access_index_backfill") + assert backfill_step["status"] == "succeeded" + assert backfill_step["results"]["status"] == "completed" + assert backfill_step["results"]["current_status"]["maintenance"]["next_action"] == "monitor" + for version_doc in maintenance.CACHE_VERSION_DOCUMENTS: + target_container = governance_container if version_doc["container"] == "governance_policies" else container + item = target_container.items[version_doc["id"]] + assert item["type"] == "cache_version" + assert item["version"] == 0 + + +def test_maintenance_status_reports_state_and_versions(): + """Maintenance status should include durable state and version doc status.""" + container = FakeCosmosContainer() + governance_container = FakeCosmosContainer() + maintenance = _load_maintenance_module(container, governance_container=governance_container) + maintenance.run_app_maintenance_once(triggered_by="test", requested_by="tester@example.com") + + status = maintenance.get_app_maintenance_status() + + assert status["success"] is True + assert status["state"]["last_status"] == "succeeded" + assert len(status["cache_version_documents"]) == len(maintenance.CACHE_VERSION_DOCUMENTS) + assert isinstance(status["shared_cache_metrics"], dict) + assert "counts" in status["shared_cache_metrics"] + assert status["conversation_cache"]["settings"]["enabled"] is True + assert status["conversation_cache"]["settings"]["ttl_seconds"] == 120 + assert "15m" in status["conversation_cache"]["metrics"]["windows"] + assert status["cosmos_indexing_policies"]["mode"] == "report_only" + assert status["stale_cache_cleanup"]["status"] == "skipped_disabled" + assert status["stale_cache_cleanup"]["candidate_count"] == 0 + assert status["document_access_index_backfill"]["state"]["status"] == "succeeded" + assert status["document_access_index_backfill"]["maintenance"]["auto_maintenance_enabled"] is True + assert { + "app_settings_cache_version", + "governance_cache_version", + "document_access_index_projection_version", + }.issubset({item["id"] for item in status["cache_version_documents"]}) + + +def test_maintenance_settings_are_normalized(): + """Maintenance settings should enforce safe minimum intervals and lease values.""" + container = FakeCosmosContainer() + maintenance = _load_maintenance_module(container) + + settings = maintenance.get_app_maintenance_settings({ + "enable_app_maintenance": True, + "enable_startup_app_maintenance": True, + "app_maintenance_check_interval_seconds": 5, + "app_maintenance_job_lease_seconds": 10, + }) + + assert settings["enabled"] is True + assert settings["run_on_startup"] is True + assert settings["check_interval_seconds"] == 60 + assert settings["lease_seconds"] == 60 + assert settings["run_document_access_index_backfill"] is True + assert settings["document_access_index_auto_maintenance"] is True + assert settings["document_access_index_active_interval_seconds"] == 30 + + disabled_settings = maintenance.get_app_maintenance_settings({ + "enable_app_maintenance": False, + "enable_startup_app_maintenance": True, + }) + startup_disabled_settings = maintenance.get_app_maintenance_settings({ + "enable_app_maintenance": True, + "enable_startup_app_maintenance": False, + }) + + assert disabled_settings["run_document_access_index_backfill"] is True + assert disabled_settings["document_access_index_auto_maintenance"] is False + assert startup_disabled_settings["run_document_access_index_backfill"] is True + assert startup_disabled_settings["document_access_index_auto_maintenance"] is False + + +def test_explicit_backfill_skip_is_preserved_for_manual_runs(): + """Manual maintenance calls should be able to skip a backfill batch for that run.""" + container = FakeCosmosContainer() + maintenance = _load_maintenance_module(container) + + result = maintenance.run_app_maintenance_once( + triggered_by="test", + requested_by="tester@example.com", + run_document_access_backfill=False, + ) + + backfill_step = next(step for step in result["steps"] if step["name"] == "document_access_index_backfill") + assert backfill_step["run_requested"] is False + assert backfill_step["results"]["backfill"]["status"] == "skipped_disabled" + assert backfill_step["results"]["maintenance_pending"] is True + + +def _seed_stale_cleanup_documents(container): + container.items["conversation_cache_version:user-1"] = { + "id": "conversation_cache_version:user-1", + "type": "cache_version", + "version": 42, + } + container.items["shared_cache_entry:conversation_cache:list:user-1:abc"] = { + "id": "shared_cache_entry:conversation_cache:list:user-1:abc", + "type": "shared_cache_entry", + "namespace": "conversation_cache", + "key": "list:user-1:abc", + } + container.items["shared_cache_entry:chat_bootstrap:user:user-1:abc"] = { + "id": "shared_cache_entry:chat_bootstrap:user:user-1:abc", + "type": "shared_cache_entry", + "namespace": "chat_bootstrap", + "key": "user:user-1:abc", + } + container.items["shared_cache_entry:custom_pages:old"] = { + "id": "shared_cache_entry:custom_pages:old", + "type": "shared_cache_entry", + "namespace": "custom_pages", + "key": "old", + "expires_at": "2000-01-01T00:00:00+00:00", + } + container.items["shared_cache_entry:custom_pages:active"] = { + "id": "shared_cache_entry:custom_pages:active", + "type": "shared_cache_entry", + "namespace": "custom_pages", + "key": "active", + "expires_at": "2999-01-01T00:00:00+00:00", + } + container.items["chat_bootstrap_global_cache_version"] = { + "id": "chat_bootstrap_global_cache_version", + "type": "cache_version", + "version": 10, + } + container.items["app_settings"] = { + "id": "app_settings", + "type": "app_settings", + } + + +def test_stale_cache_cleanup_dry_run_preserves_candidates(): + """Dry-run cleanup should report allowlisted stale docs without deleting them.""" + container = FakeCosmosContainer() + _seed_stale_cleanup_documents(container) + maintenance = _load_maintenance_module(container) + + result = maintenance.run_app_maintenance_once( + triggered_by="test", + requested_by="tester@example.com", + run_document_access_backfill=False, + run_stale_cache_cleanup=True, + apply_stale_cache_cleanup=False, + ) + + cleanup_step = next(step for step in result["steps"] if step["name"] == "stale_cache_document_cleanup") + assert result["success"] is True + assert cleanup_step["run_requested"] is True + assert cleanup_step["apply_requested"] is False + assert cleanup_step["results"]["status"] == "dry_run_completed" + assert cleanup_step["results"]["candidate_count"] == 4 + assert cleanup_step["results"]["deleted_count"] == 0 + assert "conversation_cache_version:user-1" in container.items + assert "shared_cache_entry:conversation_cache:list:user-1:abc" in container.items + assert "shared_cache_entry:chat_bootstrap:user:user-1:abc" in container.items + assert "shared_cache_entry:custom_pages:old" in container.items + + +def test_stale_cache_cleanup_apply_deletes_only_allowlisted_documents(): + """Apply cleanup should delete stale cache artifacts and preserve active settings docs.""" + container = FakeCosmosContainer() + _seed_stale_cleanup_documents(container) + maintenance = _load_maintenance_module(container) + + result = maintenance.run_app_maintenance_once( + triggered_by="test", + requested_by="tester@example.com", + run_document_access_backfill=False, + run_stale_cache_cleanup=True, + apply_stale_cache_cleanup=True, + ) + + cleanup_step = next(step for step in result["steps"] if step["name"] == "stale_cache_document_cleanup") + assert result["success"] is True + assert cleanup_step["results"]["status"] == "completed" + assert cleanup_step["results"]["candidate_count"] == 4 + assert cleanup_step["results"]["deleted_count"] == 4 + assert "conversation_cache_version:user-1" not in container.items + assert "shared_cache_entry:conversation_cache:list:user-1:abc" not in container.items + assert "shared_cache_entry:chat_bootstrap:user:user-1:abc" not in container.items + assert "shared_cache_entry:custom_pages:old" not in container.items + assert "shared_cache_entry:custom_pages:active" in container.items + assert "chat_bootstrap_global_cache_version" in container.items + assert "app_settings" in container.items + + +def test_failed_maintenance_step_sets_top_level_failure(): + """A failed step should not be reported as a fully successful maintenance run.""" + container = FakeCosmosContainer() + maintenance = _load_maintenance_module(container) + maintenance.run_document_access_index_backfill_maintenance = lambda **_kwargs: { + "success": False, + "status": "failed", + "error": "simulated failure", + } + + result = maintenance.run_app_maintenance_once(triggered_by="test", requested_by="tester@example.com") + + backfill_step = next(step for step in result["steps"] if step["name"] == "document_access_index_backfill") + assert result["success"] is False + assert result["error"] == "One or more maintenance steps failed." + assert result["state"]["last_status"] == "succeeded_with_warnings" + assert backfill_step["status"] == "failed" + + +if __name__ == "__main__": + tests = [ + test_maintenance_initializes_cache_version_documents, + test_maintenance_status_reports_state_and_versions, + test_maintenance_settings_are_normalized, + test_explicit_backfill_skip_is_preserved_for_manual_runs, + test_stale_cache_cleanup_dry_run_preserves_candidates, + test_stale_cache_cleanup_apply_deletes_only_allowlisted_documents, + test_failed_maintenance_step_sets_top_level_failure, + ] + results = [] + for test in tests: + print(f"Running {test.__name__}...") + try: + test() + print("Test passed.") + results.append(True) + except Exception as exc: + print(f"Test failed: {exc}") + results.append(False) + + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_cosmos_wave1_cache_fallback.py b/functional_tests/test_cosmos_wave1_cache_fallback.py new file mode 100644 index 00000000..8e40e94b --- /dev/null +++ b/functional_tests/test_cosmos_wave1_cache_fallback.py @@ -0,0 +1,206 @@ +# test_cosmos_wave1_cache_fallback.py +#!/usr/bin/env python3 +""" +Functional test for Cosmos Wave 1 cache fallback behavior. +Version: 0.250.005 +Implemented in: 0.250.005 + +This test ensures Redis failures in the app cache layer fall back to +Cosmos-backed cache/source reads instead of failing callers. +""" + +import copy +import importlib +import os +import sys +import types + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SINGLE_APP_DIR = os.path.join(ROOT_DIR, "application", "single_app") +if SINGLE_APP_DIR not in sys.path: + sys.path.insert(0, SINGLE_APP_DIR) + + +class FakeCosmosError(Exception): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class FakeCosmosContainer: + def __init__(self): + self.items = {} + self._etag_counter = 0 + + def _copy_with_new_etag(self, body): + self._etag_counter += 1 + item = copy.deepcopy(body) + item["_etag"] = f"etag-{self._etag_counter}" + return item + + def read_item(self, item, partition_key): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + return copy.deepcopy(self.items[item]) + + def create_item(self, body): + item_id = body["id"] + if item_id in self.items: + raise FakeCosmosError(409, f"Duplicate item {item_id}") + self.items[item_id] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[item_id]) + + def upsert_item(self, body): + self.items[body["id"]] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[body["id"]]) + + def replace_item(self, item, body, etag=None, match_condition=None, **kwargs): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + if etag and self.items[item].get("_etag") != etag: + raise FakeCosmosError(412, f"ETag mismatch for item {item}") + self.items[item] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[item]) + + def delete_item(self, item, partition_key, **kwargs): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + del self.items[item] + + +class FailingRedis: + def __init__(self, *args, **kwargs): + pass + + def get(self, *args, **kwargs): + raise RuntimeError("redis unavailable") + + def set(self, *args, **kwargs): + raise RuntimeError("redis unavailable") + + def setex(self, *args, **kwargs): + raise RuntimeError("redis unavailable") + + def setnx(self, *args, **kwargs): + raise RuntimeError("redis unavailable") + + def incr(self, *args, **kwargs): + raise RuntimeError("redis unavailable") + + def delete(self, *args, **kwargs): + raise RuntimeError("redis unavailable") + + def pipeline(self): + raise RuntimeError("redis unavailable") + + +class RaisingRedis(FailingRedis): + def __init__(self, *args, **kwargs): + raise RuntimeError("redis initialization failed") + + +def _install_fake_modules(container): + fake_config = types.ModuleType("config") + fake_config.cosmos_settings_container = container + fake_config.exceptions = types.SimpleNamespace() + sys.modules["config"] = fake_config + + fake_appinsights = types.ModuleType("functions_appinsights") + fake_appinsights.log_event = lambda *args, **kwargs: None + sys.modules["functions_appinsights"] = fake_appinsights + + +def _load_cache_module(container): + _install_fake_modules(container) + sys.modules.pop("app_settings_cache", None) + return importlib.import_module("app_settings_cache") + + +def test_redis_runtime_failure_falls_back_to_cosmos_settings(): + """A Redis read failure should return settings from Cosmos instead of raising.""" + container = FakeCosmosContainer() + container.items["app_settings"] = { + "id": "app_settings", + "feature_flag": "from-cosmos", + } + container.items["app_settings_cache_version"] = { + "id": "app_settings_cache_version", + "type": "cache_version", + "version": 7, + } + cache_module = _load_cache_module(container) + cache_module.Redis = FailingRedis + + cache_module.configure_app_cache({ + "enable_redis_cache": True, + "redis_url": "simplechat.redis.cache.windows.net", + "redis_key": "test-key", + "redis_auth_type": "key", + }) + + cached_settings = cache_module.get_settings_cache() + + assert cached_settings["feature_flag"] == "from-cosmos" + assert cache_module.get_app_settings_cache_version() == 7 + + +def test_redis_write_failure_persists_user_ui_cache_to_cosmos(): + """A Redis write failure should persist lightweight UI cache data in Cosmos.""" + container = FakeCosmosContainer() + cache_module = _load_cache_module(container) + cache_module.Redis = FailingRedis + + cache_module.configure_app_cache({ + "enable_redis_cache": True, + "redis_url": "simplechat.redis.cache.windows.net", + "redis_key": "test-key", + "redis_auth_type": "key", + }) + + cache_module.set_user_ui_settings_cache("user-1", {"theme": "dark"}, ttl_seconds=60) + cached_settings = cache_module.get_user_ui_settings_cache("user-1") + + assert cached_settings == {"theme": "dark"} + assert "app_cache_entry:USER_UI_SETTINGS:user-1" in container.items + + +def test_redis_initialization_failure_assigns_fallback_functions(): + """A Redis startup failure should not break app cache configuration.""" + container = FakeCosmosContainer() + container.items["app_settings"] = { + "id": "app_settings", + "feature_flag": "fallback-configured", + } + cache_module = _load_cache_module(container) + cache_module.Redis = RaisingRedis + + cache_module.configure_app_cache({ + "enable_redis_cache": True, + "redis_url": "simplechat.redis.cache.windows.net", + "redis_key": "test-key", + "redis_auth_type": "key", + }) + + assert cache_module.app_cache_is_using_redis is False + assert cache_module.get_settings_cache()["feature_flag"] == "fallback-configured" + + +if __name__ == "__main__": + tests = [ + test_redis_runtime_failure_falls_back_to_cosmos_settings, + test_redis_write_failure_persists_user_ui_cache_to_cosmos, + test_redis_initialization_failure_assigns_fallback_functions, + ] + results = [] + for test in tests: + print(f"Running {test.__name__}...") + try: + test() + print("Test passed.") + results.append(True) + except Exception as exc: + print(f"Test failed: {exc}") + results.append(False) + + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_cosmos_wave2a_chat_bootstrap_cache.py b/functional_tests/test_cosmos_wave2a_chat_bootstrap_cache.py new file mode 100644 index 00000000..29774320 --- /dev/null +++ b/functional_tests/test_cosmos_wave2a_chat_bootstrap_cache.py @@ -0,0 +1,365 @@ +# test_cosmos_wave2a_chat_bootstrap_cache.py +#!/usr/bin/env python3 +""" +Functional test for Cosmos Wave 2A chat bootstrap cache. +Version: 0.250.037 +Implemented in: 0.250.006 +Settings write invalidation scoped in: 0.250.037 + +This test ensures chat bootstrap cache keys are versioned and invalidated by +global and per-user cache version bumps without relying on generic settings +write invalidation. +""" + +import copy +import importlib +import os +import sys +import types +from datetime import datetime + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SINGLE_APP_DIR = os.path.join(ROOT_DIR, "application", "single_app") +if SINGLE_APP_DIR not in sys.path: + sys.path.insert(0, SINGLE_APP_DIR) + + +class FakeCosmosError(Exception): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class FakeCosmosContainer: + def __init__(self): + self.items = {} + self._etag_counter = 0 + + def _copy_with_new_etag(self, body): + self._etag_counter += 1 + item = copy.deepcopy(body) + item["_etag"] = f"etag-{self._etag_counter}" + return item + + def read_item(self, item, partition_key): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + return copy.deepcopy(self.items[item]) + + def create_item(self, body): + item_id = body["id"] + if item_id in self.items: + raise FakeCosmosError(409, f"Duplicate item {item_id}") + self.items[item_id] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[item_id]) + + def upsert_item(self, body): + self.items[body["id"]] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[body["id"]]) + + def replace_item(self, item, body, etag=None, match_condition=None, **kwargs): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + if etag and self.items[item].get("_etag") != etag: + raise FakeCosmosError(412, f"ETag mismatch for item {item}") + self.items[item] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[item]) + + def delete_item(self, item, partition_key, **kwargs): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + del self.items[item] + + +class FakeRedisClient: + def __init__(self): + self.values = {} + + def get(self, key): + return self.values.get(key) + + def set(self, key, value): + self.values[key] = value + return True + + def setex(self, key, ttl_seconds, value): + self.values[key] = value + return True + + def delete(self, key): + self.values.pop(key, None) + return True + + +def _load_chat_bootstrap_module(settings_container, redis_client=None): + fake_config = types.ModuleType("config") + fake_config.cosmos_settings_container = settings_container + sys.modules["config"] = fake_config + + fake_appinsights = types.ModuleType("functions_appinsights") + fake_appinsights.log_event = lambda *args, **kwargs: None + sys.modules["functions_appinsights"] = fake_appinsights + + fake_app_settings_cache = types.ModuleType("app_settings_cache") + fake_app_settings_cache.get_app_cache_redis_client = lambda: redis_client + fake_app_settings_cache.get_app_settings_cache_version = lambda: 0 + fake_app_settings_cache.get_governance_cache_version = lambda: 0 + sys.modules["app_settings_cache"] = fake_app_settings_cache + + for module_name in [ + "functions_shared_cache", + "functions_chat_bootstrap_cache", + ]: + sys.modules.pop(module_name, None) + return importlib.import_module("functions_chat_bootstrap_cache") + + +def _load_public_workspaces_module(public_workspaces_container, invalidation_reasons): + fake_config = types.ModuleType("config") + fake_config.cosmos_public_workspaces_container = public_workspaces_container + fake_config.datetime = datetime + fake_config.exceptions = types.SimpleNamespace(CosmosResourceNotFoundError=FakeCosmosError) + sys.modules["config"] = fake_config + + fake_group = types.ModuleType("functions_group") + sys.modules["functions_group"] = fake_group + + fake_authentication = types.ModuleType("functions_authentication") + fake_authentication.get_current_user_info = lambda: None + sys.modules["functions_authentication"] = fake_authentication + + fake_settings = types.ModuleType("functions_settings") + sys.modules["functions_settings"] = fake_settings + + fake_branding = types.ModuleType("functions_workspace_branding") + fake_branding.DEFAULT_WORKSPACE_HERO_COLOR = "#000000" + fake_branding.get_workspace_logo_metadata = lambda *args, **kwargs: {} + fake_branding.normalize_workspace_hero_color = lambda value: value or "#000000" + sys.modules["functions_workspace_branding"] = fake_branding + + fake_bootstrap_cache = types.ModuleType("functions_chat_bootstrap_cache") + fake_bootstrap_cache.bump_chat_bootstrap_global_cache_version = ( + lambda reason=None: invalidation_reasons.append(reason) + ) + sys.modules["functions_chat_bootstrap_cache"] = fake_bootstrap_cache + + sys.modules.pop("functions_public_workspaces", None) + return importlib.import_module("functions_public_workspaces") + + +def _cache_inputs(): + return { + "settings": { + "allow_user_agents": True, + "enable_semantic_kernel": True, + "enable_group_workspaces": True, + "allow_group_agents": True, + "enable_multi_model_endpoints": True, + "allow_user_custom_endpoints": True, + "allow_group_custom_endpoints": True, + "enable_public_workspaces": True, + "enable_user_workspace": True, + "allow_user_plugins": True, + "allow_group_plugins": True, + }, + "user_settings_dict": { + "personal_model_endpoints": [{"id": "personal-endpoint"}], + }, + "user_groups_raw": [{"id": "group-1", "name": "Group One"}], + "user_visible_public_workspaces": [{"id": "public-1", "name": "Public One"}], + } + + +def test_chat_bootstrap_cache_key_changes_after_global_bump(): + """A global version bump should change the chat bootstrap cache key.""" + settings_container = FakeCosmosContainer() + bootstrap_cache = _load_chat_bootstrap_module(settings_container, redis_client=FakeRedisClient()) + inputs = _cache_inputs() + + first_key = bootstrap_cache.build_chat_bootstrap_cache_key("user-1", **inputs) + bootstrap_cache.set_cached_chat_bootstrap_payload( + first_key, + { + "chat_agent_options": [{"id": "agent-1"}], + "chat_model_options": [], + "chat_prompt_options": [], + }, + ttl_seconds=300, + ) + bootstrap_cache.bump_chat_bootstrap_global_cache_version(reason="test") + second_key = bootstrap_cache.build_chat_bootstrap_cache_key("user-1", **inputs) + + assert first_key != second_key + assert bootstrap_cache.get_cached_chat_bootstrap_payload(first_key)["chat_agent_options"][0]["id"] == "agent-1" + assert bootstrap_cache.get_cached_chat_bootstrap_payload(second_key) is None + + +def test_chat_bootstrap_cache_key_changes_after_user_bump(): + """A user version bump should only change that user's cache key.""" + settings_container = FakeCosmosContainer() + bootstrap_cache = _load_chat_bootstrap_module(settings_container, redis_client=FakeRedisClient()) + inputs = _cache_inputs() + + user_one_before = bootstrap_cache.build_chat_bootstrap_cache_key("user-1", **inputs) + user_two_before = bootstrap_cache.build_chat_bootstrap_cache_key("user-2", **inputs) + bootstrap_cache.bump_chat_bootstrap_user_cache_version("user-1", reason="test") + user_one_after = bootstrap_cache.build_chat_bootstrap_cache_key("user-1", **inputs) + user_two_after = bootstrap_cache.build_chat_bootstrap_cache_key("user-2", **inputs) + + assert user_one_before != user_one_after + assert user_two_before == user_two_after + + +def test_chat_bootstrap_route_and_write_hooks_are_wired(): + """Contract test for route cache usage and write-path invalidation hooks.""" + route_source = open( + os.path.join(SINGLE_APP_DIR, "route_frontend_chats.py"), + "r", + encoding="utf-8", + ).read() + assert "build_chat_bootstrap_cache_key" in route_source + assert "get_cached_chat_bootstrap_payload" in route_source + assert "set_cached_chat_bootstrap_payload" in route_source + + for file_name, marker in { + "functions_global_agents.py": "bump_chat_bootstrap_global_cache_version(reason=\"global_agent_saved\")", + "functions_personal_agents.py": "bump_chat_bootstrap_user_cache_version(user_id, reason=\"personal_agent_saved\")", + "functions_group_agents.py": "bump_chat_bootstrap_global_cache_version(reason=\"group_agent_saved\")", + "functions_global_actions.py": "bump_chat_bootstrap_global_cache_version(reason=\"global_action_saved\")", + "functions_personal_actions.py": "bump_chat_bootstrap_user_cache_version(user_id, reason=\"personal_action_saved\")", + "functions_group_actions.py": "bump_chat_bootstrap_global_cache_version(reason=\"group_action_saved\")", + "functions_prompts.py": "_invalidate_prompt_chat_bootstrap_cache(", + }.items(): + source = open(os.path.join(SINGLE_APP_DIR, file_name), "r", encoding="utf-8").read() + assert marker in source, f"Missing invalidation hook marker in {file_name}: {marker}" + if file_name == "functions_settings.py": + assert source.count("def _refresh_app_settings_cache_after_write(") == 1 + + settings_source = open(os.path.join(SINGLE_APP_DIR, "functions_settings.py"), "r", encoding="utf-8").read() + assert "bump_chat_bootstrap_global_cache_version(reason=f\"settings_write:{context}\")" not in settings_source + + +def test_phase3_low_churn_invalidation_hooks_are_wired(): + """Contract test for Phase 3 group/public/custom-pages invalidation coverage.""" + expected_markers = { + "functions_group.py": [ + "bump_chat_bootstrap_global_cache_version(reason=\"group_created\")", + "bump_chat_bootstrap_global_cache_version(reason=\"group_deleted\")", + "bump_chat_bootstrap_global_cache_version(reason=\"group_model_endpoints_updated\")", + ], + "functions_public_workspaces.py": [ + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_created\")", + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_deleted\")", + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_document_manager_added\")", + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_document_manager_removed\")", + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_document_manager_request_approved\")", + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_document_manager_request_rejected\")", + ], + "functions_simplechat_operations.py": [ + "bump_chat_bootstrap_global_cache_version(reason=\"group_marked_inactive\")", + "bump_chat_bootstrap_global_cache_version(reason=\"group_member_added\")", + ], + "route_backend_groups.py": [ + "bump_chat_bootstrap_global_cache_version(reason=\"group_updated\")", + "bump_chat_bootstrap_global_cache_version(reason=\"group_member_request_approved\")", + "bump_chat_bootstrap_global_cache_version(reason=\"group_member_removed\")", + "bump_chat_bootstrap_global_cache_version(reason=\"group_member_role_updated\")", + "bump_chat_bootstrap_global_cache_version(reason=\"group_ownership_transferred\")", + ], + "route_backend_public_workspaces.py": [ + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_updated\")", + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_member_request_approved\")", + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_member_added\")", + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_member_removed\")", + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_member_role_updated\")", + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_ownership_transferred\")", + ], + "route_backend_control_center.py": [ + "bump_chat_bootstrap_global_cache_version(reason=\"group_status_updated\")", + "bump_chat_bootstrap_global_cache_version(reason=\"group_member_added\")", + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_status_updated\")", + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_bulk_status_updated\")", + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_member_added\")", + "bump_chat_bootstrap_global_cache_version(reason=\"group_ownership_transferred\")", + "bump_chat_bootstrap_global_cache_version(reason=\"public_workspace_ownership_transferred\")", + ], + } + + for file_name, markers in expected_markers.items(): + source = open(os.path.join(SINGLE_APP_DIR, file_name), "r", encoding="utf-8").read() + for marker in markers: + assert marker in source, f"Missing Phase 3 invalidation hook in {file_name}: {marker}" + + groups_source = open(os.path.join(SINGLE_APP_DIR, "route_backend_groups.py"), "r", encoding="utf-8").read() + group_update_route = groups_source[ + groups_source.index("def api_update_group(group_id):"): + groups_source.index("def api_get_group_logo(group_id):") + ] + assert "cosmos_groups_container.upsert_item(group_doc)" in group_update_route + assert "bump_chat_bootstrap_global_cache_version(reason=\"group_updated\")" in group_update_route + + +def test_chat_bootstrap_payload_cache_does_not_fallback_to_settings_container_without_redis(): + """Volatile chat bootstrap payloads should not write shared cache entries to settings without Redis.""" + settings_container = FakeCosmosContainer() + bootstrap_cache = _load_chat_bootstrap_module(settings_container, redis_client=None) + inputs = _cache_inputs() + + cache_key = bootstrap_cache.build_chat_bootstrap_cache_key("user-1", **inputs) + + assert bootstrap_cache.set_cached_chat_bootstrap_payload(cache_key, {"chat_agent_options": []}) is False + assert bootstrap_cache.get_cached_chat_bootstrap_payload(cache_key) is None + assert not any(item_id.startswith("shared_cache_entry:chat_bootstrap:") for item_id in settings_container.items) + + +def test_public_workspace_request_approval_persists_manager_and_invalidates_cache(): + """Approving a document-manager request should persist both membership and pending cleanup.""" + public_workspaces_container = FakeCosmosContainer() + invalidation_reasons = [] + public_workspaces_container.create_item({ + "id": "workspace-1", + "pendingDocumentManagers": [ + { + "userId": "user-1", + "email": "user1@example.com", + "displayName": "User One", + }, + ], + "documentManagers": [], + }) + public_workspaces = _load_public_workspaces_module(public_workspaces_container, invalidation_reasons) + + public_workspaces.approve_document_manager_request("workspace-1", "user-1") + + saved_workspace = public_workspaces_container.items["workspace-1"] + assert saved_workspace["pendingDocumentManagers"] == [] + assert saved_workspace["documentManagers"] == [{ + "userId": "user-1", + "email": "user1@example.com", + "displayName": "User One", + }] + assert invalidation_reasons == ["public_workspace_document_manager_request_approved"] + + +if __name__ == "__main__": + tests = [ + test_chat_bootstrap_cache_key_changes_after_global_bump, + test_chat_bootstrap_cache_key_changes_after_user_bump, + test_chat_bootstrap_route_and_write_hooks_are_wired, + test_phase3_low_churn_invalidation_hooks_are_wired, + test_chat_bootstrap_payload_cache_does_not_fallback_to_settings_container_without_redis, + test_public_workspace_request_approval_persists_manager_and_invalidates_cache, + ] + results = [] + for test in tests: + print(f"Running {test.__name__}...") + try: + test() + print("Test passed.") + results.append(True) + except Exception as exc: + print(f"Test failed: {exc}") + results.append(False) + + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_cosmos_wave2a_custom_pages_cache.py b/functional_tests/test_cosmos_wave2a_custom_pages_cache.py new file mode 100644 index 00000000..6e497b76 --- /dev/null +++ b/functional_tests/test_cosmos_wave2a_custom_pages_cache.py @@ -0,0 +1,189 @@ +# test_cosmos_wave2a_custom_pages_cache.py +#!/usr/bin/env python3 +""" +Functional test for Cosmos Wave 2A custom pages cache. +Version: 0.250.032 +Implemented in: 0.250.006 + +This test ensures custom page catalog and navigation reads use shared cache +entries and that writes invalidate those entries with a shared version bump. +""" + +import copy +import importlib +import os +import sys +import types + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SINGLE_APP_DIR = os.path.join(ROOT_DIR, "application", "single_app") +if SINGLE_APP_DIR not in sys.path: + sys.path.insert(0, SINGLE_APP_DIR) + + +class FakeCosmosError(Exception): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class FakeCosmosContainer: + def __init__(self, items=None): + self.items = {} + self._etag_counter = 0 + for item_id, item_body in copy.deepcopy(items or {}).items(): + self.items[item_id] = self._copy_with_new_etag(item_body) + + def _copy_with_new_etag(self, body): + self._etag_counter += 1 + item = copy.deepcopy(body) + item["_etag"] = f"etag-{self._etag_counter}" + return item + + def query_items(self, query, parameters=None, **kwargs): + return [copy.deepcopy(item) for item in self.items.values()] + + def read_item(self, item, partition_key): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + return copy.deepcopy(self.items[item]) + + def create_item(self, body): + item_id = body["id"] + if item_id in self.items: + raise FakeCosmosError(409, f"Duplicate item {item_id}") + self.items[item_id] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[item_id]) + + def upsert_item(self, body): + self.items[body["id"]] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[body["id"]]) + + def replace_item(self, item, body, etag=None, match_condition=None, **kwargs): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + if etag and self.items[item].get("_etag") != etag: + raise FakeCosmosError(412, f"ETag mismatch for item {item}") + self.items[item] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[item]) + + def delete_item(self, item, partition_key, **kwargs): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + del self.items[item] + + +def _page(slug, nav_order=10): + return { + "id": slug, + "slug": slug, + "title": slug.title(), + "description": "", + "enabled": True, + "entry_type": "static", + "access_level": "authenticated", + "nav_label": slug.title(), + "nav_icon": "bi-file-earmark-text", + "nav_order": nav_order, + "roles": [], + "show_in_nav": True, + "open_in_new_tab": False, + "render_jinja": False, + "html_file": f"{slug}.html", + "css_files": [], + "js_files": [], + "asset_files": [], + "json_files": [], + "source": "cosmos", + } + + +def _load_custom_pages_module(custom_pages_container, settings_container): + fake_config = types.ModuleType("config") + fake_config.cosmos_custom_pages_container = custom_pages_container + fake_config.cosmos_settings_container = settings_container + sys.modules["config"] = fake_config + + fake_appinsights = types.ModuleType("functions_appinsights") + fake_appinsights.log_event = lambda *args, **kwargs: None + sys.modules["functions_appinsights"] = fake_appinsights + + for module_name in [ + "functions_shared_cache", + "app_settings_cache", + "functions_custom_pages", + ]: + sys.modules.pop(module_name, None) + return importlib.import_module("functions_custom_pages") + + +def test_custom_pages_catalog_cache_invalidates_on_version_bump(): + """A version bump should refresh the cached custom pages catalog.""" + custom_pages_container = FakeCosmosContainer({"alpha": _page("alpha")}) + settings_container = FakeCosmosContainer() + custom_pages = _load_custom_pages_module(custom_pages_container, settings_container) + + first = custom_pages.list_cosmos_custom_pages() + custom_pages_container.items["beta"] = _page("beta", nav_order=20) + cached = custom_pages.list_cosmos_custom_pages() + custom_pages.invalidate_custom_pages_cache(reason="test") + refreshed = custom_pages.list_cosmos_custom_pages() + + assert [page["slug"] for page in first] == ["alpha"] + assert [page["slug"] for page in cached] == ["alpha"] + assert {page["slug"] for page in refreshed} == {"alpha", "beta"} + + +def test_custom_pages_save_invalidates_catalog_cache(): + """Saving a page should invalidate the catalog cache without a manual bump.""" + custom_pages_container = FakeCosmosContainer({"alpha": _page("alpha")}) + settings_container = FakeCosmosContainer() + custom_pages = _load_custom_pages_module(custom_pages_container, settings_container) + + custom_pages.list_cosmos_custom_pages() + custom_pages.save_custom_page(_page("beta", nav_order=20), user_id="admin") + refreshed = custom_pages.list_cosmos_custom_pages() + + assert {page["slug"] for page in refreshed} == {"alpha", "beta"} + + +def test_custom_pages_nav_cache_uses_cached_navigation_until_invalidated(): + """Navigation cache should be role-aware and version invalidated.""" + custom_pages_container = FakeCosmosContainer({"alpha": _page("alpha")}) + settings_container = FakeCosmosContainer() + custom_pages = _load_custom_pages_module(custom_pages_container, settings_container) + settings = { + "enable_custom_pages": True, + "custom_pages_nav_cache_ttl_seconds": 60, + } + + first_nav = custom_pages.get_custom_pages_nav(settings) + custom_pages_container.items["beta"] = _page("beta", nav_order=20) + cached_nav = custom_pages.get_custom_pages_nav(settings) + custom_pages.invalidate_custom_pages_cache(reason="test_nav") + refreshed_nav = custom_pages.get_custom_pages_nav(settings) + + assert [item["slug"] for item in first_nav] == ["alpha"] + assert [item["slug"] for item in cached_nav] == ["alpha"] + assert {item["slug"] for item in refreshed_nav} == {"alpha", "beta"} + + +if __name__ == "__main__": + tests = [ + test_custom_pages_catalog_cache_invalidates_on_version_bump, + test_custom_pages_save_invalidates_catalog_cache, + test_custom_pages_nav_cache_uses_cached_navigation_until_invalidated, + ] + results = [] + for test in tests: + print(f"Running {test.__name__}...") + try: + test() + print("Test passed.") + results.append(True) + except Exception as exc: + print(f"Test failed: {exc}") + results.append(False) + + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_cosmos_wave2b_conversation_cache.py b/functional_tests/test_cosmos_wave2b_conversation_cache.py new file mode 100644 index 00000000..849d1ad8 --- /dev/null +++ b/functional_tests/test_cosmos_wave2b_conversation_cache.py @@ -0,0 +1,465 @@ +# test_cosmos_wave2b_conversation_cache.py +#!/usr/bin/env python3 +""" +Functional test for Cosmos Wave 2B conversation list/search cache. +Version: 0.250.037 +Implemented in: 0.250.007 +Conversation cache metrics updated in: 0.250.034 +Redis-only volatile cache fallback implemented in: 0.250.037 + +This test ensures conversation list/search/feed caches are versioned per user, +invalidate for personal and collaboration mutations, and fail open when cache +writes or Redis are unavailable. +""" + +import copy +import importlib +import os +import sys +import types +from contextlib import contextmanager + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SINGLE_APP_DIR = os.path.join(ROOT_DIR, "application", "single_app") +if SINGLE_APP_DIR not in sys.path: + sys.path.insert(0, SINGLE_APP_DIR) + + +class FakeCosmosError(Exception): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class FakeCosmosContainer: + def __init__(self): + self.items = {} + self._etag_counter = 0 + + def _copy_with_new_etag(self, body): + self._etag_counter += 1 + item = copy.deepcopy(body) + item["_etag"] = f"etag-{self._etag_counter}" + return item + + def read_item(self, item, partition_key): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + return copy.deepcopy(self.items[item]) + + def create_item(self, body): + item_id = body["id"] + if item_id in self.items: + raise FakeCosmosError(409, f"Duplicate item {item_id}") + self.items[item_id] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[item_id]) + + def upsert_item(self, body): + self.items[body["id"]] = self._copy_with_new_etag(body) + return copy.deepcopy(body) + + def replace_item(self, item, body, etag=None, match_condition=None, **kwargs): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + if etag and self.items[item].get("_etag") != etag: + raise FakeCosmosError(412, f"ETag mismatch for item {item}") + self.items[item] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[item]) + + def delete_item(self, item, partition_key, **kwargs): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + del self.items[item] + + +class ConflictOnceFakeCosmosContainer(FakeCosmosContainer): + def __init__(self): + super().__init__() + self.conflict_triggered = False + + def replace_item(self, item, body, etag=None, match_condition=None, **kwargs): + if not self.conflict_triggered: + self.conflict_triggered = True + conflicting_body = copy.deepcopy(self.items[item]) + conflicting_body["version"] = int(conflicting_body.get("version", 0)) + 1 + self.items[item] = self._copy_with_new_etag(conflicting_body) + raise FakeCosmosError(412, f"ETag mismatch for item {item}") + return super().replace_item( + item, + body, + etag=etag, + match_condition=match_condition, + **kwargs, + ) + + +class FakeRedisClient: + def __init__(self): + self.values = {} + + def get(self, key): + return self.values.get(key) + + def set(self, key, value): + self.values[key] = value + return True + + def setex(self, key, ttl_seconds, value): + self.values[key] = value + return True + + def incr(self, key): + value = int(self.values.get(key) or 0) + 1 + self.values[key] = str(value) + return value + + def delete(self, key): + self.values.pop(key, None) + return True + + +@contextmanager +def _load_conversation_cache_module(settings_container, group_doc=None, redis_client=None): + module_names = [ + "config", + "functions_appinsights", + "app_settings_cache", + "functions_group", + "functions_shared_cache", + "functions_conversation_cache", + ] + saved_modules = { + module_name: sys.modules.get(module_name) + for module_name in module_names + } + + fake_config = types.ModuleType("config") + fake_config.cosmos_settings_container = settings_container + sys.modules["config"] = fake_config + + fake_appinsights = types.ModuleType("functions_appinsights") + fake_appinsights.log_event = lambda *args, **kwargs: None + sys.modules["functions_appinsights"] = fake_appinsights + + fake_app_settings_cache = types.ModuleType("app_settings_cache") + fake_app_settings_cache.get_app_cache_redis_client = lambda: redis_client + sys.modules["app_settings_cache"] = fake_app_settings_cache + + fake_group = types.ModuleType("functions_group") + fake_group.find_group_by_id = lambda group_id: copy.deepcopy(group_doc) if group_doc else None + sys.modules["functions_group"] = fake_group + + for module_name in [ + "functions_shared_cache", + "functions_conversation_cache", + ]: + sys.modules.pop(module_name, None) + try: + yield importlib.import_module("functions_conversation_cache") + finally: + for module_name, module_value in saved_modules.items(): + if module_value is None: + sys.modules.pop(module_name, None) + else: + sys.modules[module_name] = module_value + + +def test_conversation_cache_key_changes_after_user_bump(): + """A user version bump should change that user's list cache key.""" + settings_container = FakeCosmosContainer() + with _load_conversation_cache_module(settings_container, redis_client=FakeRedisClient()) as conversation_cache: + first_key = conversation_cache.build_conversation_cache_key( + "user-1", + "list", + parameters={"include_hidden": True}, + ) + conversation_cache.set_cached_conversation_payload( + first_key, + {"conversations": [{"id": "conversation-1"}]}, + ttl_seconds=120, + ) + conversation_cache.bump_conversation_cache_version("user-1", reason="test") + second_key = conversation_cache.build_conversation_cache_key( + "user-1", + "list", + parameters={"include_hidden": True}, + ) + + assert first_key != second_key + assert conversation_cache.get_cached_conversation_payload(first_key)["conversations"][0]["id"] == "conversation-1" + assert conversation_cache.get_cached_conversation_payload(second_key) is None + assert not settings_container.items + + +def test_conversation_cache_bump_uses_redis_version_counter(): + """Conversation version bumps should use Redis instead of settings-container version docs.""" + settings_container = ConflictOnceFakeCosmosContainer() + with _load_conversation_cache_module(settings_container, redis_client=FakeRedisClient()) as conversation_cache: + first_version = conversation_cache.bump_conversation_cache_version("user-1", reason="first") + second_version = conversation_cache.bump_conversation_cache_version("user-1", reason="second") + + assert first_version == 1 + assert second_version == 2 + assert "conversation_cache_version:user-1" not in settings_container.items + + +def test_conversation_cache_key_reads_redis_version_without_settings_container(): + """Conversation keys should observe external Redis version bumps without settings-container reads.""" + settings_container = FakeCosmosContainer() + redis_client = FakeRedisClient() + with _load_conversation_cache_module(settings_container, redis_client=redis_client) as conversation_cache: + conversation_cache.bump_conversation_cache_version("user-1", reason="initial") + first_key = conversation_cache.build_conversation_cache_key("user-1", "feed", parameters={}) + + redis_client.incr("conversation_cache_version:user-1") + + second_key = conversation_cache.build_conversation_cache_key("user-1", "feed", parameters={}) + + assert first_key != second_key + assert not settings_container.items + + +def test_conversation_cache_invalidation_fans_out_to_viewers(): + """Invalidating a conversation should include owners, participants, and group viewers.""" + settings_container = FakeCosmosContainer() + with _load_conversation_cache_module( + settings_container, + redis_client=FakeRedisClient(), + group_doc={ + "id": "group-1", + "owner": {"id": "group-owner"}, + "admins": [{"userId": "group-admin"}], + "documentManagers": [{"userId": "group-doc-manager"}], + "users": [{"userId": "group-user"}], + }, + ) as conversation_cache: + bumped_user_ids = [] + conversation_cache.bump_conversation_cache_version = ( + lambda user_id, reason="test": bumped_user_ids.append(user_id) or 1 + ) + + conversation_cache.invalidate_conversation_cache_for_item( + { + "id": "conversation-1", + "user_id": "owner-user", + "created_by_user_id": "creator-user", + "accepted_participant_ids": ["accepted-user"], + "pending_participant_ids": ["pending-user"], + "owner_user_ids": ["collab-owner"], + "admin_user_ids": ["collab-admin"], + "participants": [ + {"user_id": "active-participant", "status": "accepted"}, + {"user_id": "removed-participant", "status": "removed"}, + ], + "tags": [ + {"category": "participant", "user_id": "tagged-participant"}, + ], + "scope": {"group_id": "group-1"}, + }, + reason="test", + ) + + assert set(bumped_user_ids) == { + "owner-user", + "creator-user", + "accepted-user", + "pending-user", + "collab-owner", + "collab-admin", + "active-participant", + "tagged-participant", + "group-owner", + "group-admin", + "group-doc-manager", + "group-user", + } + + +def test_conversation_cache_write_failures_do_not_escape(): + """Cache write failures should not fail the caller.""" + settings_container = FakeCosmosContainer() + with _load_conversation_cache_module(settings_container, redis_client=FakeRedisClient()) as conversation_cache: + def raise_cache_write(*args, **kwargs): + raise RuntimeError("cache unavailable") + + conversation_cache.set_shared_cache_entry = raise_cache_write + assert conversation_cache.set_cached_conversation_payload( + "cache-key", + {"success": True}, + ttl_seconds="invalid", + ) is False + + +def test_conversation_cache_disabled_bypasses_reads_and_writes(): + """Disabled conversation cache should bypass cache entries and leave source reads authoritative.""" + settings_container = FakeCosmosContainer() + with _load_conversation_cache_module(settings_container, redis_client=FakeRedisClient()) as conversation_cache: + cache_key = conversation_cache.build_conversation_cache_key( + "user-1", + "list", + parameters={"include_hidden": True}, + ) + assert conversation_cache.set_cached_conversation_payload( + cache_key, + {"conversations": [{"id": "conversation-1"}]}, + ttl_seconds=120, + ) is True + + disabled_settings = {"enable_conversation_cache": False} + assert conversation_cache.get_cached_conversation_payload( + cache_key, + settings=disabled_settings, + ) is None + assert conversation_cache.set_cached_conversation_payload( + "disabled-key", + {"conversations": [{"id": "conversation-2"}]}, + settings=disabled_settings, + ) is False + assert "shared_cache_entry:conversation_cache:disabled-key" not in settings_container.items + + +def test_conversation_cache_redis_unavailable_bypasses_without_settings_writes(): + """Redis-unavailable conversation cache should fall back to source Cosmos query behavior.""" + settings_container = FakeCosmosContainer() + with _load_conversation_cache_module(settings_container, redis_client=None) as conversation_cache: + cache_key = conversation_cache.build_conversation_cache_key( + "user-1", + "list", + parameters={"include_hidden": True}, + ) + + assert cache_key is None + assert conversation_cache.bump_conversation_cache_version("user-1", reason="redis-missing") is None + assert conversation_cache.get_cached_conversation_payload("list:user-1:test") is None + assert conversation_cache.set_cached_conversation_payload( + "list:user-1:test", + {"conversations": [{"id": "conversation-1"}]}, + ttl_seconds=120, + ) is False + assert not settings_container.items + + +def test_conversation_cache_metrics_track_rolling_events(): + """Conversation cache metrics should track DAI-style rolling cache events.""" + settings_container = FakeCosmosContainer() + with _load_conversation_cache_module(settings_container, redis_client=FakeRedisClient()) as conversation_cache: + conversation_cache.reset_conversation_cache_metrics() + list_key = conversation_cache.build_conversation_cache_key( + "user-1", + "list", + parameters={"include_hidden": True}, + ) + feed_key = conversation_cache.build_conversation_cache_key( + "user-1", + "feed", + parameters={"limit": 25}, + ) + + assert conversation_cache.set_cached_conversation_payload( + list_key, + {"conversations": [{"id": "conversation-1"}]}, + ttl_seconds=120, + ) is True + assert conversation_cache.get_cached_conversation_payload(list_key)["conversations"][0]["id"] == "conversation-1" + assert conversation_cache.get_cached_conversation_payload(feed_key) is None + assert conversation_cache.get_cached_conversation_payload( + list_key, + settings={"enable_conversation_cache": False}, + ) is None + assert conversation_cache.set_cached_conversation_payload( + list_key, + {"conversations": [{"id": "conversation-2"}]}, + settings={"conversation_cache_ttl_seconds": 0}, + ) is False + assert conversation_cache.bump_conversation_cache_version("user-1", reason="metrics-test") == 1 + + metrics = conversation_cache.get_conversation_cache_metrics() + window_15m = metrics["windows"]["15m"] + + assert window_15m["hit_count"] == 1 + assert window_15m["miss_count"] == 1 + assert window_15m["bypass_count"] == 2 + assert window_15m["write_count"] == 1 + assert window_15m["invalidation_count"] == 1 + assert window_15m["error_count"] == 0 + assert window_15m["hit_rate_percent"] == 50.0 + assert window_15m["operation_counts"]["list"] == 4 + assert window_15m["operation_counts"]["feed"] == 1 + assert window_15m["operation_counts"]["version"] == 1 + assert metrics["last_invalidation"]["reason"] == "metrics-test" + + +def test_conversation_cache_route_and_mutation_hooks_are_wired(): + """Contract test for list/search/feed cache usage and invalidation hooks.""" + route_source = open( + os.path.join(SINGLE_APP_DIR, "route_backend_conversations.py"), + "r", + encoding="utf-8", + ).read() + assert "build_conversation_cache_key(user_id, \"list\"" in route_source + assert "_build_conversation_cache_access_parameters(user_id)" in route_source + assert "get_conversation_cache_settings(settings)" in route_source + assert "cache_settings.get('enabled')" in route_source + assert "\"access\": access_parameters" in route_source or "'access': access_parameters" in route_source + assert "feed_cache_parameters" in route_source and "\"feed\"" in route_source + assert "search_cache_parameters" in route_source and "\"search\"" in route_source + assert "get_cached_conversation_payload" in route_source + assert "set_cached_conversation_payload" in route_source + assert "invalidate_conversation_cache_for_item(conversation_item, reason=\"conversation_chat_type_normalized\")" in route_source + + settings_source = open(os.path.join(SINGLE_APP_DIR, "functions_settings.py"), "r", encoding="utf-8").read() + assert "'enable_conversation_cache': True" in settings_source + + cache_source = open(os.path.join(SINGLE_APP_DIR, "functions_conversation_cache.py"), "r", encoding="utf-8").read() + maintenance_source = open(os.path.join(SINGLE_APP_DIR, "functions_app_maintenance.py"), "r", encoding="utf-8").read() + assert "def get_conversation_cache_metrics()" in cache_source + assert "bump_shared_cache_version" not in cache_source + assert "get_shared_cache_version" not in cache_source + assert "allow_cosmos_fallback=False" in cache_source + assert "redis_unavailable" in cache_source + assert "_record_conversation_cache_metric(\"hit\", operation=operation)" in cache_source + assert "_record_conversation_cache_metric(\"miss\", operation=operation)" in cache_source + assert "_record_conversation_cache_metric(\"bypass\", operation=operation, reason=\"disabled\")" in cache_source + assert "_record_conversation_cache_metric(\"invalidate\", operation=\"version\", reason=reason)" in cache_source + assert "'conversation_cache': {" in maintenance_source + assert "'metrics': get_conversation_cache_metrics()" in maintenance_source + + for file_name, marker in { + "route_backend_conversations.py": "bump_conversation_cache_version(user_id, reason=\"conversation_pin_toggled\")", + "route_backend_chats.py": "invalidate_conversation_cache_for_item(conversation_item, reason=\"chat_stream_completed\")", + "functions_collaboration.py": "invalidate_conversation_cache_for_item(conversation_doc, reason=\"collaboration_message_saved\")", + "route_backend_notifications.py": "bump_conversation_cache_version(user_id, reason=\"notification_marked_read\")", + }.items(): + source = open(os.path.join(SINGLE_APP_DIR, file_name), "r", encoding="utf-8").read() + assert marker in source, f"Missing conversation cache hook marker in {file_name}: {marker}" + if file_name == "route_backend_chats.py": + assert source.count("reason=\"chat_image_generated\"") == 1 + if file_name == "functions_collaboration.py": + assert "collaboration_notification_created" not in source + + +if __name__ == "__main__": + tests = [ + test_conversation_cache_key_changes_after_user_bump, + test_conversation_cache_bump_uses_redis_version_counter, + test_conversation_cache_key_reads_redis_version_without_settings_container, + test_conversation_cache_invalidation_fans_out_to_viewers, + test_conversation_cache_write_failures_do_not_escape, + test_conversation_cache_disabled_bypasses_reads_and_writes, + test_conversation_cache_redis_unavailable_bypasses_without_settings_writes, + test_conversation_cache_metrics_track_rolling_events, + test_conversation_cache_route_and_mutation_hooks_are_wired, + ] + results = [] + for test in tests: + print(f"Running {test.__name__}...") + try: + test() + print("Test passed.") + results.append(True) + except Exception as exc: + print(f"Test failed: {exc}") + results.append(False) + + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_cosmos_wave3a_indexing_maintenance.py b/functional_tests/test_cosmos_wave3a_indexing_maintenance.py new file mode 100644 index 00000000..835d622a --- /dev/null +++ b/functional_tests/test_cosmos_wave3a_indexing_maintenance.py @@ -0,0 +1,350 @@ +# test_cosmos_wave3a_indexing_maintenance.py +#!/usr/bin/env python3 +""" +Functional test for Cosmos Wave 3A indexing policy maintenance. +Version: 0.250.047 +Implemented in: 0.250.008 +Maintenance cleanup integration updated in: 0.250.038 +Manual admin apply override updated in: 0.250.039 + +This test ensures expected Cosmos indexing policies can be compared, safely +merged, and invoked through the app maintenance framework without live Azure +resources. +""" + +import copy +import importlib +import os +import sys +import types +from contextlib import contextmanager + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SINGLE_APP_DIR = os.path.join(ROOT_DIR, "application", "single_app") +if SINGLE_APP_DIR not in sys.path: + sys.path.insert(0, SINGLE_APP_DIR) + + +class FakeCosmosError(Exception): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class FakeCosmosContainer: + def __init__(self, container_name, indexing_policy=None, default_ttl=None, full_text_policy=None): + self.container_name = container_name + self.items = {} + self._etag_counter = 0 + self.container_properties = { + "id": container_name, + "_etag": f"{container_name}-etag-0", + "indexingPolicy": copy.deepcopy(indexing_policy or { + "indexingMode": "consistent", + "automatic": True, + "includedPaths": [{"path": "/*"}], + "excludedPaths": [{"path": "/\"_etag\"/?"}], + "compositeIndexes": [], + }), + } + if default_ttl is not None: + self.container_properties["defaultTtl"] = default_ttl + if full_text_policy is not None: + self.container_properties["fullTextPolicy"] = copy.deepcopy(full_text_policy) + + def _copy_with_new_etag(self, body): + self._etag_counter += 1 + item = copy.deepcopy(body) + item["_etag"] = f"{self.container_name}-item-etag-{self._etag_counter}" + return item + + def read(self): + return copy.deepcopy(self.container_properties) + + def replace_properties(self, kwargs): + self._etag_counter += 1 + self.container_properties = { + "id": kwargs["container"], + "_etag": f"{self.container_name}-etag-{self._etag_counter}", + "partitionKey": kwargs.get("partition_key"), + "indexingPolicy": copy.deepcopy(kwargs["indexing_policy"]), + } + property_mappings = { + "default_ttl": "defaultTtl", + "analytical_storage_ttl": "analyticalStorageTtl", + "conflict_resolution_policy": "conflictResolutionPolicy", + "full_text_policy": "fullTextPolicy", + } + for call_key, property_key in property_mappings.items(): + if call_key in kwargs: + self.container_properties[property_key] = copy.deepcopy(kwargs[call_key]) + self.container_properties["_etag"] = f"{self.container_name}-etag-{self._etag_counter}" + + def read_item(self, item, partition_key): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + return copy.deepcopy(self.items[item]) + + def create_item(self, body): + item_id = body["id"] + if item_id in self.items: + raise FakeCosmosError(409, f"Duplicate item {item_id}") + self.items[item_id] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[item_id]) + + def upsert_item(self, body): + self.items[body["id"]] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[body["id"]]) + + def replace_item(self, item, body, etag=None, match_condition=None, **kwargs): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + if etag and self.items[item].get("_etag") != etag: + raise FakeCosmosError(412, f"ETag mismatch for item {item}") + self.items[item] = self._copy_with_new_etag(body) + return copy.deepcopy(self.items[item]) + + def delete_item(self, item, partition_key, **kwargs): + if item not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + del self.items[item] + + def query_items(self, query, parameters=None, enable_cross_partition_query=False, **kwargs): + parameter_map = { + parameter["name"]: parameter["value"] + for parameter in list(parameters or []) + } + item_type = parameter_map.get("@type") + item_status = parameter_map.get("@status") + document_metadata_type = parameter_map.get("@document_metadata_type") + results = [] + for item in self.items.values(): + if item_type is not None and item.get("type") != item_type: + continue + if item_status is not None and item.get("status") != item_status: + continue + if document_metadata_type is not None and item.get("type") not in (None, document_metadata_type): + continue + results.append(copy.deepcopy(item)) + return results + + +class FakeCosmosDatabase: + def __init__(self, containers): + self.containers = containers + self.replace_calls = [] + + def replace_container(self, **kwargs): + self.replace_calls.append(copy.deepcopy(kwargs)) + container_name = kwargs["container"] + self.containers[container_name].replace_properties(kwargs) + return self.containers[container_name] + + +def _build_fake_environment(): + containers = { + "conversations": FakeCosmosContainer("conversations"), + "messages": FakeCosmosContainer("messages"), + "collaboration_messages": FakeCosmosContainer("collaboration_messages"), + "document_access_index": FakeCosmosContainer("document_access_index"), + "documents": FakeCosmosContainer( + "documents", + indexing_policy={ + "indexingMode": "consistent", + "automatic": True, + "includedPaths": [{"path": "/*"}, {"path": "/id/?"}], + "excludedPaths": [{"path": "/large_unused_payload/*"}], + "compositeIndexes": [[{"path": "/existing", "order": "ascending"}]], + }, + default_ttl=-1, + full_text_policy={ + "defaultLanguage": "en-US", + "fullTextPaths": [{"path": "/content", "language": "en-US"}], + }, + ), + "group_documents": FakeCosmosContainer("group_documents"), + "public_documents": FakeCosmosContainer("public_documents"), + "groups": FakeCosmosContainer("groups"), + "public_workspaces": FakeCosmosContainer("public_workspaces"), + "user_settings": FakeCosmosContainer("user_settings"), + } + database = FakeCosmosDatabase(containers) + settings_container = FakeCosmosContainer("settings") + governance_container = FakeCosmosContainer("governance_policies") + return containers, database, settings_container, governance_container + + +@contextmanager +def _load_wave3_modules(): + original_modules = {} + module_names = [ + "config", + "functions_appinsights", + "functions_group", + "functions_shared_cache", + "functions_cosmos_stale_cleanup", + "functions_cosmos_indexing", + "functions_document_access_index", + "functions_app_maintenance", + "functions_settings", + ] + for module_name in module_names: + if module_name in sys.modules: + original_modules[module_name] = sys.modules[module_name] + del sys.modules[module_name] + + containers, database, settings_container, governance_container = _build_fake_environment() + fake_config = types.ModuleType("config") + fake_config.VERSION = "0.250.039" + fake_config.cosmos_database = database + fake_config.cosmos_settings_container = settings_container + fake_config.cosmos_governance_policies_container = governance_container + fake_config.cosmos_conversations_container = containers["conversations"] + fake_config.cosmos_conversations_container_name = "conversations" + fake_config.cosmos_messages_container = containers["messages"] + fake_config.cosmos_messages_container_name = "messages" + fake_config.cosmos_collaboration_messages_container = containers["collaboration_messages"] + fake_config.cosmos_collaboration_messages_container_name = "collaboration_messages" + fake_config.cosmos_user_documents_container = containers["documents"] + fake_config.cosmos_user_documents_container_name = "documents" + fake_config.cosmos_group_documents_container = containers["group_documents"] + fake_config.cosmos_group_documents_container_name = "group_documents" + fake_config.cosmos_public_documents_container = containers["public_documents"] + fake_config.cosmos_public_documents_container_name = "public_documents" + fake_config.cosmos_document_access_index_container = containers["document_access_index"] + fake_config.cosmos_groups_container = containers["groups"] + fake_config.cosmos_public_workspaces_container = containers["public_workspaces"] + fake_config.cosmos_user_settings_container = containers["user_settings"] + sys.modules["config"] = fake_config + + fake_appinsights = types.ModuleType("functions_appinsights") + fake_appinsights.log_event = lambda *args, **kwargs: None + fake_appinsights.debug_print = lambda *args, **kwargs: None + fake_appinsights.is_debug_enabled = lambda: False + sys.modules["functions_appinsights"] = fake_appinsights + + fake_group = types.ModuleType("functions_group") + fake_group.find_group_by_id = lambda group_id: None + sys.modules["functions_group"] = fake_group + + fake_settings = types.ModuleType("functions_settings") + fake_settings.get_settings = lambda: {} + sys.modules["functions_settings"] = fake_settings + + try: + indexing = importlib.import_module("functions_cosmos_indexing") + maintenance = importlib.import_module("functions_app_maintenance") + yield indexing, maintenance, database, containers + finally: + for module_name in module_names: + sys.modules.pop(module_name, None) + sys.modules.update(original_modules) + + +def _canonical_composite(index): + return tuple((path["path"], path["order"]) for path in index) + + +def test_indexing_policy_report_is_read_only(): + """Report-only mode should find missing indexes without replacing containers.""" + with _load_wave3_modules() as (indexing, _maintenance, database, _containers): + report = indexing.run_cosmos_indexing_policy_maintenance(apply_changes=False) + + assert report["success"] is True + assert report["mode"] == "report_only" + assert report["containers_missing_expected_indexes"] > 0 + assert report["updated_container_count"] == 0 + assert database.replace_calls == [] + + +def test_indexing_policy_apply_merges_without_removing_existing_paths(): + """Apply mode should add missing composites while preserving current policy paths.""" + with _load_wave3_modules() as (indexing, _maintenance, database, containers): + report = indexing.run_cosmos_indexing_policy_maintenance(apply_changes=True) + documents_policy = containers["documents"].container_properties["indexingPolicy"] + documents_call = next(call for call in database.replace_calls if call["container"] == "documents") + + assert report["success"] is True + assert report["updated_container_count"] == len(indexing.COSMOS_INDEXING_POLICY_DEFINITIONS) + assert documents_policy["includedPaths"] == [{"path": "/*"}, {"path": "/id/?"}] + assert documents_policy["excludedPaths"] == [{"path": "/large_unused_payload/*"}] + assert _canonical_composite([{"path": "/existing", "order": "ascending"}]) in { + _canonical_composite(index) + for index in documents_policy["compositeIndexes"] + } + assert documents_call["default_ttl"] == -1 + assert documents_call["full_text_policy"] == { + "defaultLanguage": "en-US", + "fullTextPaths": [{"path": "/content", "language": "en-US"}], + } + assert containers["documents"].container_properties["fullTextPolicy"] == documents_call["full_text_policy"] + + +def test_app_maintenance_runs_indexing_policy_step(): + """App maintenance should include the Wave 3A indexing policy maintenance step.""" + with _load_wave3_modules() as (indexing, maintenance, database, _containers): + result = maintenance.run_app_maintenance_once( + triggered_by="test", + requested_by="tester@example.com", + settings={indexing.COSMOS_INDEXING_POLICY_APPLY_SETTING: True}, + ) + status = maintenance.get_app_maintenance_status( + settings={indexing.COSMOS_INDEXING_POLICY_APPLY_SETTING: True}, + ) + + indexing_step = next(step for step in result["steps"] if step["name"] == "cosmos_indexing_policy_maintenance") + assert result["success"] is True + assert indexing_step["status"] == "succeeded" + assert indexing_step["results"]["mode"] == "apply" + assert database.replace_calls + assert status["cosmos_indexing_policies"]["mode"] == "report_only" + assert status["stale_cache_cleanup"]["status"] == "skipped_disabled" + assert status["settings"]["apply_cosmos_indexing_policies"] is True + assert status["document_access_index_backfill"]["state"]["status"] == "succeeded" + + +def test_app_maintenance_manual_indexing_apply_override(): + """Manual admin runs should apply indexes when payload override is true.""" + with _load_wave3_modules() as (_indexing, maintenance, database, _containers): + result = maintenance.run_app_maintenance_once( + triggered_by="admin_manual", + requested_by="tester@example.com", + settings={}, + apply_indexing_policies=True, + run_document_access_backfill=False, + run_stale_cache_cleanup=False, + ) + + indexing_step = next(step for step in result["steps"] if step["name"] == "cosmos_indexing_policy_maintenance") + cleanup_step = next(step for step in result["steps"] if step["name"] == "stale_cache_document_cleanup") + backfill_step = next(step for step in result["steps"] if step["name"] == "document_access_index_backfill") + assert result["success"] is True + assert indexing_step["apply_requested"] is True + assert indexing_step["results"]["mode"] == "apply" + assert indexing_step["results"]["updated_container_count"] > 0 + assert database.replace_calls + assert cleanup_step["run_requested"] is False + assert cleanup_step["results"]["status"] == "skipped_disabled" + assert backfill_step["run_requested"] is False + + +if __name__ == "__main__": + tests = [ + test_indexing_policy_report_is_read_only, + test_indexing_policy_apply_merges_without_removing_existing_paths, + test_app_maintenance_runs_indexing_policy_step, + test_app_maintenance_manual_indexing_apply_override, + ] + results = [] + for test in tests: + print(f"Running {test.__name__}...") + try: + test() + print("Test passed.") + results.append(True) + except Exception as exc: + print(f"Test failed: {exc}") + results.append(False) + + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_cosmos_wave3b_document_access_index.py b/functional_tests/test_cosmos_wave3b_document_access_index.py new file mode 100644 index 00000000..0f9d0339 --- /dev/null +++ b/functional_tests/test_cosmos_wave3b_document_access_index.py @@ -0,0 +1,361 @@ +# test_cosmos_wave3b_document_access_index.py +#!/usr/bin/env python3 +""" +Functional test for Cosmos Wave 3B document access index write-through. +Version: 0.250.047 +Implemented in: 0.250.009 +Default read enablement updated in: 0.250.027 +Redis DAI cache invalidation updated in: 0.250.029 +Repair backlog state cleanup updated in: 0.250.030 + +This test ensures document_access_index projection rows are deterministic, +scope-partitioned, clean up stale access rows, and fail open with repair state +when projection writes are unavailable. +""" + +import copy +import importlib +import os +import sys +import types +from contextlib import contextmanager + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SINGLE_APP_DIR = os.path.join(ROOT_DIR, "application", "single_app") +if SINGLE_APP_DIR not in sys.path: + sys.path.insert(0, SINGLE_APP_DIR) + + +class FakeCosmosError(Exception): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class FakeCosmosContainer: + def __init__(self, fail_upsert=False): + self.items = {} + self.fail_upsert = fail_upsert + + def _partition_key_for_body(self, body): + return body.get("scope_key") or body.get("id") + + def upsert_item(self, body): + if self.fail_upsert: + raise RuntimeError("projection container unavailable") + partition_key = self._partition_key_for_body(body) + self.items[(partition_key, body["id"])] = copy.deepcopy(body) + return copy.deepcopy(body) + + def delete_item(self, item, partition_key): + key = (partition_key, item) + if key not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + del self.items[key] + + def read_item(self, item, partition_key): + key = (partition_key, item) + if key not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + return copy.deepcopy(self.items[key]) + + def query_items(self, query, parameters=None, enable_cross_partition_query=False, **kwargs): + parameter_map = { + parameter["name"]: parameter["value"] + for parameter in list(parameters or []) + } + document_id = parameter_map.get("@document_id") + source_scope = parameter_map.get("@source_scope") + item_type = parameter_map.get("@type") + results = [] + for (scope_key, item_id), item in self.items.items(): + if item_type is not None and item.get("type") != item_type: + continue + if source_scope is not None and item.get("source_scope") != source_scope: + continue + if document_id is not None and item.get("source_document_id") != document_id: + continue + results.append({"id": item_id, "scope_key": scope_key}) + return results + + +class FakeRepairBacklogStateWriteFailContainer(FakeCosmosContainer): + def upsert_item(self, body): + if body.get("type") == "document_access_index_repair_backlog_state": + raise RuntimeError("state write throttled") + return super().upsert_item(body) + + +@contextmanager +def _load_document_access_index_module(index_container=None, settings_container=None, settings=None): + original_modules = {} + for module_name in [ + "config", + "functions_appinsights", + "functions_settings", + "functions_document_access_index", + ]: + if module_name in sys.modules: + original_modules[module_name] = sys.modules[module_name] + del sys.modules[module_name] + + index_container = index_container or FakeCosmosContainer() + settings_container = settings_container or FakeCosmosContainer() + fake_config = types.ModuleType("config") + fake_config.cosmos_document_access_index_container = index_container + fake_config.cosmos_settings_container = settings_container + fake_config.cosmos_user_documents_container = FakeCosmosContainer() + fake_config.cosmos_user_documents_container_name = "documents" + fake_config.cosmos_group_documents_container = FakeCosmosContainer() + fake_config.cosmos_group_documents_container_name = "group_documents" + fake_config.cosmos_public_documents_container = FakeCosmosContainer() + fake_config.cosmos_public_documents_container_name = "public_documents" + fake_config.cosmos_groups_container = FakeCosmosContainer() + fake_config.cosmos_public_workspaces_container = FakeCosmosContainer() + fake_config.cosmos_user_settings_container = FakeCosmosContainer() + sys.modules["config"] = fake_config + + fake_appinsights = types.ModuleType("functions_appinsights") + fake_appinsights.log_event = lambda *args, **kwargs: None + sys.modules["functions_appinsights"] = fake_appinsights + + fake_settings = types.ModuleType("functions_settings") + fake_settings.get_settings = lambda: settings or { + "enable_document_access_index_container": True, + "enable_document_access_index_write_through": True, + "enable_document_access_index_reads": True, + "enable_document_access_index_shadow_validation": False, + "enable_startup_document_access_index_backfill": False, + } + sys.modules["functions_settings"] = fake_settings + + try: + yield importlib.import_module("functions_document_access_index"), index_container, settings_container + finally: + for module_name in [ + "config", + "functions_appinsights", + "functions_settings", + "functions_document_access_index", + ]: + sys.modules.pop(module_name, None) + sys.modules.update(original_modules) + + +def _personal_document(shared_user_ids=None): + return { + "id": "doc-1", + "type": "document_metadata", + "user_id": "owner-1", + "file_name": "report.pdf", + "title": "Report", + "version": 2, + "revision_family_id": "family-1", + "is_current_version": True, + "search_visibility_state": "active", + "status": "Processing complete", + "percentage_complete": 100, + "last_updated": "2026-06-30T10:00:00Z", + "shared_user_ids": list(shared_user_ids or []), + } + + +def test_projection_rows_are_scope_partitioned_and_deterministic(): + """Personal owner and shared-user rows should share user scope partitions.""" + with _load_document_access_index_module() as (indexing, _index_container, _settings_container): + rows = indexing.build_document_access_index_rows( + _personal_document(["user-2,not_approved", "user-3,approved"]) + ) + + rows_by_scope = {row["scope_key"]: row for row in rows} + assert set(rows_by_scope) == {"user:owner-1", "user:user-2", "user:user-3"} + assert rows_by_scope["user:owner-1"]["access_role"] == "owner" + assert rows_by_scope["user:user-2"]["approval_status"] == "not_approved" + assert rows_by_scope["user:user-2"]["access_granted"] is False + assert rows_by_scope["user:user-3"]["access_granted"] is True + assert all(row["id"].endswith(":2") for row in rows) + + +def test_projection_share_collisions_prefer_least_privilege(): + """Duplicate explicit statuses and legacy bare shares prefer least privilege.""" + with _load_document_access_index_module() as (indexing, _index_container, _settings_container): + rows = indexing.build_document_access_index_rows( + _personal_document(["user-2,approved", "user-2,not_approved", "legacy-user"]) + ) + + rows_by_scope = {row["scope_key"]: row for row in rows} + assert rows_by_scope["user:user-2"]["approval_status"] == "not_approved" + assert rows_by_scope["user:user-2"]["access_granted"] is False + assert rows_by_scope["user:legacy-user"]["approval_status"] == "not_approved" + assert rows_by_scope["user:legacy-user"]["access_granted"] is False + + +def test_projection_sync_removes_stale_share_rows(): + """Sync should remove rows for users no longer present in source sharing fields.""" + with _load_document_access_index_module() as (indexing, index_container, _settings_container): + indexing.sync_document_access_index_for_document(_personal_document(["user-2,approved"])) + assert ("user:user-2", "dai:user:user-2:personal:doc-1:2") in index_container.items + + result = indexing.sync_document_access_index_for_document(_personal_document([])) + + assert result["deleted_count"] == 1 + assert ("user:user-2", "dai:user:user-2:personal:doc-1:2") not in index_container.items + assert ("user:owner-1", "dai:user:owner-1:personal:doc-1:2") in index_container.items + + +def test_projection_delete_removes_all_source_rows(): + """Delete synchronization should remove every projection row for the source document.""" + with _load_document_access_index_module() as (indexing, index_container, _settings_container): + document = _personal_document(["user-2,approved", "user-3,approved"]) + indexing.sync_document_access_index_for_document(document) + result = indexing.delete_document_access_index_for_document(document) + + assert result["deleted_count"] == 3 + assert index_container.items == {} + + +def test_projection_failure_records_repair_state_without_raising(): + """Fail-open wrapper should record repair state when projection writes fail.""" + with _load_document_access_index_module(index_container=FakeCosmosContainer(fail_upsert=True)) as ( + indexing, + _index_container, + settings_container, + ): + result = indexing.sync_document_access_index_for_document_fail_open( + _personal_document(["user-2,approved"]), + operation="test_failure", + ) + + assert result["success"] is False + repair_docs = [ + item + for item in settings_container.items.values() + if item.get("type") == "document_access_index_repair" + ] + assert len(repair_docs) == 1 + assert repair_docs[0]["source_document_id"] == "doc-1" + assert repair_docs[0]["operation"] == "test_failure" + repair_backlog_state = settings_container.items.get(( + "document_access_index_repair_backlog_state", + "document_access_index_repair_backlog_state", + )) + assert repair_backlog_state["has_repair_backlog"] is True + + +def test_projection_failure_records_repair_doc_when_backlog_state_write_fails(): + """Backlog state write failures must not suppress source repair records.""" + settings_container = FakeRepairBacklogStateWriteFailContainer() + with _load_document_access_index_module( + index_container=FakeCosmosContainer(fail_upsert=True), + settings_container=settings_container, + ) as ( + indexing, + _index_container, + _settings_container, + ): + result = indexing.sync_document_access_index_for_document_fail_open( + _personal_document(["user-2,approved"]), + operation="test_state_failure", + ) + + repair_docs = [ + item + for item in settings_container.items.values() + if item.get("type") == "document_access_index_repair" + ] + repair_backlog_states = [ + item + for item in settings_container.items.values() + if item.get("type") == "document_access_index_repair_backlog_state" + ] + assert result["success"] is False + assert len(repair_docs) == 1 + assert repair_docs[0]["operation"] == "test_state_failure" + assert repair_backlog_states == [] + + +def test_projection_repair_clear_removes_stale_backlog_state_when_false_write_fails(): + """A successful repair clear should not leave a stale true backlog state behind.""" + settings_container = FakeRepairBacklogStateWriteFailContainer() + with _load_document_access_index_module(settings_container=settings_container) as ( + indexing, + _index_container, + _settings_container, + ): + document = _personal_document(["user-2,approved"]) + source_scope = "personal" + repair_doc_id = indexing._repair_document_id(source_scope, document["id"]) + state_doc_id = indexing.DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID + settings_container.items[(repair_doc_id, repair_doc_id)] = { + "id": repair_doc_id, + "type": "document_access_index_repair", + "status": "repair_required", + "operation": "stale_state_test", + "source_scope": source_scope, + "source_document_id": document["id"], + } + settings_container.items[(state_doc_id, state_doc_id)] = { + "id": state_doc_id, + "type": "document_access_index_repair_backlog_state", + "has_repair_backlog": True, + "schema_version": indexing.DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION, + } + + result = indexing.sync_document_access_index_for_document(document, force=True) + + assert result["success"] is True + assert (repair_doc_id, repair_doc_id) not in settings_container.items + assert (state_doc_id, state_doc_id) not in settings_container.items + + +def test_wave3b_container_flags_and_hooks_are_wired(): + """Contract test for Wave 3B config, settings, and write-through hook markers.""" + config_source = open(os.path.join(SINGLE_APP_DIR, "config.py"), "r", encoding="utf-8").read() + settings_source = open(os.path.join(SINGLE_APP_DIR, "functions_settings.py"), "r", encoding="utf-8").read() + documents_source = open(os.path.join(SINGLE_APP_DIR, "functions_documents.py"), "r", encoding="utf-8").read() + personal_route_source = open(os.path.join(SINGLE_APP_DIR, "route_backend_documents.py"), "r", encoding="utf-8").read() + + assert "cosmos_document_access_index_container_name = \"document_access_index\"" in config_source + assert "PartitionKey(path=\"/scope_key\")" in config_source + assert "'enable_document_access_index_container': True" in settings_source + assert "'enable_document_access_index_write_through': True" in settings_source + assert "'enable_document_access_index_reads': True" in settings_source + assert "'enable_startup_document_access_index_backfill': True" in settings_source + for marker in [ + "operation='document_created'", + "operation='document_updated'", + "operation='document_deleted'", + "operation='document_revision_promoted'", + "operation='document_shared_with_user'", + "operation='document_unshared_from_user'", + "operation='document_shared_with_group'", + "operation='document_unshared_from_group'", + ]: + assert marker in documents_source + assert "operation='document_share_approved'" in personal_route_source + + +if __name__ == "__main__": + tests = [ + test_projection_rows_are_scope_partitioned_and_deterministic, + test_projection_share_collisions_prefer_least_privilege, + test_projection_sync_removes_stale_share_rows, + test_projection_delete_removes_all_source_rows, + test_projection_failure_records_repair_state_without_raising, + test_projection_failure_records_repair_doc_when_backlog_state_write_fails, + test_projection_repair_clear_removes_stale_backlog_state_when_false_write_fails, + test_wave3b_container_flags_and_hooks_are_wired, + ] + results = [] + for test in tests: + print(f"Running {test.__name__}...") + try: + test() + print("Test passed.") + results.append(True) + except Exception as exc: + print(f"Test failed: {exc}") + results.append(False) + + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_cosmos_wave4a1_admin_document_access_ui.py b/functional_tests/test_cosmos_wave4a1_admin_document_access_ui.py new file mode 100644 index 00000000..280afb18 --- /dev/null +++ b/functional_tests/test_cosmos_wave4a1_admin_document_access_ui.py @@ -0,0 +1,245 @@ +# test_cosmos_wave4a1_admin_document_access_ui.py +#!/usr/bin/env python3 +""" +Functional test for Cosmos Wave 4A1 document access admin UI. +Version: 0.250.047 +Implemented in: 0.250.011 +Default read enablement updated in: 0.250.027 +Redis DAI cache dashboard updated in: 0.250.029 +Maintenance status gates updated in: 0.250.030 +DAI debug UI cleanup updated in: 0.250.031 +Conversation cache controls updated in: 0.250.033 +Conversation cache metrics updated in: 0.250.034 +Conversation mark-read cache invalidation tuned in: 0.250.035 + +This test ensures Admin Settings exposes safe document_access_index +operational controls, status polling hooks, manual batch execution, +reset confirmation, and the Wave 5B default read path. +""" + +import os +import sys + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SINGLE_APP_DIR = os.path.join(ROOT_DIR, "application", "single_app") + + +def _read(relative_path): + with open(os.path.join(ROOT_DIR, relative_path), "r", encoding="utf-8") as file: + return file.read() + + +def test_admin_template_exposes_safe_document_access_controls(): + """The Scale tab should render document access index status and safe controls.""" + template = _read(os.path.join("application", "single_app", "templates", "admin_settings.html")) + required_ids = [ + "document-access-index-section", + "enable_document_access_index_write_through", + "enable_startup_document_access_index_backfill", + "document_access_index_backfill_batch_size", + "document_access_index_repair_batch_size", + "enable_document_access_index_shadow_validation", + "document-access-index-refresh-btn", + "document-access-index-run-batch-btn", + "document-access-index-reset-btn", + "documentAccessIndexResetModal", + "document-access-index-reset-confirm-btn", + "document-access-index-backfill-status", + "document-access-index-repair-count", + "document-access-index-maintenance-mode", + "document-access-index-maintenance-next-action", + "document-access-index-maintenance-more-work", + "document-access-index-maintenance-active-interval", + "document-access-index-read-15m-attempts", + "document-access-index-read-15m-served", + "document-access-index-read-15m-fallbacks", + "document-access-index-read-15m-fallback-rate", + "document-access-index-read-15m-ru", + "document-access-index-read-15m-latency", + "document-access-index-read-last-fallback", + "document-access-index-read-last-sample", + "document-access-index-shadow-last-status", + "document-access-index-shadow-mismatches", + "document-access-index-shadow-ru-comparison", + "document-access-index-shadow-validation-ru", + "document-access-index-shadow-candidate-ru", + "document-access-index-shadow-wave5-ru-savings", + "document-access-index-shadow-ms-comparison", + "document-access-index-shadow-ms-savings", + "document-access-index-rolling-5m-ru-comparison", + "document-access-index-rolling-5m-wave5-ru-savings", + "document-access-index-rolling-15m-ru-comparison", + "document-access-index-rolling-15m-wave5-ru-savings", + "document-access-index-rolling-15m-validation-overhead", + "document-access-index-rolling-15m-samples", + "conversation-cache-section", + "conversation-cache-refresh-btn", + "enable_conversation_cache", + "conversation_cache_ttl_seconds", + "conversation-cache-runtime-status", + "conversation-cache-15m-hit-rate", + "conversation-cache-15m-hits-misses", + "conversation-cache-15m-bypasses-errors", + "conversation-cache-15m-writes-invalidations", + "conversation-cache-15m-operation-counts", + "conversation-cache-last-event", + "conversation-cache-last-invalidation", + ] + + for element_id in required_ids: + assert f'id="{element_id}"' in template + + assert "Cosmos Document Access Index" in template + assert "Document access projection maintenance is automatic." in template + assert "Production read metrics below show DAI-served reads, Redis cache hits, source fallbacks, RU, and latency without requiring shadow validation." in template + assert "Rolling decision metrics aggregate shadow-validation samples over recent windows." in template + assert "Document access index reads" in template + assert "Enable shadow validation" in template + assert 'name="enable_document_access_index_reads"' in template + assert "Wave 5B default" in template + assert "Wave 6" in template + assert "Redis document list cache" in template + assert "Conversation Cache" in template + assert "Enable conversation cache" in template + assert 'name="enable_conversation_cache"' in template + assert 'name="conversation_cache_ttl_seconds"' in template + assert "Redis is optional; cache misses and disabled cache paths continue using source Cosmos queries." in template + assert "15m Cache Hit Rate" in template + assert "15m Writes / Invalidations" in template + assert "Phase 4" not in template[template.index('id="conversation-cache-section"'):template.index('id="document-access-index-section"')] + assert 'name="enable_document_access_index_shadow_validation"' in template + assert "{% if enable_dai_debug %}" in template + assert "data-testid=\"document-access-index-debug-controls\"" in template + assert "data-testid=\"document-access-index-debug-read-controls\"" in template + assert 'name="enable_dai_debug"' not in template + + +def test_admin_save_persists_document_access_settings(): + """Admin settings POST should preserve hidden DAI debug controls safely.""" + route_source = _read(os.path.join("application", "single_app", "route_frontend_admin_settings.py")) + backend_route_source = _read(os.path.join("application", "single_app", "route_backend_settings.py")) + + assert "'enable_document_access_index_write_through': True" in route_source + assert "'enable_startup_document_access_index_backfill': True" in route_source + assert "'document_access_index_active_maintenance_interval_seconds': settings.get(" in route_source + assert "'document_access_index_backfill_batch_size': document_access_index_backfill_batch_size" in route_source + assert "'document_access_index_repair_batch_size': document_access_index_repair_batch_size" in route_source + assert "'enable_document_access_index_reads': True" in route_source + assert "'enable_dai_debug': dai_debug_enabled" in route_source + assert "dai_debug_enabled = bool(settings.get('enable_dai_debug', False))" in route_source + assert "'enable_document_access_index_cache': document_access_index_cache_enabled" in route_source + assert "'document_access_index_cache_ttl_seconds': document_access_index_cache_ttl_seconds" in route_source + assert "'enable_conversation_cache': form_data.get('enable_conversation_cache') == 'on'" in route_source + assert "'conversation_cache_ttl_seconds': conversation_cache_ttl_seconds" in route_source + assert "settings.get('conversation_cache_ttl_seconds', 120)" in route_source + assert "'enable_document_access_index_shadow_validation': document_access_index_shadow_validation_enabled" in route_source + assert "settings.get('document_access_index_cache_ttl_seconds', 900)" in route_source + assert "if 'apply_cosmos_indexing_policies' in payload:" in backend_route_source + assert "apply_indexing_policies=apply_indexing_policies" in backend_route_source + + +def test_document_access_status_contract_includes_dashboard_settings(): + """Status payload should include the flags needed by the admin dashboard.""" + index_source = _read(os.path.join("application", "single_app", "functions_document_access_index.py")) + maintenance_source = _read(os.path.join("application", "single_app", "functions_app_maintenance.py")) + conversation_cache_source = _read(os.path.join("application", "single_app", "functions_conversation_cache.py")) + + assert "'write_through_enabled': True" in index_source + assert "'startup_backfill_enabled': True" in index_source + assert "'container_enabled': normalized_settings.get('container_enabled')" in index_source + assert "'write_through_enabled': normalized_settings.get('write_through_enabled')" in index_source + assert "'reads_enabled': normalized_settings.get('reads_enabled')" in index_source + assert "'shadow_validation_enabled': normalized_settings.get('shadow_validation_enabled')" in index_source + assert "'startup_backfill_enabled': normalized_settings.get('startup_backfill_enabled')" in index_source + assert "'read_metrics': get_document_access_index_read_metrics()" in index_source + assert "'cache_metrics': get_document_access_index_cache_metrics()" in index_source + assert "'cache_enabled': normalized_settings.get('cache_enabled')" in index_source + assert "'cache_ttl_seconds': normalized_settings.get('cache_ttl_seconds')" in index_source + assert "'maintenance': _build_document_access_index_maintenance_summary(" in index_source + assert "'shadow_validation': shadow_validation" in index_source + assert "'source_query_ru': None" in index_source + assert "'projection_query_ru': None" in index_source + assert "'validation_index_ru': None" in index_source + assert "'candidate_read_ru': None" in index_source + assert "'estimated_ru_savings': None" in index_source + assert "'estimated_wave5_ru_savings': None" in index_source + assert "'source_query_ms': None" in index_source + assert "'projection_query_ms': None" in index_source + assert "'validation_index_ms': None" in index_source + assert "'candidate_read_ms': None" in index_source + assert "'estimated_ms_savings': None" in index_source + assert "'estimated_wave5_ms_savings': None" in index_source + assert "shadow_validation['rolling_metrics'] = _empty_shadow_rolling_metrics()" in index_source + assert "get_conversation_cache_metrics" in maintenance_source + assert "get_conversation_cache_settings" in maintenance_source + assert "'conversation_cache': {" in maintenance_source + assert "'metrics': get_conversation_cache_metrics()" in maintenance_source + assert "def get_conversation_cache_metrics()" in conversation_cache_source + assert "CONVERSATION_CACHE_METRIC_WINDOWS_MINUTES = (5, 15, 60)" in conversation_cache_source + + +def test_admin_javascript_uses_existing_maintenance_api_safely(): + """Client-side controls should use Bootstrap/DOM APIs and the maintenance endpoints.""" + admin_js = _read(os.path.join("application", "single_app", "static", "js", "admin", "admin_settings.js")) + document_access_js = admin_js[ + admin_js.index("function setDocumentAccessIndexMessage"): + admin_js.index("function setupCosmosThroughputControls") + ] + + assert "function setupDocumentAccessIndexControls()" in admin_js + assert "function loadDocumentAccessIndexStatus" in admin_js + assert "function runDocumentAccessIndexBackfillBatch" in admin_js + assert "'/api/admin/settings/app-maintenance/status'" in admin_js + assert "'/api/admin/settings/app-maintenance/run'" in admin_js + assert "apply_cosmos_indexing_policies: false" in admin_js + assert "run_document_access_index_backfill: true" in admin_js + assert "reset_document_access_index_backfill: reset" in admin_js + assert "function isDocumentAccessIndexBackfillRunning" in admin_js + assert "getNormalizedDocumentAccessIndexStatus(status) === 'running'" in admin_js + assert "function isDocumentAccessIndexBackfillInProgress" in admin_js + assert "function getDocumentAccessIndexRollingWindow" in admin_js + assert "function getDocumentAccessIndexReadWindow" in admin_js + assert "document-access-index-read-15m-fallback-rate" in admin_js + assert "function renderConversationCacheStatus" in admin_js + assert "function formatConversationCacheOperationCounts" in admin_js + assert "conversation-cache-refresh-btn" in admin_js + assert "conversation-cache-15m-hit-rate" in admin_js + assert "conversation-cache-15m-writes-invalidations" in admin_js + assert "document-access-index-maintenance-next-action" in admin_js + assert "formatDocumentAccessIndexSampleSummary" in admin_js + assert "function isDocumentAccessIndexBackfillActive" not in admin_js + assert "documentAccessIndexResetModal" in admin_js + assert "bootstrap.Modal.getInstance(modalElement)?.hide();" in admin_js + assert "return ['running', 'in_progress'].includes" not in document_access_js + assert "confirm(" not in document_access_js + assert "alert(" not in document_access_js + + +def test_wave4a1_version_is_current(): + """Config version should reflect the current cache admin UI change.""" + config_source = _read(os.path.join("application", "single_app", "config.py")) + + assert 'VERSION = "0.250.047"' in config_source + + +if __name__ == "__main__": + tests = [ + test_admin_template_exposes_safe_document_access_controls, + test_admin_save_persists_document_access_settings, + test_document_access_status_contract_includes_dashboard_settings, + test_admin_javascript_uses_existing_maintenance_api_safely, + test_wave4a1_version_is_current, + ] + results = [] + for test in tests: + print(f"Running {test.__name__}...") + try: + test() + print("Test passed.") + results.append(True) + except Exception as exc: + print(f"Test failed: {exc}") + results.append(False) + + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_cosmos_wave4a_document_access_backfill.py b/functional_tests/test_cosmos_wave4a_document_access_backfill.py new file mode 100644 index 00000000..24dde02a --- /dev/null +++ b/functional_tests/test_cosmos_wave4a_document_access_backfill.py @@ -0,0 +1,437 @@ +# test_cosmos_wave4a_document_access_backfill.py +#!/usr/bin/env python3 +""" +Functional test for Cosmos Wave 4A document access index backfill. +Version: 0.250.047 +Implemented in: 0.250.010 +Default read enablement updated in: 0.250.027 +Redis DAI cache invalidation updated in: 0.250.029 +Maintenance status gates updated in: 0.250.030 +DAI cache TTL default updated in: 0.250.031 +Settings state read cache updated in: 0.250.037 + +This test ensures the document_access_index backfill is resumable, +scope-limited, and can reconcile repair records left by fail-open +write-through operations. +""" + +import copy +import importlib +import os +import sys +import types +from contextlib import contextmanager + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SINGLE_APP_DIR = os.path.join(ROOT_DIR, "application", "single_app") +if SINGLE_APP_DIR not in sys.path: + sys.path.insert(0, SINGLE_APP_DIR) + + +class FakeCosmosError(Exception): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class FakePageIterator: + def __init__(self, items, continuation_token=None, max_item_count=100): + self.items = list(items) + self.index = int(continuation_token or 0) + self.max_item_count = max(int(max_item_count or 100), 1) + self.continuation_token = None + self._used = False + + def __iter__(self): + return self + + def __next__(self): + if self._used or self.index >= len(self.items): + raise StopIteration + start = self.index + end = min(start + self.max_item_count, len(self.items)) + self._used = True + self.continuation_token = str(end) if end < len(self.items) else None + return copy.deepcopy(self.items[start:end]) + + +class FakePagedResult: + def __init__(self, items, max_item_count=100): + self.items = list(items) + self.max_item_count = max_item_count + + def __iter__(self): + return iter(copy.deepcopy(self.items)) + + def by_page(self, continuation_token=None): + return FakePageIterator( + self.items, + continuation_token=continuation_token, + max_item_count=self.max_item_count, + ) + + +class FakeCosmosContainer: + def __init__(self, initial_items=None): + self.items = {} + self.read_counts = {} + for item in list(initial_items or []): + self.upsert_item(item) + + def _partition_key_for_body(self, body): + return body.get("scope_key") or body.get("id") + + def upsert_item(self, body): + partition_key = self._partition_key_for_body(body) + self.items[(partition_key, body["id"])] = copy.deepcopy(body) + return copy.deepcopy(body) + + def read_item(self, item, partition_key): + key = (partition_key, item) + self.read_counts[key] = self.read_counts.get(key, 0) + 1 + if key not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + return copy.deepcopy(self.items[key]) + + def delete_item(self, item, partition_key, **kwargs): + key = (partition_key, item) + if key not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + del self.items[key] + + def query_items(self, query, parameters=None, enable_cross_partition_query=False, **kwargs): + parameter_map = { + parameter["name"]: parameter["value"] + for parameter in list(parameters or []) + } + item_type = parameter_map.get("@type") + item_status = parameter_map.get("@status") + document_metadata_type = parameter_map.get("@document_metadata_type") + source_scope = parameter_map.get("@source_scope") + document_id = parameter_map.get("@document_id") + results = [] + for item in self.items.values(): + if item_type is not None and item.get("type") != item_type: + continue + if item_status is not None and item.get("status") != item_status: + continue + if document_metadata_type is not None and item.get("type") not in (None, document_metadata_type): + continue + if source_scope is not None and item.get("source_scope") != source_scope: + continue + if document_id is not None and item.get("source_document_id") != document_id: + continue + results.append(copy.deepcopy(item)) + if "SELECT VALUE COUNT" in query.upper(): + return FakePagedResult([len(results)], max_item_count=kwargs.get("max_item_count", 100)) + results.sort(key=lambda item: item.get("id", "")) + return FakePagedResult(results, max_item_count=kwargs.get("max_item_count", 100)) + + +class FakeRepairBacklogStateWriteFailContainer(FakeCosmosContainer): + def upsert_item(self, body): + if body.get("type") == "document_access_index_repair_backlog_state": + raise RuntimeError("state write throttled") + return super().upsert_item(body) + + +def _document(document_id, owner_id): + return { + "id": document_id, + "type": "document_metadata", + "user_id": owner_id, + "file_name": f"{document_id}.pdf", + "version": 1, + "is_current_version": True, + "search_visibility_state": "active", + "shared_user_ids": [], + } + + +@contextmanager +def _load_document_access_index_module(personal_documents=None, settings_container=None): + original_modules = {} + for module_name in [ + "config", + "functions_appinsights", + "functions_settings", + "functions_document_access_index", + ]: + if module_name in sys.modules: + original_modules[module_name] = sys.modules[module_name] + del sys.modules[module_name] + + index_container = FakeCosmosContainer() + settings_container = settings_container or FakeCosmosContainer() + personal_container = FakeCosmosContainer(personal_documents) + group_container = FakeCosmosContainer() + public_container = FakeCosmosContainer() + + fake_config = types.ModuleType("config") + fake_config.cosmos_document_access_index_container = index_container + fake_config.cosmos_settings_container = settings_container + fake_config.cosmos_user_documents_container = personal_container + fake_config.cosmos_user_documents_container_name = "documents" + fake_config.cosmos_group_documents_container = group_container + fake_config.cosmos_group_documents_container_name = "group_documents" + fake_config.cosmos_public_documents_container = public_container + fake_config.cosmos_public_documents_container_name = "public_documents" + fake_config.cosmos_groups_container = FakeCosmosContainer() + fake_config.cosmos_public_workspaces_container = FakeCosmosContainer() + fake_config.cosmos_user_settings_container = FakeCosmosContainer() + sys.modules["config"] = fake_config + + fake_appinsights = types.ModuleType("functions_appinsights") + fake_appinsights.log_event = lambda *args, **kwargs: None + sys.modules["functions_appinsights"] = fake_appinsights + + fake_settings = types.ModuleType("functions_settings") + fake_settings.get_settings = lambda: { + "enable_document_access_index_container": True, + "enable_document_access_index_write_through": True, + "enable_document_access_index_reads": True, + "enable_document_access_index_shadow_validation": False, + "enable_startup_document_access_index_backfill": True, + "document_access_index_backfill_batch_size": 1, + "document_access_index_repair_batch_size": 10, + } + sys.modules["functions_settings"] = fake_settings + + try: + yield ( + importlib.import_module("functions_document_access_index"), + index_container, + settings_container, + personal_container, + ) + finally: + for module_name in [ + "config", + "functions_appinsights", + "functions_settings", + "functions_document_access_index", + ]: + sys.modules.pop(module_name, None) + sys.modules.update(original_modules) + + +def test_backfill_batches_are_resumable_and_complete(): + """Backfill should checkpoint continuation tokens between bounded batches.""" + source_documents = [ + _document("doc-1", "owner-1"), + _document("doc-2", "owner-2"), + ] + with _load_document_access_index_module(source_documents) as ( + indexing, + index_container, + _settings_container, + _personal_container, + ): + first_result = indexing.run_document_access_index_backfill_once( + batch_size=1, + source_scopes=["personal"], + reset=True, + force=True, + ) + second_result = indexing.run_document_access_index_backfill_once( + batch_size=1, + source_scopes=["personal"], + force=True, + ) + + assert first_result["success"] is True + assert first_result["status"] == "in_progress" + assert first_result["processed_count"] == 1 + assert first_result["state"]["continuation_tokens"]["personal"] == "1" + assert second_result["success"] is True + assert second_result["status"] == "succeeded" + assert second_result["processed_count"] == 1 + assert ("user:owner-1", "dai:user:owner-1:personal:doc-1:1") in index_container.items + assert ("user:owner-2", "dai:user:owner-2:personal:doc-2:1") in index_container.items + + +def test_repair_reconciliation_cleans_up_failed_delete_projection_rows(): + """Repair reconciliation should clean stale projection rows for delete repairs.""" + with _load_document_access_index_module([]) as ( + indexing, + index_container, + settings_container, + _personal_container, + ): + deleted_document = _document("doc-deleted", "owner-1") + indexing.sync_document_access_index_for_document(deleted_document, force=True) + settings_container.upsert_item({ + "id": "document_access_projection_repair:personal:doc-deleted", + "type": "document_access_index_repair", + "status": "repair_required", + "operation": "document_deleted", + "source_scope": "personal", + "source_document_id": "doc-deleted", + }) + + result = indexing.reconcile_document_access_index_repair_documents(force=True) + + assert result["success"] is True + assert result["repairs_processed"] == 1 + assert index_container.items == {} + remaining_repair_docs = [ + item + for item in settings_container.items.values() + if item.get("type") == "document_access_index_repair" + ] + repair_backlog_state = settings_container.items.get(( + "document_access_index_repair_backlog_state", + "document_access_index_repair_backlog_state", + )) + assert remaining_repair_docs == [] + assert repair_backlog_state["has_repair_backlog"] is False + + +def test_repair_reconciliation_tolerates_backlog_state_write_failure(): + """A failed backlog-state write should not fail completed repair reconciliation.""" + settings_container = FakeRepairBacklogStateWriteFailContainer() + with _load_document_access_index_module([], settings_container=settings_container) as ( + indexing, + index_container, + _settings_container, + _personal_container, + ): + deleted_document = _document("doc-state-write", "owner-1") + indexing.sync_document_access_index_for_document(deleted_document, force=True) + settings_container.upsert_item({ + "id": "document_access_projection_repair:personal:doc-state-write", + "type": "document_access_index_repair", + "status": "repair_required", + "operation": "document_deleted", + "source_scope": "personal", + "source_document_id": "doc-state-write", + }) + + result = indexing.reconcile_document_access_index_repair_documents(force=True) + + assert result["success"] is True + assert result["repairs_processed"] == 1 + assert index_container.items == {} + + +def test_succeeded_with_errors_without_repairs_is_not_active_backfill_work(): + """Completed backfills with historical errors should idle once repairs are clear.""" + with _load_document_access_index_module([]) as ( + indexing, + _index_container, + settings_container, + _personal_container, + ): + settings_container.upsert_item({ + "id": "document_access_index_backfill_state", + "type": "document_access_index_backfill_state", + "status": "succeeded_with_errors", + "source_scopes": ["personal", "group", "public"], + "completed_source_scopes": ["personal", "group", "public"], + "total_documents_processed": 3, + "total_documents_failed": 1, + "schema_version": 2, + }) + + status = indexing.get_document_access_index_backfill_status() + + assert status["state"]["status"] == "succeeded_with_errors" + assert status["repair_required_count"] == 0 + assert status["maintenance"]["has_more_work"] is False + assert status["maintenance"]["next_action"] == "monitor" + assert indexing.is_document_access_index_maintenance_pending(status) is False + + +def test_backfill_status_uses_short_ttl_cache_for_state_reads(): + """Repeated status reads should not point-read unchanged DAI state docs every time.""" + settings_container = FakeCosmosContainer() + state_doc_id = "document_access_index_backfill_state" + settings_container.upsert_item({ + "id": state_doc_id, + "type": state_doc_id, + "status": "succeeded", + "source_scopes": ["personal", "group", "public"], + "completed_source_scopes": ["personal", "group", "public"], + "total_documents_processed": 3, + "schema_version": 2, + }) + + with _load_document_access_index_module([], settings_container=settings_container) as ( + indexing, + _index_container, + _settings_container, + _personal_container, + ): + first_status = indexing.get_document_access_index_backfill_status() + second_status = indexing.get_document_access_index_backfill_status() + + assert first_status["state"]["status"] == "succeeded" + assert second_status["state"]["status"] == "succeeded" + assert settings_container.read_counts[(state_doc_id, state_doc_id)] == 1 + + +def test_backfill_status_does_not_read_shadow_state_when_shadow_disabled(): + """Disabled shadow validation should not point-read shadow validation state.""" + settings_container = FakeCosmosContainer() + shadow_state_doc_id = "document_access_index_shadow_validation_state" + settings_container.upsert_item({ + "id": shadow_state_doc_id, + "type": shadow_state_doc_id, + "status": "historical", + "missing_count": 99, + }) + + with _load_document_access_index_module([], settings_container=settings_container) as ( + indexing, + _index_container, + _settings_container, + _personal_container, + ): + status = indexing.get_document_access_index_backfill_status() + + assert status["settings"]["shadow_validation_enabled"] is False + assert status["shadow_validation"]["status"] == "not_run" + assert (shadow_state_doc_id, shadow_state_doc_id) not in settings_container.read_counts + + +def test_wave4a_settings_and_maintenance_contract_are_wired(): + """Contract test for Wave 4A defaults and maintenance hook markers.""" + config_source = open(os.path.join(SINGLE_APP_DIR, "config.py"), "r", encoding="utf-8").read() + settings_source = open(os.path.join(SINGLE_APP_DIR, "functions_settings.py"), "r", encoding="utf-8").read() + maintenance_source = open(os.path.join(SINGLE_APP_DIR, "functions_app_maintenance.py"), "r", encoding="utf-8").read() + route_source = open(os.path.join(SINGLE_APP_DIR, "route_backend_settings.py"), "r", encoding="utf-8").read() + + assert 'VERSION = "0.250.047"' in config_source + assert "'enable_startup_document_access_index_backfill': True" in settings_source + assert "'document_access_index_backfill_batch_size': 200" in settings_source + assert "'document_access_index_repair_batch_size': 100" in settings_source + assert "'document_access_index_active_maintenance_interval_seconds': 30" in settings_source + assert "'enable_document_access_index_cache': True" in settings_source + assert "'document_access_index_cache_ttl_seconds': 900" in settings_source + assert "run_document_access_index_backfill_maintenance" in maintenance_source + assert "run_document_access_backfill=payload.get('run_document_access_index_backfill')" in route_source + + +if __name__ == "__main__": + tests = [ + test_backfill_batches_are_resumable_and_complete, + test_repair_reconciliation_cleans_up_failed_delete_projection_rows, + test_repair_reconciliation_tolerates_backlog_state_write_failure, + test_succeeded_with_errors_without_repairs_is_not_active_backfill_work, + test_backfill_status_uses_short_ttl_cache_for_state_reads, + test_backfill_status_does_not_read_shadow_state_when_shadow_disabled, + test_wave4a_settings_and_maintenance_contract_are_wired, + ] + results = [] + for test in tests: + print(f"Running {test.__name__}...") + try: + test() + print("Test passed.") + results.append(True) + except Exception as exc: + print(f"Test failed: {exc}") + results.append(False) + + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_cosmos_wave4b_document_access_shadow_validation.py b/functional_tests/test_cosmos_wave4b_document_access_shadow_validation.py new file mode 100644 index 00000000..97c78aac --- /dev/null +++ b/functional_tests/test_cosmos_wave4b_document_access_shadow_validation.py @@ -0,0 +1,750 @@ +# test_cosmos_wave4b_document_access_shadow_validation.py +#!/usr/bin/env python3 +""" +Functional test for Cosmos Wave 4B document access shadow validation. +Version: 0.250.047 +Implemented in: 0.250.012 +Metrics added in: 0.250.013 +Candidate read metrics added in: 0.250.014 +Candidate read scope query corrected in: 0.250.015 +Settings fail-open and source timestamp projection added in: 0.250.016 +Lazy Cosmos diagnostics hook filtering added in: 0.250.017 +Rolling aggregate metrics added in: 0.250.021 +Read switch canary added in: 0.250.022 +Default read enablement updated in: 0.250.027 +Redis DAI cache metrics updated in: 0.250.029 +Family-identity shadow validation updated in: 0.250.030 + +This test ensures document_access_index shadow validation compares source +document list results with projection rows, records parity and estimated +RU/latency diagnostics, and does not enable read-path switchover. +""" + +import copy +import importlib +import os +import sys +import types +from contextlib import contextmanager +from datetime import datetime, timezone + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SINGLE_APP_DIR = os.path.join(ROOT_DIR, "application", "single_app") +if SINGLE_APP_DIR not in sys.path: + sys.path.insert(0, SINGLE_APP_DIR) + + +class FakeCosmosError(Exception): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class FakePagedResponse: + def by_page(self): + return [] + + +class FakeCosmosContainer: + def __init__(self, fail_query=False, request_charge=1.0, simulate_lazy_response_hook=False): + self.items = {} + self.fail_query = fail_query + self.request_charge = request_charge + self.simulate_lazy_response_hook = simulate_lazy_response_hook + self.etag_counter = 0 + + def _body_with_next_etag(self, body): + stored_body = copy.deepcopy(body) + self.etag_counter += 1 + stored_body["_etag"] = str(self.etag_counter) + return stored_body + + def _partition_key_for_body(self, body): + return body.get("scope_key") or body.get("id") + + def upsert_item(self, body): + partition_key = self._partition_key_for_body(body) + stored_body = self._body_with_next_etag(body) + self.items[(partition_key, body["id"])] = stored_body + return copy.deepcopy(stored_body) + + def create_item(self, body): + partition_key = self._partition_key_for_body(body) + key = (partition_key, body["id"]) + if key in self.items: + raise FakeCosmosError(409, f"Item already exists {body['id']}") + stored_body = self._body_with_next_etag(body) + self.items[key] = stored_body + return copy.deepcopy(stored_body) + + def replace_item(self, item, body, etag=None, match_condition=None): + partition_key = self._partition_key_for_body(body) + key = (partition_key, item) + if key not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + if etag and self.items[key].get("_etag") != etag: + raise FakeCosmosError(412, f"ETag conflict for {item}") + stored_body = self._body_with_next_etag(body) + self.items[key] = stored_body + return copy.deepcopy(stored_body) + + def read_item(self, item, partition_key): + key = (partition_key, item) + if key not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + return copy.deepcopy(self.items[key]) + + def delete_item(self, item, partition_key, **kwargs): + key = (partition_key, item) + if key not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + del self.items[key] + + def query_items(self, query, parameters=None, partition_key=None, **kwargs): + if self.fail_query: + raise RuntimeError("projection query unavailable") + + response_hook = kwargs.get("response_hook") + if response_hook: + if self.simulate_lazy_response_hook: + response_hook( + { + "x-ms-request-charge": "99.9", + "x-ms-activity-id": "stale-activity-id", + "x-ms-documentdb-query-metrics": "stale-query-metrics", + }, + FakePagedResponse(), + ) + response_hook( + { + "x-ms-request-charge": str(self.request_charge), + "x-ms-activity-id": "fake-activity-id", + "x-ms-documentdb-query-metrics": "fake-query-metrics", + }, + {}, + ) + + parameter_map = { + parameter["name"]: parameter["value"] + for parameter in list(parameters or []) + } + item_type = parameter_map.get("@type") + source_scope = parameter_map.get("@source_scope") + scope_key = parameter_map.get("@scope_key") or partition_key + document_id = parameter_map.get("@document_id") + results = [] + + for (item_partition_key, _item_id), item in self.items.items(): + if scope_key is not None and item_partition_key != scope_key: + continue + if item_type is not None and item.get("type") != item_type: + continue + if source_scope is not None and item.get("source_scope") != source_scope: + continue + if document_id is not None and item.get("source_document_id") != document_id: + continue + if ( + "AND c.access_granted = true" in query + and "OR c.approval_status = @approval_not_approved" not in query + and item.get("access_granted") is not True + ): + continue + if ( + "OR c.approval_status = @approval_not_approved" in query + and item.get("access_granted") is not True + and item.get("approval_status") != parameter_map.get("@approval_not_approved") + ): + continue + results.append(copy.deepcopy(item)) + + if "SELECT VALUE COUNT" in query.upper(): + return [len(results)] + return results + + +class ConflictOnceShadowStateContainer(FakeCosmosContainer): + def __init__(self, conflict_sample): + super().__init__() + self.conflict_sample = conflict_sample + self.inject_conflict_once = True + + def replace_item(self, item, body, etag=None, match_condition=None): + if self.inject_conflict_once: + self.inject_conflict_once = False + current_body = self.read_item(item, item) + current_body.setdefault("recent_metric_samples", []).append(copy.deepcopy(self.conflict_sample)) + self.upsert_item(current_body) + return super().replace_item(item, body, etag=etag, match_condition=match_condition) + + +def _document(document_id, owner_id, **overrides): + document = { + "id": document_id, + "type": "document_metadata", + "user_id": owner_id, + "file_name": f"{document_id}.pdf", + "title": document_id.title(), + "version": 1, + "revision_family_id": document_id, + "is_current_version": True, + "search_visibility_state": "active", + "status": "Processing complete", + "percentage_complete": 100, + "authors": [], + "keywords": [], + "abstract": "", + "tags": [], + "shared_user_ids": [], + } + document.update(overrides) + return document + + +@contextmanager +def _load_document_access_index_module(index_container=None, settings_container=None, settings=None): + original_modules = {} + for module_name in [ + "config", + "functions_appinsights", + "functions_settings", + "functions_document_access_index", + ]: + if module_name in sys.modules: + original_modules[module_name] = sys.modules[module_name] + del sys.modules[module_name] + + index_container = index_container or FakeCosmosContainer() + settings_container = settings_container or FakeCosmosContainer() + fake_config = types.ModuleType("config") + fake_config.cosmos_document_access_index_container = index_container + fake_config.cosmos_settings_container = settings_container + fake_config.cosmos_user_documents_container = FakeCosmosContainer() + fake_config.cosmos_user_documents_container_name = "documents" + fake_config.cosmos_group_documents_container = FakeCosmosContainer() + fake_config.cosmos_group_documents_container_name = "group_documents" + fake_config.cosmos_public_documents_container = FakeCosmosContainer() + fake_config.cosmos_public_documents_container_name = "public_documents" + fake_config.cosmos_groups_container = FakeCosmosContainer() + fake_config.cosmos_public_workspaces_container = FakeCosmosContainer() + fake_config.cosmos_user_settings_container = FakeCosmosContainer() + sys.modules["config"] = fake_config + + fake_appinsights = types.ModuleType("functions_appinsights") + fake_appinsights.log_event = lambda *args, **kwargs: None + sys.modules["functions_appinsights"] = fake_appinsights + + fake_settings = types.ModuleType("functions_settings") + fake_settings.get_settings = lambda: settings or { + "enable_document_access_index_container": True, + "enable_document_access_index_write_through": True, + "enable_document_access_index_reads": True, + "enable_document_access_index_shadow_validation": True, + "enable_startup_document_access_index_backfill": False, + } + sys.modules["functions_settings"] = fake_settings + + try: + yield importlib.import_module("functions_document_access_index"), index_container, settings_container + finally: + for module_name in [ + "config", + "functions_appinsights", + "functions_settings", + "functions_document_access_index", + ]: + sys.modules.pop(module_name, None) + sys.modules.update(original_modules) + + +def test_shadow_validation_matches_projection_rows(): + """Shadow validation should pass when source and projection identities match.""" + document = _document("doc-1", "owner-1", title="Quarterly Report") + + with _load_document_access_index_module() as (indexing, _index_container, settings_container): + indexing.sync_document_access_index_for_document(document, force=True) + result = indexing.validate_document_access_index_shadow( + [document], + source_scope="personal", + user_id="owner-1", + filters={"search": "report"}, + context="test_personal_match", + ) + + assert result["success"] is True + assert result["status"] == "matched" + assert result["authoritative_count"] == 1 + assert result["projection_count"] == 1 + state_doc = settings_container.read_item( + "document_access_index_shadow_validation_state", + "document_access_index_shadow_validation_state", + ) + assert state_doc["status"] == "matched" + + +def test_shadow_validation_reports_missing_and_extra_projection_rows(): + """Shadow validation should record missing and extra projection identities.""" + authoritative_document = _document("doc-source", "owner-1") + projection_only_document = _document("doc-projection", "owner-1") + + with _load_document_access_index_module() as (indexing, _index_container, _settings_container): + indexing.sync_document_access_index_for_document(projection_only_document, force=True) + result = indexing.validate_document_access_index_shadow( + [authoritative_document], + source_scope="personal", + user_id="owner-1", + context="test_personal_mismatch", + ) + + assert result["success"] is False + assert result["status"] == "mismatch" + assert result["missing_count"] == 1 + assert result["extra_count"] == 1 + assert "doc-source" in result["missing_sample"] + assert "doc-projection" in result["extra_sample"] + + +def test_shadow_validation_applies_projected_filter_fields(): + """Projected authors, keywords, abstracts, and tags should support filter parity.""" + matching_document = _document( + "doc-match", + "owner-1", + authors=["Ada Lovelace"], + keywords=["Cosmos"], + abstract="Projection validation notes", + tags=["planning", "wave4b"], + ) + filtered_document = _document( + "doc-filtered", + "owner-1", + authors=["Grace Hopper"], + keywords=["Compiler"], + abstract="Different content", + tags=["other"], + ) + + with _load_document_access_index_module() as (indexing, _index_container, _settings_container): + indexing.sync_document_access_index_for_document(matching_document, force=True) + indexing.sync_document_access_index_for_document(filtered_document, force=True) + result = indexing.validate_document_access_index_shadow( + [matching_document], + source_scope="personal", + user_id="owner-1", + filters={ + "author": "ada", + "keywords": "cosmos", + "abstract": "validation", + "tags": ["planning", "wave4b"], + }, + context="test_personal_filters", + ) + + assert result["success"] is True + assert result["projection_count"] == 1 + + +def test_shadow_validation_preserves_missing_group_classification_for_none_filter(): + """Group/public classification=none parity should preserve missing source values.""" + legacy_group_document = _document( + "doc-legacy", + "owner-1", + group_id="group-1", + ) + + with _load_document_access_index_module() as (indexing, index_container, _settings_container): + indexing.sync_document_access_index_for_document(legacy_group_document, force=True) + projection_rows = list(index_container.items.values()) + assert projection_rows[0].get("document_classification") is None + result = indexing.validate_document_access_index_shadow( + [legacy_group_document], + source_scope="group", + group_ids=["group-1"], + filters={ + "classification": "none", + "classification_none_matches_literal": False, + }, + context="test_group_classification_none", + ) + + assert result["success"] is True + assert result["projection_count"] == 1 + + +def test_shadow_validation_fails_open_on_projection_query_errors(): + """Projection query errors should be recorded and should not raise to callers.""" + document = _document("doc-1", "owner-1") + + with _load_document_access_index_module(index_container=FakeCosmosContainer(fail_query=True)) as ( + indexing, + _index_container, + settings_container, + ): + result = indexing.validate_document_access_index_shadow( + [document], + source_scope="personal", + user_id="owner-1", + context="test_query_error", + ) + + assert result["success"] is False + assert result["status"] == "error" + state_doc = settings_container.read_item( + "document_access_index_shadow_validation_state", + "document_access_index_shadow_validation_state", + ) + assert state_doc["status"] == "error" + + +def test_shadow_validation_records_ru_and_latency_estimates(): + """Shadow validation should persist source/index RU and latency comparison fields.""" + document = _document("doc-1", "owner-1", _ts=12345) + source_container = FakeCosmosContainer(request_charge=12.5, simulate_lazy_response_hook=True) + index_container = FakeCosmosContainer(request_charge=2.25, simulate_lazy_response_hook=True) + source_container.upsert_item(document) + + with _load_document_access_index_module(index_container=index_container) as ( + indexing, + _index_container, + settings_container, + ): + indexing.sync_document_access_index_for_document(document, force=True) + projection_rows = list(index_container.items.values()) + assert any(row.get("source_ts") == 12345 for row in projection_rows) + source_documents, source_query_metrics = indexing.query_items_with_cosmos_diagnostics( + source_container, + diagnostics_label="source_documents", + query="SELECT * FROM c", + parameters=[], + enable_cross_partition_query=True, + ) + result = indexing.validate_document_access_index_shadow( + source_documents, + source_scope="personal", + user_id="owner-1", + source_query_metrics=source_query_metrics, + context="test_shadow_metrics", + ) + + assert result["success"] is True + assert result["source_query_ru"] == 12.5 + assert result["projection_query_ru"] == 2.25 + assert result["validation_index_ru"] == 2.25 + assert result["candidate_read_ru"] == 2.25 + assert result["estimated_ru_savings"] == 10.25 + assert result["estimated_wave5_ru_savings"] == 10.25 + assert result["shadow_overhead_ru"] == 2.25 + assert result["source_query_ms"] is not None + assert result["projection_query_ms"] is not None + assert result["validation_index_ms"] is not None + assert result["candidate_read_ms"] is not None + assert result["estimated_ms_savings"] is not None + assert result["estimated_wave5_ms_savings"] is not None + assert result["source_query_page_count"] == 1 + assert result["candidate_read_page_count"] == 1 + state_doc = settings_container.read_item( + "document_access_index_shadow_validation_state", + "document_access_index_shadow_validation_state", + ) + assert state_doc["estimated_ru_savings"] == 10.25 + assert state_doc["estimated_wave5_ru_savings"] == 10.25 + rolling_15m = state_doc["rolling_metrics"]["windows"]["15m"] + assert rolling_15m["sample_count"] == 1 + assert rolling_15m["matched_count"] == 1 + assert rolling_15m["source_query_ru"] == 12.5 + assert rolling_15m["candidate_read_ru"] == 2.25 + assert rolling_15m["estimated_wave5_ru_savings"] == 10.25 + assert rolling_15m["validation_index_ru"] == 2.25 + + +def test_shadow_validation_rolls_up_multiple_samples_for_admin_decisions(): + """Rolling metrics should aggregate multiple validations for workflow-level decisions.""" + document = _document("doc-1", "owner-1", _ts=12345) + source_container = FakeCosmosContainer(request_charge=12.5) + index_container = FakeCosmosContainer(request_charge=2.25) + source_container.upsert_item(document) + + with _load_document_access_index_module(index_container=index_container) as ( + indexing, + _index_container, + settings_container, + ): + indexing.sync_document_access_index_for_document(document, force=True) + source_documents, source_query_metrics = indexing.query_items_with_cosmos_diagnostics( + source_container, + diagnostics_label="source_documents", + query="SELECT * FROM c", + parameters=[], + enable_cross_partition_query=True, + ) + indexing.validate_document_access_index_shadow( + source_documents, + source_scope="personal", + user_id="owner-1", + source_query_metrics=source_query_metrics, + context="test_shadow_metrics_first", + ) + + source_container.request_charge = 7.5 + index_container.request_charge = 1.5 + source_documents, source_query_metrics = indexing.query_items_with_cosmos_diagnostics( + source_container, + diagnostics_label="source_documents", + query="SELECT * FROM c", + parameters=[], + enable_cross_partition_query=True, + ) + indexing.validate_document_access_index_shadow( + source_documents, + source_scope="personal", + user_id="owner-1", + source_query_metrics=source_query_metrics, + context="test_shadow_metrics_second", + ) + partial_candidate_fields = indexing._build_shadow_metric_fields( + {"request_charge": 5.0, "elapsed_ms": 2.0, "item_count": 1, "page_count": 1}, + {"request_charge": 1.0, "elapsed_ms": 1.0, "item_count": 1, "page_count": 1}, + { + "request_charge": 0.5, + "elapsed_ms": 1.0, + "item_count": 1, + "page_count": 1, + "partial_failure": True, + }, + ) + assert partial_candidate_fields["candidate_read_ru"] is None + assert partial_candidate_fields["estimated_wave5_ru_savings"] is None + indexing._write_shadow_validation_state({ + "success": False, + "status": "error", + "context": "test_partial_candidate_metrics", + "source_scope": "personal", + "scope_keys": ["user:owner-1"], + "authoritative_count": 1, + "projection_count": 1, + "missing_count": 0, + "extra_count": 0, + "checked_at": indexing._utc_now_iso(), + "source_query_ru": 5.0, + "validation_index_ru": 1.0, + "candidate_read_ru": None, + "estimated_wave5_ru_savings": None, + "source_query_item_count": 1, + "candidate_read_item_count": 0, + "source_query_page_count": 1, + "candidate_read_page_count": 0, + }) + + state_doc = settings_container.read_item( + "document_access_index_shadow_validation_state", + "document_access_index_shadow_validation_state", + ) + assert len(state_doc["recent_metric_samples"]) == 3 + rolling_5m = state_doc["rolling_metrics"]["windows"]["5m"] + rolling_15m = state_doc["rolling_metrics"]["windows"]["15m"] + for rolling_window in [rolling_5m, rolling_15m]: + assert rolling_window["sample_count"] == 3 + assert rolling_window["comparable_sample_count"] == 2 + assert rolling_window["matched_count"] == 2 + assert rolling_window["error_count"] == 1 + assert rolling_window["source_query_ru"] == 20.0 + assert rolling_window["candidate_read_ru"] == 3.75 + assert rolling_window["validation_index_ru"] == 4.75 + assert rolling_window["estimated_wave5_ru_savings"] == 16.25 + assert rolling_window["source_query_item_count"] == 2 + assert rolling_window["candidate_read_item_count"] == 2 + + +def test_shadow_validation_retries_state_write_conflicts_without_losing_samples(): + """Concurrent shadow state updates should be retried without dropping metric samples.""" + conflict_sample = { + "checked_at": datetime.now(timezone.utc).isoformat(), + "status": "matched", + "context": "concurrent_shadow_sample", + "source_scope": "personal", + "scope_key_count": 1, + "authoritative_count": 1, + "projection_count": 1, + "missing_count": 0, + "extra_count": 0, + "source_query_ru": 3.0, + "validation_index_ru": 1.0, + "candidate_read_ru": 1.0, + "estimated_wave5_ru_savings": 2.0, + "shadow_overhead_ru": 1.0, + "source_query_item_count": 1, + "candidate_read_item_count": 1, + "source_query_page_count": 1, + "candidate_read_page_count": 1, + } + settings_container = ConflictOnceShadowStateContainer(conflict_sample) + + with _load_document_access_index_module(settings_container=settings_container) as ( + indexing, + _index_container, + _settings_container, + ): + indexing._write_shadow_validation_state({ + "success": True, + "status": "matched", + "context": "first_shadow_sample", + "source_scope": "personal", + "scope_keys": ["user:owner-1"], + "authoritative_count": 1, + "projection_count": 1, + "missing_count": 0, + "extra_count": 0, + "checked_at": indexing._utc_now_iso(), + "source_query_ru": 2.0, + "validation_index_ru": 0.5, + "candidate_read_ru": 0.5, + "estimated_wave5_ru_savings": 1.5, + "shadow_overhead_ru": 0.5, + "source_query_item_count": 1, + "candidate_read_item_count": 1, + "source_query_page_count": 1, + "candidate_read_page_count": 1, + }) + indexing._write_shadow_validation_state({ + "success": True, + "status": "matched", + "context": "second_shadow_sample", + "source_scope": "personal", + "scope_keys": ["user:owner-1"], + "authoritative_count": 1, + "projection_count": 1, + "missing_count": 0, + "extra_count": 0, + "checked_at": indexing._utc_now_iso(), + "source_query_ru": 4.0, + "validation_index_ru": 1.5, + "candidate_read_ru": 1.5, + "estimated_wave5_ru_savings": 2.5, + "shadow_overhead_ru": 1.5, + "source_query_item_count": 1, + "candidate_read_item_count": 1, + "source_query_page_count": 1, + "candidate_read_page_count": 1, + }) + + state_doc = settings_container.read_item( + "document_access_index_shadow_validation_state", + "document_access_index_shadow_validation_state", + ) + contexts = {sample["context"] for sample in state_doc["recent_metric_samples"]} + assert contexts == { + "first_shadow_sample", + "concurrent_shadow_sample", + "second_shadow_sample", + } + rolling_15m = state_doc["rolling_metrics"]["windows"]["15m"] + assert rolling_15m["sample_count"] == 3 + assert rolling_15m["comparable_sample_count"] == 3 + assert rolling_15m["source_query_ru"] == 9.0 + assert rolling_15m["candidate_read_ru"] == 3.0 + assert rolling_15m["estimated_wave5_ru_savings"] == 6.0 + + +def test_shadow_validation_skips_when_settings_are_unavailable(): + """Settings failures should disable optional shadow validation instead of failing source reads.""" + document = _document("doc-1", "owner-1") + + with _load_document_access_index_module() as (indexing, _index_container, _settings_container): + def raise_settings_error(): + raise RuntimeError("settings unavailable") + + indexing.get_settings = raise_settings_error + result = indexing.validate_document_access_index_shadow( + [document], + source_scope="personal", + user_id="owner-1", + context="test_settings_unavailable", + ) + + assert result["success"] is True + assert result["status"] == "skipped_disabled" + + +def test_wave4b_admin_routes_and_version_are_wired(): + """Contract test for Wave 4B UI setting, route hooks, and version.""" + config_source = open(os.path.join(SINGLE_APP_DIR, "config.py"), "r", encoding="utf-8").read() + route_settings_source = open( + os.path.join(SINGLE_APP_DIR, "route_frontend_admin_settings.py"), + "r", + encoding="utf-8", + ).read() + admin_template = open( + os.path.join(SINGLE_APP_DIR, "templates", "admin_settings.html"), + "r", + encoding="utf-8", + ).read() + personal_route = open(os.path.join(SINGLE_APP_DIR, "route_backend_documents.py"), "r", encoding="utf-8").read() + group_route = open(os.path.join(SINGLE_APP_DIR, "route_backend_group_documents.py"), "r", encoding="utf-8").read() + public_route = open(os.path.join(SINGLE_APP_DIR, "route_external_public_documents.py"), "r", encoding="utf-8").read() + functions_documents = open(os.path.join(SINGLE_APP_DIR, "functions_documents.py"), "r", encoding="utf-8").read() + + assert 'VERSION = "0.250.047"' in config_source + assert "'enable_dai_debug': dai_debug_enabled" in route_settings_source + assert "'enable_document_access_index_shadow_validation': document_access_index_shadow_validation_enabled" in route_settings_source + assert "if enable_dai_debug" in admin_template + assert 'name="enable_document_access_index_shadow_validation"' in admin_template + assert 'name="enable_document_access_index_reads"' in admin_template + assert "Wave 5B default" in admin_template + assert "Wave 6" in admin_template + assert "enable_document_access_index_shadow_validation_preview" not in admin_template + assert "validate_document_access_index_shadow(" in personal_route + assert "validate_document_access_index_shadow(" in group_route + assert "validate_document_access_index_shadow(" in public_route + assert "if collect_shadow_metrics:" in functions_documents + assert "source_query_metrics = None" in functions_documents + assert "def _upsert_document_and_sync_access_index" in functions_documents + assert "persisted_document if isinstance(persisted_document, dict) else document_item" in functions_documents + normalize_source = functions_documents[ + functions_documents.index("def normalize_document_revision_families"): + functions_documents.index("def _get_document_family_items_from_document") + ] + assert "operation='document_revision_normalized'" in normalize_source + assert "cosmos_container.upsert_item(document_item)" not in normalize_source + assert "cosmos_container.upsert_item(document_metadata)\n sync_document_access_index_for_document_fail_open" not in functions_documents + assert "document_for_sync = persisted_document if isinstance(persisted_document, dict) else document_item" in personal_route + index_source = open(os.path.join(SINGLE_APP_DIR, "functions_document_access_index.py"), "r", encoding="utf-8").read() + candidate_query_source = index_source[ + index_source.index("def _query_candidate_projection_rows_for_scope"): + index_source.index("def _collect_candidate_read_metrics") + ] + assert "ORDER BY" not in candidate_query_source + assert "OFFSET" not in candidate_query_source + assert "LIMIT" not in candidate_query_source + assert "TOP" not in candidate_query_source + assert "c.source_ts" in candidate_query_source + assert "partition_key=scope_key" in candidate_query_source + assert "DOCUMENT_ACCESS_SHADOW_METRIC_WINDOWS_MINUTES = (5, 15)" in index_source + assert "shadow_validation['rolling_metrics'] = _empty_shadow_rolling_metrics()" in index_source + assert "state_body = _merge_shadow_rolling_metrics(state_body, previous_state=previous_state)" in index_source + assert "MatchConditions.IfNotModified" in index_source + + +if __name__ == "__main__": + tests = [ + test_shadow_validation_matches_projection_rows, + test_shadow_validation_reports_missing_and_extra_projection_rows, + test_shadow_validation_applies_projected_filter_fields, + test_shadow_validation_preserves_missing_group_classification_for_none_filter, + test_shadow_validation_fails_open_on_projection_query_errors, + test_shadow_validation_records_ru_and_latency_estimates, + test_shadow_validation_rolls_up_multiple_samples_for_admin_decisions, + test_shadow_validation_retries_state_write_conflicts_without_losing_samples, + test_shadow_validation_skips_when_settings_are_unavailable, + test_wave4b_admin_routes_and_version_are_wired, + ] + results = [] + for test in tests: + print(f"Running {test.__name__}...") + try: + test() + print("Test passed.") + results.append(True) + except Exception as exc: + print(f"Test failed: {exc}") + results.append(False) + + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_cosmos_wave5a3_redis_monitoring.py b/functional_tests/test_cosmos_wave5a3_redis_monitoring.py new file mode 100644 index 00000000..ed971b98 --- /dev/null +++ b/functional_tests/test_cosmos_wave5a3_redis_monitoring.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python3 +# test_cosmos_wave5a3_redis_monitoring.py +""" +Functional test for Wave 5A3 Redis monitoring. +Version: 0.250.043 +Implemented in: 0.250.026 +Redis Explorer implemented in: 0.250.040 +Redis Explorer DAI resolution implemented in: 0.250.043 + +This test ensures Redis monitoring reports sanitized health, memory, stats, +keyspace, DAI cache hygiene, runtime signals, and read-only Redis Explorer +previews without exposing Redis secrets. +""" + +import fnmatch +import json +import os +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +APP_DIR = os.path.join(REPO_ROOT, "application", "single_app") +sys.path.insert(0, APP_DIR) + +import app_settings_cache +from functions_redis_monitoring import ( + get_redis_explorer_keys, + get_redis_explorer_value, + get_redis_monitoring_status, +) + + +class FakeRedisClient: + def __init__(self): + self.items = { + "simplechat:conversation_cache:user-1": { + "type": "string", + "value": json.dumps({ + "conversation_id": "conversation-1", + "title": "Visible cache title", + "api_key": "secret-api-key", + "nested": { + "token": "secret-token", + "safe": "safe-value", + }, + }), + "ttl": 120, + }, + "simplechat:dai:list:user-1": { + "type": "list", + "value": ["document-a", "document-b"], + "ttl": -1, + }, + "DAI_LIST_CACHE:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": { + "type": "string", + "value": json.dumps({"documents": [{"id": "document-a"}]}), + "ttl": 900, + }, + "DAI_LIST_CACHE_VERSION:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": { + "type": "string", + "value": "23", + "ttl": -1, + }, + "flask_session:abc123": { + "type": "string", + "value": "session-cookie-secret", + "ttl": 300, + }, + } + + def ping(self): + return True + + def info(self): + return { + "redis_version": "7.2.4", + "uptime_in_seconds": 3600, + "uptime_in_days": 0, + "connected_clients": 12, + "blocked_clients": 0, + "tracking_clients": 1, + "maxclients": 1000, + "used_memory": 1024, + "used_memory_human": "1.00K", + "maxmemory": 4096, + "maxmemory_human": "4.00K", + "maxmemory_policy": "allkeys-lru", + "mem_fragmentation_ratio": "1.15", + "total_connections_received": 42, + "total_commands_processed": 500, + "instantaneous_ops_per_sec": 9, + "keyspace_hits": 80, + "keyspace_misses": 20, + "expired_keys": 3, + "evicted_keys": 1, + "rejected_connections": 0, + "total_error_replies": 2, + "db0": {"keys": 17, "expires": 5, "avg_ttl": 30000}, + } + + def scan(self, cursor=0, match="*", count=25): + keys = [ + key + for key in sorted(self.items) + if fnmatch.fnmatch(key, match.replace("\\", "")) + ] + cursor_index = int(cursor or 0) + next_index = min(cursor_index + int(count or 25), len(keys)) + next_cursor = 0 if next_index >= len(keys) else next_index + return next_cursor, keys[cursor_index:next_index] + + def type(self, key): + return self.items.get(key, {}).get("type", "none") + + def ttl(self, key): + return self.items.get(key, {}).get("ttl", -2) + + def memory_usage(self, key): + value = self.items.get(key, {}).get("value") + return len(json.dumps(value)) + + def get(self, key): + return self.items.get(key, {}).get("value") + + def lrange(self, key, start, end): + value = self.items.get(key, {}).get("value") or [] + return value[start:end + 1] + + def hscan(self, key, cursor=0, count=20): + value = self.items.get(key, {}).get("value") or {} + return 0, value + + def sscan(self, key, cursor=0, count=20): + value = self.items.get(key, {}).get("value") or [] + return 0, value[:count] + + def zrange(self, key, start, end, withscores=False): + value = self.items.get(key, {}).get("value") or [] + return value[start:end + 1] + + def xrange(self, key, min="-", max="+", count=20): + value = self.items.get(key, {}).get("value") or [] + return value[:count] + + +class FakeRedisInfoFailure: + def ping(self): + return True + + def info(self): + raise RuntimeError("secret-key-material") + + +def _fake_dai_hash_resolver(scope_hashes): + """Return safe SimpleChat resolution metadata for test DAI version hashes.""" + return { + scope_hash: { + "kind": "document_access_index_version", + "resolved": True, + "resolution_status": "resolved", + "label": "Document Access Index scope version", + "cache_hash": scope_hash, + "scope_key": "user:test-user", + "entity_type": "user", + "entity_id": "test-user", + "entity_name": None, + "entity_status": "active", + "row_count": 2, + "source_scopes": ["personal"], + "access_roles": ["owner"], + "note": "Resolved by functional test resolver.", + } + for scope_hash in scope_hashes + } + + +def test_redis_monitoring_healthy_metrics(): + """Validate healthy Redis monitoring metrics and sanitized payload shape.""" + previous_runtime_flag = app_settings_cache.app_cache_is_using_redis + try: + app_settings_cache.app_cache_is_using_redis = True + status = get_redis_monitoring_status( + { + "enable_redis_cache": True, + "redis_url": "example.redis.cache.windows.net", + "redis_auth_type": "managed_identity", + "redis_key": "do-not-expose", + }, + app_cache_client=FakeRedisClient(), + session_type="filesystem", + now_func=lambda: "2026-07-03T00:00:00Z", + ) + finally: + app_settings_cache.app_cache_is_using_redis = previous_runtime_flag + + assert status["checked_at"] == "2026-07-03T00:00:00Z" + assert status["configuration"] == { + "enabled": True, + "configured": True, + "auth_type": "managed_identity", + } + assert status["runtime"]["app_cache_using_redis"] is True + assert status["runtime"]["monitoring_source"] == "app_cache" + assert status["health"]["status"] == "healthy" + assert status["health"]["ping_success"] is True + assert status["health"]["ping_latency_ms"] is not None + assert status["memory"]["usage_percent"] == 25.0 + assert status["stats"]["keyspace_hit_rate_percent"] == 80.0 + assert status["keyspace"]["total_keys"] == 17 + assert status["keyspace"]["expiring_keys"] == 5 + assert status["dai_cache"]["payload_key_count"] == 1 + assert status["dai_cache"]["version_marker_count"] == 1 + assert status["dai_cache"]["version_marker_no_expiry_count"] == 1 + assert status["dai_cache"]["version_marker_ttl_seconds"] == 3600 + + serialized_status = json.dumps(status) + assert "do-not-expose" not in serialized_status + assert "example.redis.cache.windows.net" not in serialized_status + + +def test_redis_monitoring_reports_runtime_unavailable(): + """Validate enabled Redis without an active client reports an actionable status.""" + status = get_redis_monitoring_status( + { + "enable_redis_cache": True, + "redis_url": "example.redis.cache.windows.net", + "redis_auth_type": "key", + "redis_key": "do-not-expose", + }, + app_cache_client=None, + session_redis_client=None, + session_type="filesystem", + now_func=lambda: "2026-07-03T00:00:00Z", + ) + + assert status["health"]["status"] == "unavailable" + assert status["runtime"]["client_available"] is False + assert status["health"]["last_error"] == "Redis is configured, but no active runtime Redis client is available." + assert "do-not-expose" not in json.dumps(status) + + +def test_redis_monitoring_sanitizes_info_failures(): + """Validate Redis INFO failures are surfaced without raw exception text.""" + status = get_redis_monitoring_status( + { + "enable_redis_cache": True, + "redis_url": "example.redis.cache.windows.net", + "redis_auth_type": "key", + "redis_key": "do-not-expose", + }, + app_cache_client=FakeRedisInfoFailure(), + now_func=lambda: "2026-07-03T00:00:00Z", + ) + + assert status["health"]["status"] == "degraded" + assert status["health"]["ping_success"] is True + assert status["health"]["last_error"] == "Redis INFO metrics failed (RuntimeError)." + serialized_status = json.dumps(status) + assert "secret-key-material" not in serialized_status + assert "do-not-expose" not in serialized_status + + +def test_redis_explorer_lists_keys_with_safe_metadata(): + """Validate Redis Explorer uses paged SCAN output and safe key metadata.""" + previous_runtime_flag = app_settings_cache.app_cache_is_using_redis + try: + app_settings_cache.app_cache_is_using_redis = True + page = get_redis_explorer_keys( + { + "enable_redis_cache": True, + "redis_url": "example.redis.cache.windows.net", + "redis_key": "do-not-expose", + }, + app_cache_client=FakeRedisClient(), + key_filter="simplechat", + page_size=1, + ) + finally: + app_settings_cache.app_cache_is_using_redis = previous_runtime_flag + + serialized_page = json.dumps(page) + assert page["success"] is True + assert page["page_size"] == 1 + assert len(page["keys"]) == 1 + assert page["has_more"] is True + assert page["keys"][0]["key"].startswith("simplechat:") + assert "do-not-expose" not in serialized_page + assert "example.redis.cache.windows.net" not in serialized_page + + +def test_redis_explorer_resolves_dai_version_marker_metadata(): + """Validate Redis Explorer resolves DAI marker hashes to safe entity metadata.""" + previous_runtime_flag = app_settings_cache.app_cache_is_using_redis + try: + app_settings_cache.app_cache_is_using_redis = True + page = get_redis_explorer_keys( + { + "enable_redis_cache": True, + "redis_url": "example.redis.cache.windows.net", + "redis_key": "do-not-expose", + }, + app_cache_client=FakeRedisClient(), + key_filter="DAI_LIST_CACHE_VERSION", + page_size=10, + dai_hash_resolver=_fake_dai_hash_resolver, + ) + preview = get_redis_explorer_value( + { + "enable_redis_cache": True, + "redis_url": "example.redis.cache.windows.net", + "redis_key": "do-not-expose", + }, + key="DAI_LIST_CACHE_VERSION:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + app_cache_client=FakeRedisClient(), + dai_hash_resolver=_fake_dai_hash_resolver, + ) + finally: + app_settings_cache.app_cache_is_using_redis = previous_runtime_flag + + serialized_payload = json.dumps({"page": page, "preview": preview}) + assert page["success"] is True + assert len(page["keys"]) == 1 + assert page["keys"][0]["resolution"]["resolved"] is True + assert page["keys"][0]["resolution"]["scope_key"] == "user:test-user" + assert preview["success"] is True + assert preview["resolution"]["entity_type"] == "user" + assert preview["resolution"]["row_count"] == 2 + assert "do-not-expose" not in serialized_payload + assert "example.redis.cache.windows.net" not in serialized_payload + + +def test_redis_explorer_value_sanitizes_json_preview(): + """Validate Redis Explorer redacts sensitive JSON fields in previews.""" + previous_runtime_flag = app_settings_cache.app_cache_is_using_redis + try: + app_settings_cache.app_cache_is_using_redis = True + preview = get_redis_explorer_value( + { + "enable_redis_cache": True, + "redis_url": "example.redis.cache.windows.net", + "redis_key": "do-not-expose", + }, + key="simplechat:conversation_cache:user-1", + app_cache_client=FakeRedisClient(), + ) + finally: + app_settings_cache.app_cache_is_using_redis = previous_runtime_flag + + serialized_preview = json.dumps(preview) + assert preview["success"] is True + assert preview["type"] == "string" + assert preview["preview_format"] == "json" + assert preview["redacted"] is True + assert "Visible cache title" in preview["preview"] + assert "safe-value" in preview["preview"] + assert "secret-api-key" not in serialized_preview + assert "secret-token" not in serialized_preview + assert "do-not-expose" not in serialized_preview + assert "example.redis.cache.windows.net" not in serialized_preview + + +def test_redis_explorer_restricts_session_key_preview(): + """Validate sensitive key names return restricted previews.""" + previous_runtime_flag = app_settings_cache.app_cache_is_using_redis + try: + app_settings_cache.app_cache_is_using_redis = True + preview = get_redis_explorer_value( + { + "enable_redis_cache": True, + "redis_url": "example.redis.cache.windows.net", + }, + key="flask_session:abc123", + app_cache_client=FakeRedisClient(), + ) + finally: + app_settings_cache.app_cache_is_using_redis = previous_runtime_flag + + serialized_preview = json.dumps(preview) + assert preview["success"] is True + assert preview["preview_restricted"] is True + assert preview["redacted"] is True + assert "Preview restricted" in preview["preview"] + assert "session-cookie-secret" not in serialized_preview + + +if __name__ == "__main__": + tests = [ + test_redis_monitoring_healthy_metrics, + test_redis_monitoring_reports_runtime_unavailable, + test_redis_monitoring_sanitizes_info_failures, + test_redis_explorer_lists_keys_with_safe_metadata, + test_redis_explorer_resolves_dai_version_marker_metadata, + test_redis_explorer_value_sanitizes_json_preview, + test_redis_explorer_restricts_session_key_preview, + ] + results = [] + for test in tests: + print(f"\nRunning {test.__name__}...") + try: + test() + print("PASS") + results.append(True) + except Exception as exc: + print(f"FAIL: {exc}") + import traceback + traceback.print_exc() + results.append(False) + + passed = sum(results) + print(f"\nResults: {passed}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_cosmos_wave5a_document_access_read_switch.py b/functional_tests/test_cosmos_wave5a_document_access_read_switch.py new file mode 100644 index 00000000..ae780431 --- /dev/null +++ b/functional_tests/test_cosmos_wave5a_document_access_read_switch.py @@ -0,0 +1,891 @@ +# test_cosmos_wave5a_document_access_read_switch.py +#!/usr/bin/env python3 +""" +Functional test for Cosmos Wave 5A/5B document access index read path. +Version: 0.250.047 +Implemented in: 0.250.022 +Public workspace UI coverage updated in: 0.250.023 +Tag listing coverage updated in: 0.250.024 +Production read metrics updated in: 0.250.025 +Default read enablement updated in: 0.250.027 +Redis DAI cache updated in: 0.250.029 +Legacy tag family projection updated in: 0.250.030 + +This test ensures the default DAI read path only serves document list reads +when backfill is complete and repair backlog is clear. It also verifies +projected rows are shaped like source documents so existing list and tag +endpoints can safely fall back to source containers. +""" + +import copy +import importlib +import os +import sys +import types +from contextlib import contextmanager + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SINGLE_APP_DIR = os.path.join(ROOT_DIR, "application", "single_app") +if SINGLE_APP_DIR not in sys.path: + sys.path.insert(0, SINGLE_APP_DIR) + + +class FakeCosmosError(Exception): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class FakeCosmosContainer: + def __init__(self, fail_query=False): + self.items = {} + self.fail_query = fail_query + self.queries = [] + + def _partition_key_for_body(self, body): + return body.get("scope_key") or body.get("id") + + def upsert_item(self, body): + partition_key = self._partition_key_for_body(body) + self.items[(partition_key, body["id"])] = copy.deepcopy(body) + return copy.deepcopy(body) + + def read_item(self, item, partition_key): + key = (partition_key, item) + if key not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + return copy.deepcopy(self.items[key]) + + def query_items(self, query, parameters=None, partition_key=None, **kwargs): + if self.fail_query: + raise RuntimeError("container query unavailable") + + self.queries.append({ + "query": query, + "partition_key": partition_key, + "parameters": copy.deepcopy(parameters or []), + }) + parameter_map = { + parameter["name"]: parameter["value"] + for parameter in list(parameters or []) + } + item_type = parameter_map.get("@type") + source_scope = parameter_map.get("@source_scope") + document_id = parameter_map.get("@document_id") + projection_version = parameter_map.get("@projection_version") + status = parameter_map.get("@status") + access_role = parameter_map.get("@access_role") + scope_key = parameter_map.get("@scope_key") or partition_key + results = [] + + for (item_partition_key, _item_id), item in self.items.items(): + if scope_key is not None and item_partition_key != scope_key: + continue + if item_type is not None and item.get("type") != item_type: + continue + if source_scope is not None and item.get("source_scope") != source_scope: + continue + if document_id is not None and item.get("source_document_id") != document_id: + continue + if projection_version is not None and item.get("projection_version") != projection_version: + continue + if status is not None and item.get("status") != status: + continue + if access_role is not None and item.get("access_role") != access_role: + continue + if ( + "AND c.access_granted = true" in query + and "OR c.approval_status = @approval_not_approved" not in query + and item.get("access_granted") is not True + ): + continue + if ( + "OR c.approval_status = @approval_not_approved" in query + and item.get("access_granted") is not True + and item.get("approval_status") != parameter_map.get("@approval_not_approved") + ): + continue + if "IS_NULL(c.percentage_complete)" in query and item.get("percentage_complete") is not None: + continue + results.append(copy.deepcopy(item)) + if "SELECT VALUE COUNT" in query.upper(): + return [len(results)] + + return results + + +class FakeRepairBacklogStateReadFailContainer(FakeCosmosContainer): + def read_item(self, item, partition_key): + if item == "document_access_index_repair_backlog_state": + raise RuntimeError("repair backlog state unavailable") + return super().read_item(item, partition_key) + + +def _settings(reads_enabled=True, write_through_enabled=True): + return { + "enable_document_access_index_container": True, + "enable_document_access_index_write_through": write_through_enabled, + "enable_document_access_index_reads": reads_enabled, + "enable_document_access_index_shadow_validation": False, + "enable_startup_document_access_index_backfill": False, + } + + +def _succeeded_backfill_state(): + return { + "id": "document_access_index_backfill_state", + "type": "document_access_index_backfill_state", + "status": "succeeded", + "source_scopes": ["personal", "group", "public"], + "completed_source_scopes": ["personal", "group", "public"], + "total_documents_processed": 3, + "total_documents_failed": 0, + "schema_version": 2, + } + + +def _succeeded_with_errors_backfill_state(): + state = _succeeded_backfill_state() + state["status"] = "succeeded_with_errors" + state["total_documents_failed"] = 1 + return state + + +def _document(document_id, owner_id, **overrides): + document = { + "id": document_id, + "type": "document_metadata", + "user_id": owner_id, + "file_name": f"{document_id}.pdf", + "title": document_id.title(), + "version": 1, + "revision_family_id": document_id, + "is_current_version": True, + "search_visibility_state": "active", + "status": "Processing complete", + "percentage_complete": 100, + "authors": ["Ada Lovelace"], + "keywords": ["Cosmos"], + "abstract": "Wave 5A projection read test", + "tags": ["wave5a"], + "shared_user_ids": [], + "_ts": 12345, + } + document.update(overrides) + return document + + +@contextmanager +def _load_document_access_index_module(index_container=None, settings_container=None, settings=None): + original_modules = {} + for module_name in [ + "config", + "functions_appinsights", + "functions_settings", + "functions_document_access_index", + ]: + if module_name in sys.modules: + original_modules[module_name] = sys.modules[module_name] + del sys.modules[module_name] + + index_container = index_container or FakeCosmosContainer() + settings_container = settings_container or FakeCosmosContainer() + fake_config = types.ModuleType("config") + fake_config.cosmos_document_access_index_container = index_container + fake_config.cosmos_settings_container = settings_container + fake_config.cosmos_user_documents_container = FakeCosmosContainer() + fake_config.cosmos_user_documents_container_name = "documents" + fake_config.cosmos_group_documents_container = FakeCosmosContainer() + fake_config.cosmos_group_documents_container_name = "group_documents" + fake_config.cosmos_public_documents_container = FakeCosmosContainer() + fake_config.cosmos_public_documents_container_name = "public_documents" + fake_config.cosmos_groups_container = FakeCosmosContainer() + fake_config.cosmos_public_workspaces_container = FakeCosmosContainer() + fake_config.cosmos_user_settings_container = FakeCosmosContainer() + sys.modules["config"] = fake_config + + fake_appinsights = types.ModuleType("functions_appinsights") + fake_appinsights.log_event = lambda *args, **kwargs: None + sys.modules["functions_appinsights"] = fake_appinsights + + fake_settings = types.ModuleType("functions_settings") + fake_settings.get_settings = lambda: settings or _settings() + sys.modules["functions_settings"] = fake_settings + + try: + yield importlib.import_module("functions_document_access_index"), index_container, settings_container + finally: + for module_name in [ + "config", + "functions_appinsights", + "functions_settings", + "functions_document_access_index", + ]: + sys.modules.pop(module_name, None) + sys.modules.update(original_modules) + + +def test_read_switch_requires_completed_backfill_and_clear_repairs(): + """Reads should remain source-backed until DAI is explicitly ready.""" + with _load_document_access_index_module() as (indexing, _index_container, _settings_container): + legacy_disabled_setting_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=_settings(reads_enabled=False), + ) + not_ready_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=_settings(reads_enabled=True), + ) + legacy_write_through_false_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=_settings(reads_enabled=True, write_through_enabled=False), + ) + + assert legacy_disabled_setting_result["success"] is False + assert legacy_disabled_setting_result["status"] == "backfill_not_ready" + assert legacy_disabled_setting_result["readiness"]["settings"]["reads_enabled"] is True + assert not_ready_result["success"] is False + assert not_ready_result["status"] == "backfill_not_ready" + assert legacy_write_through_false_result["success"] is False + assert legacy_write_through_false_result["status"] == "backfill_not_ready" + assert legacy_write_through_false_result["readiness"]["settings"]["write_through_enabled"] is True + + +def test_read_switch_falls_back_when_repair_backlog_state_is_unknown(): + """A state read failure should block DAI reads and leave source fallback available.""" + settings_container = FakeRepairBacklogStateReadFailContainer() + with _load_document_access_index_module(settings_container=settings_container) as ( + indexing, + _index_container, + settings_container, + ): + settings_container.upsert_item(_succeeded_backfill_state()) + result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=_settings(reads_enabled=True), + ) + + assert result["success"] is False + assert result["status"] == "repair_backlog_unknown" + assert result["readiness"]["backfill_status"] == "succeeded" + + +def test_read_switch_accepts_succeeded_with_errors_when_repairs_are_clear(): + """Resolved historical backfill errors should not block DAI reads forever.""" + with _load_document_access_index_module() as (indexing, index_container, settings_container): + settings_container.upsert_item(_succeeded_with_errors_backfill_state()) + indexing.sync_document_access_index_for_document( + _document("doc-owner", "owner-1"), + force=True, + ) + + result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=_settings(reads_enabled=True), + ) + + assert result["success"] is True + assert result["status"] == "served_from_index" + assert result["readiness"]["backfill_status"] == "succeeded_with_errors" + assert len(result["documents"]) == 1 + assert ("user:owner-1", "dai:user:owner-1:personal:doc-owner:1") in index_container.items + + +def test_read_switch_returns_list_ready_projection_documents(): + """DAI reads should be single-partition and shaped like source list documents.""" + with _load_document_access_index_module() as (indexing, index_container, settings_container): + settings_container.upsert_item(_succeeded_backfill_state()) + indexing.sync_document_access_index_for_document( + _document("doc-owner", "owner-1"), + force=True, + ) + indexing.sync_document_access_index_for_document( + _document( + "doc-shared", + "owner-2", + title="Shared Cosmos Notes", + shared_user_ids=["reader-1,approved", "auditor-1,approved"], + number_of_pages=12, + publication_date="2026-07-02", + enhanced_citations=True, + blob_path="documents/doc-shared.pdf", + document_intelligence_extraction_mode="layout", + generated_artifact_promotion_status="pending_approval", + file_sync={"enabled": True, "scope": "personal"}, + created_from_chat_upload=True, + conversation_id="conversation-123", + conversation_title_at_upload="Cosmos tuning chat", + ), + force=True, + ) + + result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="reader-1", + filters={ + "search": "cosmos", + "author": "ada", + "keywords": "cosmos", + "abstract": "projection", + "tags": ["wave5a"], + "array_match_mode": "contains", + }, + ) + read_metrics = indexing.get_document_access_index_read_metrics() + repair_existence_queries = [ + query + for query in settings_container.queries + if "SELECT TOP 1 VALUE c.id" in query["query"] + ] + + assert result["success"] is True + assert result["status"] == "served_from_index" + assert repair_existence_queries + assert result["scope_keys"] == ["user:reader-1"] + assert len(result["documents"]) == 1 + document = result["documents"][0] + assert document["id"] == "doc-shared" + assert document["user_id"] == "owner-2" + assert document["shared_user_ids"] == ["reader-1,approved", "auditor-1,approved"] + assert document["_ts"] == 12345 + assert document["number_of_pages"] == 12 + assert document["publication_date"] == "2026-07-02" + assert document["enhanced_citations"] is True + assert document["document_intelligence_extraction_mode"] == "layout" + assert document["generated_artifact_promotion_status"] == "pending_approval" + assert document["file_sync"] == {"enabled": True, "scope": "personal"} + assert document["created_from_chat_upload"] is True + assert document["conversation_id"] == "conversation-123" + assert document["conversation_title_at_upload"] == "Cosmos tuning chat" + read_queries = [ + query for query in index_container.queries + if query["partition_key"] == "user:reader-1" + ] + assert len(read_queries) == 1 + assert all("ORDER BY" not in query["query"] for query in read_queries) + assert all("OFFSET" not in query["query"] for query in read_queries) + assert all("LIMIT" not in query["query"] for query in read_queries) + assert all("TOP" not in query["query"] for query in read_queries) + assert read_metrics["windows"]["15m"]["sample_count"] == 1 + assert read_metrics["windows"]["15m"]["served_from_index_count"] == 1 + assert read_metrics["windows"]["15m"]["source_fallback_count"] == 0 + assert read_metrics["windows"]["15m"]["item_count"] == 1 + + +def test_read_switch_returns_multi_public_workspace_documents(): + """Public chat document lists should read one DAI partition per visible public workspace.""" + with _load_document_access_index_module() as (indexing, index_container, settings_container): + settings_container.upsert_item(_succeeded_backfill_state()) + indexing.sync_document_access_index_for_document( + _document( + "public-doc-a", + "owner-public-a", + public_workspace_id="public-a", + title="Public Cosmos Notes A", + ), + force=True, + ) + indexing.sync_document_access_index_for_document( + _document( + "public-doc-b", + "owner-public-b", + public_workspace_id="public-b", + title="Public Cosmos Notes B", + ), + force=True, + ) + + result = indexing.query_document_access_index_documents( + source_scope="public", + public_workspace_ids=["public-a", "public-b"], + ) + + assert result["success"] is True + assert result["status"] == "served_from_index" + assert result["scope_keys"] == ["public:public-a", "public:public-b"] + assert {document["id"] for document in result["documents"]} == {"public-doc-a", "public-doc-b"} + read_partitions = { + query["partition_key"] + for query in index_container.queries + if query["partition_key"] in {"public:public-a", "public:public-b"} + } + assert read_partitions == {"public:public-a", "public:public-b"} + + +def test_read_switch_includes_pending_projection_rows_for_approval_ui(): + """Document-list reads should include pending shares so users can approve them.""" + with _load_document_access_index_module() as (indexing, index_container, settings_container): + settings_container.upsert_item(_succeeded_backfill_state()) + indexing.sync_document_access_index_for_document( + _document( + "personal-pending-share", + "owner-1", + shared_user_ids=["reader-1,not_approved"], + ), + force=True, + ) + indexing.sync_document_access_index_for_document( + _document( + "personal-approved-share", + "owner-2", + shared_user_ids=["reader-1,approved"], + ), + force=True, + ) + + result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="reader-1", + ) + + assert result["success"] is True + assert result["status"] == "served_from_index" + assert {document["id"] for document in result["documents"]} == { + "personal-pending-share", + "personal-approved-share", + } + pending_document = next( + document for document in result["documents"] + if document["id"] == "personal-pending-share" + ) + assert pending_document["shared_user_ids"] == ["reader-1,not_approved"] + candidate_queries = [ + query["query"] + for query in index_container.queries + if "c.conversation_title_at_upload" in query["query"] + ] + assert candidate_queries + assert all("c.approval_status = @approval_not_approved" in query for query in candidate_queries) + + +def test_read_switch_exact_array_filters_are_case_sensitive(): + """Exact author/keyword filters should preserve source route case-sensitive semantics.""" + with _load_document_access_index_module() as (indexing, _index_container, settings_container): + settings_container.upsert_item(_succeeded_backfill_state()) + indexing.sync_document_access_index_for_document( + _document( + "public-case-sensitive", + "owner-public", + public_workspace_id="public-a", + authors=["Ada Lovelace"], + ), + force=True, + ) + + exact_match = indexing.query_document_access_index_documents( + source_scope="public", + public_workspace_id="public-a", + filters={ + "author": "Ada Lovelace", + "array_match_mode": "exact", + }, + ) + lower_case_mismatch = indexing.query_document_access_index_documents( + source_scope="public", + public_workspace_id="public-a", + filters={ + "author": "ada lovelace", + "array_match_mode": "exact", + }, + ) + + assert [document["id"] for document in exact_match["documents"]] == ["public-case-sensitive"] + assert lower_case_mismatch["documents"] == [] + + +def test_read_switch_classification_filters_are_case_sensitive(): + """Non-none classification filters should preserve source route exact casing.""" + with _load_document_access_index_module() as (indexing, _index_container, settings_container): + settings_container.upsert_item(_succeeded_backfill_state()) + indexing.sync_document_access_index_for_document( + _document( + "public-classified", + "owner-public", + public_workspace_id="public-a", + document_classification="Confidential", + ), + force=True, + ) + + exact_match = indexing.query_document_access_index_documents( + source_scope="public", + public_workspace_id="public-a", + filters={"classification": "Confidential"}, + ) + lower_case_mismatch = indexing.query_document_access_index_documents( + source_scope="public", + public_workspace_id="public-a", + filters={"classification": "confidential"}, + ) + + assert [document["id"] for document in exact_match["documents"]] == ["public-classified"] + assert lower_case_mismatch["documents"] == [] + + +def test_read_switch_public_projection_includes_generated_artifact_requester(): + """Public DAI rows should include requester identity used by generated-artifact actions.""" + with _load_document_access_index_module() as (indexing, _index_container, settings_container): + settings_container.upsert_item(_succeeded_backfill_state()) + indexing.sync_document_access_index_for_document( + _document( + "public-artifact", + "owner-public", + public_workspace_id="public-a", + generated_artifact_promotion_status="pending_approval", + generated_artifact_requested_by_user_id="requester-1", + ), + force=True, + ) + + result = indexing.query_document_access_index_documents( + source_scope="public", + public_workspace_id="public-a", + ) + + document = result["documents"][0] + assert document["id"] == "public-artifact" + assert document["generated_artifact_requested_by_user_id"] == "requester-1" + assert document["user_id"] == "owner-public" + + +def test_read_switch_normalizes_enhanced_citations_from_blob_state(): + """DAI list rows should mirror source enhanced_citations normalization from blob references.""" + with _load_document_access_index_module() as (indexing, _index_container, settings_container): + settings_container.upsert_item(_succeeded_backfill_state()) + indexing.sync_document_access_index_for_document( + _document( + "blob-backed", + "owner-1", + enhanced_citations=False, + blob_path="documents/blob-backed.pdf", + ), + force=True, + ) + indexing.sync_document_access_index_for_document( + _document( + "metadata-only", + "owner-1", + enhanced_citations=True, + blob_path=None, + ), + force=True, + ) + + result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + ) + + documents_by_id = {document["id"]: document for document in result["documents"]} + assert documents_by_id["blob-backed"]["enhanced_citations"] is True + assert documents_by_id["metadata-only"]["enhanced_citations"] is False + + +def test_read_switch_collapses_legacy_revisions_by_scope_and_file_name(): + """Legacy rows without revision metadata should collapse like source select_current_documents().""" + older_document = _document("legacy-v1", "owner-1", file_name="legacy.pdf", version=1, _ts=100) + newer_document = _document("legacy-v2", "owner-1", file_name="legacy.pdf", version=2, _ts=200) + older_document.pop("revision_family_id", None) + newer_document.pop("revision_family_id", None) + older_document.pop("is_current_version", None) + newer_document.pop("is_current_version", None) + + with _load_document_access_index_module() as (indexing, _index_container, settings_container): + settings_container.upsert_item(_succeeded_backfill_state()) + indexing.sync_document_access_index_for_document(older_document, force=True) + indexing.sync_document_access_index_for_document(newer_document, force=True) + + result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + ) + + assert [document["id"] for document in result["documents"]] == ["legacy-v2"] + assert result["documents"][0]["version"] == 2 + + +def test_tag_count_read_switch_uses_owner_projection_rows_by_scope(): + """Tag list reads should count current owner-scope projection rows without source scans.""" + with _load_document_access_index_module() as (indexing, index_container, settings_container): + settings_container.upsert_item(_succeeded_backfill_state()) + indexing.sync_document_access_index_for_document( + _document("personal-owner", "owner-1", tags=["wave5a", "cosmos"]), + force=True, + ) + older_legacy_tag = _document("personal-legacy-tag-v1", "owner-1", file_name="legacy-tag.pdf", version=1, tags=["legacytag"]) + newer_legacy_tag = _document("personal-legacy-tag-v2", "owner-1", file_name="legacy-tag.pdf", version=2, tags=["legacytag"]) + distinct_legacy_tag = _document("personal-legacy-tag-v3", "owner-1", file_name="legacy-tag-2.pdf", version=1, tags=["legacytag"]) + older_legacy_tag.pop("revision_family_id", None) + newer_legacy_tag.pop("revision_family_id", None) + distinct_legacy_tag.pop("revision_family_id", None) + older_legacy_tag.pop("is_current_version", None) + newer_legacy_tag.pop("is_current_version", None) + distinct_legacy_tag.pop("is_current_version", None) + indexing.sync_document_access_index_for_document(older_legacy_tag, force=True) + indexing.sync_document_access_index_for_document(newer_legacy_tag, force=True) + indexing.sync_document_access_index_for_document(distinct_legacy_tag, force=True) + indexing.sync_document_access_index_for_document( + _document( + "personal-shared", + "owner-2", + tags=["shared-only"], + shared_user_ids=["owner-1,approved"], + ), + force=True, + ) + indexing.sync_document_access_index_for_document( + _document( + "group-owned", + "owner-1", + group_id="group-a", + tags=["group-alpha"], + shared_group_ids=["group-b,approved"], + ), + force=True, + ) + indexing.sync_document_access_index_for_document( + _document( + "public-owned", + "owner-public", + public_workspace_id="public-a", + tags=["public-alpha"], + ), + force=True, + ) + + personal_result = indexing.query_document_access_index_tag_counts( + source_scope="personal", + user_id="owner-1", + ) + group_result = indexing.query_document_access_index_tag_counts( + source_scope="group", + group_ids=["group-a", "group-b"], + ) + public_result = indexing.query_document_access_index_tag_counts( + source_scope="public", + public_workspace_ids=["public-a"], + ) + + assert personal_result["success"] is True + assert personal_result["status"] == "served_from_index" + assert personal_result["tag_counts"] == {"cosmos": 1, "legacytag": 2, "wave5a": 1} + assert "shared-only" not in personal_result["tag_counts"] + assert group_result["success"] is True + assert group_result["tag_counts_by_scope_key"]["group:group-a"] == {"group-alpha": 1} + assert group_result["tag_counts_by_scope_key"]["group:group-b"] == {} + assert public_result["success"] is True + assert public_result["tag_counts_by_scope_key"]["public:public-a"] == {"public-alpha": 1} + tag_read_queries = [ + query for query in index_container.queries + if "tag_read:" in query["query"] or "ARRAY_LENGTH(c.tags)" in query["query"] + ] + assert tag_read_queries + assert all(query["partition_key"] for query in tag_read_queries) + assert all("ORDER BY" not in query["query"] for query in tag_read_queries) + assert all("OFFSET" not in query["query"] for query in tag_read_queries) + assert all("LIMIT" not in query["query"] for query in tag_read_queries) + assert all("TOP" not in query["query"] for query in tag_read_queries) + assert all("c.file_name" in query["query"] for query in tag_read_queries) + + +def test_legacy_count_read_switch_uses_unfiltered_owner_projection_rows(): + """Legacy prompts should use unfiltered owner-scope DAI rows, not current list filters.""" + with _load_document_access_index_module() as (indexing, _index_container, settings_container): + settings_container.upsert_item(_succeeded_backfill_state()) + indexing.sync_document_access_index_for_document( + _document("personal-owner-legacy", "owner-1", percentage_complete=None), + force=True, + ) + indexing.sync_document_access_index_for_document( + _document( + "personal-old-revision-legacy", + "owner-1", + is_current_version=False, + percentage_complete=None, + ), + force=True, + ) + indexing.sync_document_access_index_for_document( + _document( + "personal-shared-legacy", + "owner-2", + percentage_complete=None, + shared_user_ids=["owner-1,approved"], + ), + force=True, + ) + indexing.sync_document_access_index_for_document( + _document("group-owner-legacy", "owner-1", group_id="group-a", percentage_complete=None), + force=True, + ) + indexing.sync_document_access_index_for_document( + _document( + "group-shared-legacy", + "owner-1", + group_id="group-b", + percentage_complete=None, + shared_group_ids=["group-a,approved"], + ), + force=True, + ) + indexing.sync_document_access_index_for_document( + _document("public-owner-legacy", "owner-public", public_workspace_id="public-a", percentage_complete=None), + force=True, + ) + + personal_result = indexing.query_document_access_index_legacy_count( + source_scope="personal", + user_id="owner-1", + ) + group_result = indexing.query_document_access_index_legacy_count( + source_scope="group", + group_ids=["group-a"], + ) + public_result = indexing.query_document_access_index_legacy_count( + source_scope="public", + public_workspace_id="public-a", + ) + + assert personal_result["success"] is True + assert personal_result["legacy_count"] == 2 + assert group_result["success"] is True + assert group_result["legacy_count"] == 1 + assert public_result["success"] is True + assert public_result["legacy_count"] == 1 + + +def test_read_switch_query_failure_reports_source_fallback_status(): + """DAI query failures should return a safe fallback status instead of raising.""" + settings_container = FakeCosmosContainer() + settings_container.upsert_item(_succeeded_backfill_state()) + with _load_document_access_index_module( + index_container=FakeCosmosContainer(fail_query=True), + settings_container=settings_container, + ) as (indexing, _index_container, _settings_container): + result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + ) + read_metrics = indexing.get_document_access_index_read_metrics() + + assert result["success"] is False + assert result["status"] == "query_failed" + assert result["documents"] == [] + assert read_metrics["windows"]["15m"]["sample_count"] == 1 + assert read_metrics["windows"]["15m"]["source_fallback_count"] == 1 + assert read_metrics["last_fallback_sample"]["status"] == "query_failed" + + +def test_wave5b_route_and_admin_contract_are_wired(): + """Routes and Admin Settings should expose the Wave 5B default read path.""" + config_source = open(os.path.join(SINGLE_APP_DIR, "config.py"), "r", encoding="utf-8").read() + index_source = open( + os.path.join(SINGLE_APP_DIR, "functions_document_access_index.py"), + "r", + encoding="utf-8", + ).read() + personal_route = open(os.path.join(SINGLE_APP_DIR, "route_backend_documents.py"), "r", encoding="utf-8").read() + group_route = open(os.path.join(SINGLE_APP_DIR, "route_backend_group_documents.py"), "r", encoding="utf-8").read() + public_route = open(os.path.join(SINGLE_APP_DIR, "route_external_public_documents.py"), "r", encoding="utf-8").read() + backend_public_route = open( + os.path.join(SINGLE_APP_DIR, "route_backend_public_documents.py"), + "r", + encoding="utf-8", + ).read() + admin_template = open( + os.path.join(SINGLE_APP_DIR, "templates", "admin_settings.html"), + "r", + encoding="utf-8", + ).read() + admin_route = open( + os.path.join(SINGLE_APP_DIR, "route_frontend_admin_settings.py"), + "r", + encoding="utf-8", + ).read() + settings_source = open(os.path.join(SINGLE_APP_DIR, "functions_settings.py"), "r", encoding="utf-8").read() + + assert 'VERSION = "0.250.047"' in config_source + assert "DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION = 2" in index_source + assert "def query_document_access_index_documents(" in index_source + assert "def query_document_access_index_tag_counts(" in index_source + assert "def query_document_access_index_legacy_count(" in index_source + assert "def get_document_access_index_read_metrics(" in index_source + assert "def get_document_access_index_cache_metrics(" in index_source + assert "def invalidate_document_access_index_cache_scope_keys(" in index_source + assert "_record_document_access_read_metric(" in index_source + assert "'served_from_index'" in index_source + assert "'backfill_not_ready'" in index_source + assert "partition_key=scope_key" in index_source + assert "c.approval_status = @approval_not_approved" in index_source + assert "c.projection_version = @projection_version" in index_source + assert "_safe_int(state.get('schema_version')) != DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION" in index_source + assert "number_of_pages" in index_source + assert "generated_artifact_promotion_status" in index_source + assert "conversation_title_at_upload" in index_source + assert "query_document_access_index_documents(" in personal_route + assert "query_document_access_index_documents(" in group_route + assert "query_document_access_index_documents(" in public_route + assert "query_document_access_index_documents(" in backend_public_route + assert "query_document_access_index_tag_counts(" in personal_route + assert "query_document_access_index_tag_counts(" in group_route + assert "query_document_access_index_tag_counts(" in backend_public_route + assert "query_document_access_index_legacy_count(" in personal_route + assert "query_document_access_index_legacy_count(" in group_route + assert "query_document_access_index_legacy_count(" in public_route + assert "query_document_access_index_legacy_count(" in backend_public_route + assert "build_workspace_tags_from_counts(" in personal_route + assert "build_workspace_tags_from_counts(" in group_route + assert "build_workspace_tags_from_counts(" in backend_public_route + assert "if index_read_result.get('success')" in personal_route + assert "if index_read_result.get('success')" in group_route + assert "if index_read_result.get('success')" in public_route + assert "if index_read_result.get('success')" in backend_public_route + assert "legacy_count = 0\n if used_document_access_index:" in personal_route + assert "public_workspace_ids=workspace_ids" in backend_public_route + assert "context='api_list_public_documents'" in backend_public_route + assert "context='api_list_public_workspace_documents'" in backend_public_route + assert "'enable_document_access_index_reads': True" in settings_source + assert "normalize_document_access_index_required_settings(merged)" in settings_source + assert 'name="enable_document_access_index_reads"' in admin_template + assert "Wave 5B default" in admin_template + assert "Wave 6" in admin_template + assert "Redis document list cache" in admin_template + assert "'enable_document_access_index_reads': True" in admin_route + + +if __name__ == "__main__": + tests = [ + test_read_switch_requires_completed_backfill_and_clear_repairs, + test_read_switch_falls_back_when_repair_backlog_state_is_unknown, + test_read_switch_accepts_succeeded_with_errors_when_repairs_are_clear, + test_read_switch_returns_list_ready_projection_documents, + test_read_switch_returns_multi_public_workspace_documents, + test_read_switch_includes_pending_projection_rows_for_approval_ui, + test_read_switch_exact_array_filters_are_case_sensitive, + test_read_switch_classification_filters_are_case_sensitive, + test_read_switch_public_projection_includes_generated_artifact_requester, + test_read_switch_normalizes_enhanced_citations_from_blob_state, + test_read_switch_collapses_legacy_revisions_by_scope_and_file_name, + test_tag_count_read_switch_uses_owner_projection_rows_by_scope, + test_legacy_count_read_switch_uses_unfiltered_owner_projection_rows, + test_read_switch_query_failure_reports_source_fallback_status, + test_wave5b_route_and_admin_contract_are_wired, + ] + results = [] + for test in tests: + print(f"Running {test.__name__}...") + try: + test() + print("Test passed.") + results.append(True) + except Exception as exc: + print(f"Test failed: {exc}") + results.append(False) + + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_cosmos_wave6_document_access_cache.py b/functional_tests/test_cosmos_wave6_document_access_cache.py new file mode 100644 index 00000000..86c8ea07 --- /dev/null +++ b/functional_tests/test_cosmos_wave6_document_access_cache.py @@ -0,0 +1,785 @@ +# test_cosmos_wave6_document_access_cache.py +#!/usr/bin/env python3 +""" +Functional test for Cosmos Wave 6 document access Redis cache. +Version: 0.250.043 +Implemented in: 0.250.029 +DAI cache TTL default updated in: 0.250.031 +DAI version marker TTL hygiene added in: 0.250.043 + +This test ensures DAI document-list reads use Redis read-through caching with +scope-version invalidation, bounded TTLs, and safe fallback to direct DAI reads +when Redis cache operations fail. +""" + +import copy +import fnmatch +import importlib +import os +import sys +import types +from contextlib import contextmanager + + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SINGLE_APP_DIR = os.path.join(ROOT_DIR, "application", "single_app") +if SINGLE_APP_DIR not in sys.path: + sys.path.insert(0, SINGLE_APP_DIR) + + +class FakeCosmosError(Exception): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class FakeCosmosContainer: + def __init__(self): + self.items = {} + self.queries = [] + + def _partition_key_for_body(self, body): + return body.get("scope_key") or body.get("id") + + def upsert_item(self, body): + partition_key = self._partition_key_for_body(body) + self.items[(partition_key, body["id"])] = copy.deepcopy(body) + return copy.deepcopy(body) + + def read_item(self, item, partition_key): + key = (partition_key, item) + if key not in self.items: + raise FakeCosmosError(404, f"Missing item {item}") + return copy.deepcopy(self.items[key]) + + def delete_item(self, item, partition_key): + self.items.pop((partition_key, item), None) + + def query_items(self, query, parameters=None, partition_key=None, **kwargs): + self.queries.append({ + "query": query, + "partition_key": partition_key, + "parameters": copy.deepcopy(parameters or []), + }) + parameter_map = { + parameter["name"]: parameter["value"] + for parameter in list(parameters or []) + } + item_type = parameter_map.get("@type") + source_scope = parameter_map.get("@source_scope") + document_id = parameter_map.get("@document_id") + projection_version = parameter_map.get("@projection_version") + status = parameter_map.get("@status") + access_role = parameter_map.get("@access_role") + scope_key = parameter_map.get("@scope_key") or partition_key + results = [] + + for (item_partition_key, _item_id), item in self.items.items(): + if scope_key is not None and item_partition_key != scope_key: + continue + if item_type is not None and item.get("type") != item_type: + continue + if source_scope is not None and item.get("source_scope") != source_scope: + continue + if document_id is not None and item.get("source_document_id") != document_id: + continue + if projection_version is not None and item.get("projection_version") != projection_version: + continue + if status is not None and item.get("status") != status: + continue + if access_role is not None and item.get("access_role") != access_role: + continue + if ( + "AND c.access_granted = true" in query + and "OR c.approval_status = @approval_not_approved" not in query + and item.get("access_granted") is not True + ): + continue + if ( + "OR c.approval_status = @approval_not_approved" in query + and item.get("access_granted") is not True + and item.get("approval_status") != parameter_map.get("@approval_not_approved") + ): + continue + if "ARRAY_LENGTH(c.tags) > 0" in query and not item.get("tags"): + continue + if "IS_NULL(c.percentage_complete)" in query and item.get("percentage_complete") is not None: + continue + results.append(copy.deepcopy(item)) + + query_upper = query.upper() + if "SELECT VALUE COUNT" in query_upper: + return [len(results)] + if "SELECT TOP 1 VALUE C.ID" in query_upper: + return [item["id"] for item in results[:1]] + return results + + +class FakeRedisClient: + def __init__(self): + self.values = {} + self.ttls = {} + self.get_calls = [] + self.set_calls = [] + self.setex_calls = [] + self.expire_calls = [] + self.incr_calls = [] + + def get(self, key): + self.get_calls.append(key) + return self.values.get(key) + + def setnx(self, key, value): + if key not in self.values: + self.values[key] = str(value) + return True + return False + + def set(self, key, value, ex=None, nx=False): + if nx and key in self.values: + return False + self.values[key] = str(value) + self.set_calls.append((key, value, ex, nx)) + if ex is not None: + self.ttls[key] = ex + return True + + def expire(self, key, ttl_seconds): + if key not in self.values: + return False + self.ttls[key] = ttl_seconds + self.expire_calls.append((key, ttl_seconds)) + return True + + def ttl(self, key): + if key not in self.values: + return -2 + return self.ttls.get(key, -1) + + def scan(self, cursor=0, match="*", count=100): + keys = [ + key + for key in sorted(self.values) + if fnmatch.fnmatch(key, match) + ] + cursor_index = int(cursor or 0) + next_index = min(cursor_index + int(count or 100), len(keys)) + next_cursor = 0 if next_index >= len(keys) else next_index + return next_cursor, keys[cursor_index:next_index] + + def memory_usage(self, key): + return len(str(self.values.get(key, ""))) + + def incr(self, key): + next_value = int(self.values.get(key) or 0) + 1 + self.values[key] = str(next_value) + self.incr_calls.append(key) + return next_value + + def setex(self, key, ttl_seconds, value): + self.values[key] = value + self.ttls[key] = ttl_seconds + self.setex_calls.append((key, ttl_seconds, value)) + return True + + +class FailingRepairDocumentContainer(FakeCosmosContainer): + def __init__(self): + super().__init__() + self.fail_repair_upserts = False + + def upsert_item(self, body): + if self.fail_repair_upserts and body.get("type") == "document_access_index_repair": + raise RuntimeError("repair document write failed") + return super().upsert_item(body) + + +class ToggleFailUpsertContainer(FakeCosmosContainer): + def __init__(self): + super().__init__() + self.fail_upsert = False + + def upsert_item(self, body): + if self.fail_upsert: + raise RuntimeError("projection upsert failed") + return super().upsert_item(body) + + +class FailingRedisClient(FakeRedisClient): + def get(self, key): + raise RuntimeError("redis unavailable") + + +class FailingIncrRedisClient(FakeRedisClient): + def __init__(self): + super().__init__() + self.fail_incr = False + + def incr(self, key): + if self.fail_incr: + raise RuntimeError("redis incr unavailable") + return super().incr(key) + + +def _settings(): + return { + "enable_document_access_index_container": True, + "enable_document_access_index_write_through": True, + "enable_document_access_index_reads": True, + "enable_document_access_index_shadow_validation": False, + "enable_startup_document_access_index_backfill": True, + "enable_document_access_index_cache": True, + "document_access_index_cache_ttl_seconds": 900, + } + + +def _succeeded_backfill_state(): + return { + "id": "document_access_index_backfill_state", + "type": "document_access_index_backfill_state", + "status": "succeeded", + "source_scopes": ["personal", "group", "public"], + "completed_source_scopes": ["personal", "group", "public"], + "total_documents_processed": 1, + "total_documents_failed": 0, + "schema_version": 2, + } + + +def _document(document_id, owner_id, **overrides): + document = { + "id": document_id, + "type": "document_metadata", + "user_id": owner_id, + "file_name": f"{document_id}.pdf", + "title": document_id.title(), + "version": 1, + "revision_family_id": document_id, + "is_current_version": True, + "search_visibility_state": "active", + "status": "Processing complete", + "percentage_complete": 100, + "tags": ["wave6", "cosmos"], + "shared_user_ids": [], + "_ts": 12345, + } + document.update(overrides) + return document + + +@contextmanager +def _load_document_access_index_module( + redis_client, + settings=None, + settings_container=None, + index_container=None, + groups_container=None, + public_workspaces_container=None, + user_settings_container=None, +): + original_modules = {} + for module_name in [ + "app_settings_cache", + "config", + "functions_appinsights", + "functions_settings", + "functions_document_access_index", + ]: + if module_name in sys.modules: + original_modules[module_name] = sys.modules[module_name] + del sys.modules[module_name] + + index_container = index_container or FakeCosmosContainer() + settings_container = settings_container or FakeCosmosContainer() + groups_container = groups_container or FakeCosmosContainer() + public_workspaces_container = public_workspaces_container or FakeCosmosContainer() + user_settings_container = user_settings_container or FakeCosmosContainer() + + fake_app_settings_cache = types.ModuleType("app_settings_cache") + fake_app_settings_cache.get_app_cache_redis_client = lambda: redis_client + sys.modules["app_settings_cache"] = fake_app_settings_cache + + fake_config = types.ModuleType("config") + fake_config.cosmos_document_access_index_container = index_container + fake_config.cosmos_groups_container = groups_container + fake_config.cosmos_settings_container = settings_container + fake_config.cosmos_public_workspaces_container = public_workspaces_container + fake_config.cosmos_user_settings_container = user_settings_container + fake_config.cosmos_user_documents_container = FakeCosmosContainer() + fake_config.cosmos_user_documents_container_name = "documents" + fake_config.cosmos_group_documents_container = FakeCosmosContainer() + fake_config.cosmos_group_documents_container_name = "group_documents" + fake_config.cosmos_public_documents_container = FakeCosmosContainer() + fake_config.cosmos_public_documents_container_name = "public_documents" + sys.modules["config"] = fake_config + + fake_appinsights = types.ModuleType("functions_appinsights") + fake_appinsights.log_event = lambda *args, **kwargs: None + sys.modules["functions_appinsights"] = fake_appinsights + + fake_settings = types.ModuleType("functions_settings") + fake_settings.get_settings = lambda: settings or _settings() + sys.modules["functions_settings"] = fake_settings + + try: + yield importlib.import_module("functions_document_access_index"), index_container, settings_container + finally: + for module_name in [ + "app_settings_cache", + "config", + "functions_appinsights", + "functions_settings", + "functions_document_access_index", + ]: + sys.modules.pop(module_name, None) + sys.modules.update(original_modules) + + +def _prepare_ready_projection(indexing, index_container, settings_container, document=None): + settings_container.upsert_item(_succeeded_backfill_state()) + indexing.sync_document_access_index_for_document( + document or _document("doc-1", "owner-1"), + operation="test_projection_seed", + force=True, + ) + index_container.queries.clear() + + +def test_document_list_cache_hit_and_scope_version_invalidation(): + """Document list reads should hit Redis until the scope version changes.""" + redis_client = FakeRedisClient() + with _load_document_access_index_module(redis_client) as (indexing, index_container, settings_container): + _prepare_ready_projection(indexing, index_container, settings_container) + + first_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=_settings(), + ) + second_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=_settings(), + ) + indexing.invalidate_document_access_index_cache_scope_keys( + ["user:owner-1"], + reason="test_document_added", + settings=_settings(), + ) + third_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=_settings(), + ) + cache_metrics = indexing.get_document_access_index_cache_metrics() + read_metrics = indexing.get_document_access_index_read_metrics() + + assert first_result["status"] == "served_from_index" + assert second_result["status"] == "served_from_cache" + assert third_result["status"] == "served_from_index" + assert [document["id"] for document in second_result["documents"]] == ["doc-1"] + candidate_queries = [ + query for query in index_container.queries + if "SELECT c.id, c.document_id" in query["query"] + ] + assert len(candidate_queries) == 2 + assert redis_client.setex_calls + assert all(ttl_seconds == 900 for _key, ttl_seconds, _value in redis_client.setex_calls) + assert len(redis_client.incr_calls) >= 2 + version_keys = [ + key for key in redis_client.values + if key.startswith("DAI_LIST_CACHE_VERSION:") + ] + assert version_keys + assert all(redis_client.ttl(key) == 3600 for key in version_keys) + assert redis_client.expire_calls + assert cache_metrics["windows"]["15m"]["hit_count"] == 1 + assert cache_metrics["windows"]["15m"]["miss_count"] == 2 + assert cache_metrics["windows"]["15m"]["invalidation_count"] >= 1 + assert read_metrics["windows"]["15m"]["served_from_cache_count"] == 1 + + +def test_document_access_cache_version_marker_ttl_hygiene_refreshes_existing_markers(): + """Maintenance should add bounded TTLs to existing no-expiry DAI version markers.""" + redis_client = FakeRedisClient() + redis_client.values["DAI_LIST_CACHE_VERSION:no-expiry"] = "0" + redis_client.values["DAI_LIST_CACHE_VERSION:short-ttl"] = "4" + redis_client.ttls["DAI_LIST_CACHE_VERSION:short-ttl"] = 30 + + with _load_document_access_index_module(redis_client) as (indexing, _index_container, _settings_container): + result = indexing.refresh_document_access_cache_version_marker_ttls( + settings=_settings(), + batch_size=10, + ) + + assert result["success"] is True + assert result["status"] == "completed" + assert result["scanned_count"] == 2 + assert result["refreshed_count"] == 2 + assert result["no_expiry_count"] == 1 + assert result["unsafe_ttl_count"] == 1 + assert redis_client.ttl("DAI_LIST_CACHE_VERSION:no-expiry") == 3600 + assert redis_client.ttl("DAI_LIST_CACHE_VERSION:short-ttl") == 3600 + + +def test_document_access_cache_version_hash_resolution_returns_safe_scope_metadata(): + """DAI version marker hashes should resolve to known SimpleChat scopes when possible.""" + redis_client = FakeRedisClient() + groups_container = FakeCosmosContainer() + groups_container.upsert_item({ + "id": "group-1", + "name": "Incident Workspace", + "status": "active", + }) + group_document = _document( + "doc-1", + "owner-1", + user_id="owner-1", + group_id="group-1", + ) + + with _load_document_access_index_module( + redis_client, + groups_container=groups_container, + ) as (indexing, _index_container, settings_container): + settings_container.upsert_item(_succeeded_backfill_state()) + indexing.sync_document_access_index_for_document( + group_document, + operation="test_resolution_seed", + force=True, + ) + scope_hash = indexing.build_document_access_cache_scope_hash("group:group-1") + resolutions = indexing.resolve_document_access_cache_version_hashes([scope_hash]) + + resolution = resolutions[scope_hash] + assert resolution["resolved"] is True + assert resolution["entity_type"] == "group_workspace" + assert resolution["entity_id"] == "group-1" + assert resolution["entity_name"] == "Incident Workspace" + assert resolution["scope_key"] == "group:group-1" + assert resolution["row_count"] == 1 + + +def test_invalidation_runs_when_cache_setting_is_disabled(): + """Mutations should bump Redis scope versions even while cache reads are disabled.""" + redis_client = FakeRedisClient() + disabled_cache_settings = _settings() + disabled_cache_settings["enable_document_access_index_cache"] = False + + with _load_document_access_index_module(redis_client) as (indexing, index_container, settings_container): + _prepare_ready_projection(indexing, index_container, settings_container) + cached_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=_settings(), + ) + updated_document = _document("doc-1", "owner-1", title="Updated Title") + sync_result = indexing.sync_document_access_index_for_document( + updated_document, + operation="test_cache_disabled_update", + settings=disabled_cache_settings, + force=True, + ) + fresh_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=_settings(), + ) + + assert cached_result["status"] == "served_from_index" + assert sync_result["success"] is True + assert fresh_result["status"] == "served_from_index" + assert fresh_result["documents"][0]["title"] == "Updated Title" + assert len(redis_client.incr_calls) >= 2 + + +def test_failed_invalidation_marks_repair_and_blocks_cached_reads(): + """Redis invalidation failures should prevent stale cached access lists from being served.""" + redis_client = FailingIncrRedisClient() + + with _load_document_access_index_module(redis_client) as (indexing, index_container, settings_container): + _prepare_ready_projection(indexing, index_container, settings_container) + cached_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=_settings(), + ) + + redis_client.fail_incr = True + update_result = indexing.sync_document_access_index_for_document_fail_open( + _document("doc-1", "owner-1", title="Should Not Serve From Cache"), + operation="test_invalidation_failure", + settings=_settings(), + ) + blocked_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=_settings(), + ) + + assert cached_result["status"] == "served_from_index" + assert update_result["success"] is False + assert update_result["status"] == "repair_required" + assert blocked_result["success"] is False + assert blocked_result["status"] == "repair_backlog_present" + + +def test_missing_redis_client_when_configured_marks_repair(): + """Configured Redis outages during invalidation should block DAI/cache until repair succeeds.""" + redis_client = None + redis_configured_settings = _settings() + redis_configured_settings["enable_redis_cache"] = True + + with _load_document_access_index_module(redis_client) as (indexing, _index_container, settings_container): + _prepare_ready_projection(indexing, indexing.cosmos_document_access_index_container, settings_container) + update_result = indexing.sync_document_access_index_for_document_fail_open( + _document("doc-1", "owner-1", title="Redis Unavailable"), + operation="test_redis_unavailable_update", + settings=redis_configured_settings, + ) + blocked_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=redis_configured_settings, + ) + + assert update_result["success"] is False + assert update_result["status"] == "repair_required" + assert blocked_result["success"] is False + assert blocked_result["status"] == "repair_backlog_present" + + +def test_repair_invalidates_removed_share_scope_after_invalidation_failure(): + """Repair should invalidate cache scopes that were removed before invalidation failed.""" + redis_client = FailingIncrRedisClient() + shared_document = _document("doc-1", "owner-1", shared_user_ids=["user-2,approved"]) + + with _load_document_access_index_module(redis_client) as (indexing, index_container, settings_container): + _prepare_ready_projection(indexing, index_container, settings_container, document=shared_document) + cached_shared_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="user-2", + settings=_settings(), + ) + + redis_client.fail_incr = True + updated_document = _document("doc-1", "owner-1", shared_user_ids=[]) + update_result = indexing.sync_document_access_index_for_document_fail_open( + updated_document, + operation="test_share_revocation", + settings=_settings(), + ) + repair_docs = indexing._query_repair_documents() + + redis_client.fail_incr = False + indexing.cosmos_user_documents_container.upsert_item(updated_document) + repair_result = indexing.reconcile_document_access_index_repair_documents( + settings=_settings(), + force=True, + ) + post_repair_shared_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="user-2", + settings=_settings(), + ) + + assert cached_shared_result["status"] == "served_from_index" + assert cached_shared_result["documents"][0]["id"] == "doc-1" + assert update_result["success"] is False + assert update_result["status"] == "repair_required" + assert repair_docs and "user:user-2" in repair_docs[0].get("cache_scope_keys", []) + assert repair_result["repairs_failed"] == 0 + assert repair_result["repairs_succeeded"] == 1 + assert post_repair_shared_result["status"] == "served_from_index" + assert post_repair_shared_result["documents"] == [] + + +def test_partial_projection_failure_preserves_removed_share_scope_for_repair(): + """Non-cache projection failures after stale-row deletion should keep revoked scopes for repair.""" + redis_client = FakeRedisClient() + index_container = ToggleFailUpsertContainer() + shared_document = _document("doc-1", "owner-1", shared_user_ids=["user-2,approved"]) + + with _load_document_access_index_module( + redis_client, + index_container=index_container, + ) as (indexing, loaded_index_container, settings_container): + _prepare_ready_projection(indexing, loaded_index_container, settings_container, document=shared_document) + cached_shared_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="user-2", + settings=_settings(), + ) + + index_container.fail_upsert = True + update_result = indexing.sync_document_access_index_for_document_fail_open( + _document("doc-1", "owner-1", shared_user_ids=[]), + operation="test_partial_projection_failure", + settings=_settings(), + ) + repair_docs = indexing._query_repair_documents() + + assert cached_shared_result["documents"][0]["id"] == "doc-1" + assert update_result["success"] is False + assert update_result["status"] == "repair_required" + assert repair_docs and "user:user-2" in repair_docs[0].get("cache_scope_keys", []) + + +def test_stale_false_repair_backlog_state_is_verified_against_repair_docs(): + """A stale false backlog state should not allow DAI/cache reads when repair docs exist.""" + redis_client = FakeRedisClient() + + with _load_document_access_index_module(redis_client) as (indexing, _index_container, settings_container): + settings_container.upsert_item({ + "id": indexing.DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, + "type": indexing.DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_TYPE, + "has_repair_backlog": False, + "schema_version": indexing.DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION, + }) + settings_container.upsert_item({ + "id": indexing._repair_document_id("personal", "doc-stale"), + "type": indexing.DOCUMENT_ACCESS_REPAIR_TYPE, + "status": "repair_required", + "operation": "test_stale_false_state", + "source_scope": "personal", + "source_document_id": "doc-stale", + "schema_version": indexing.DOCUMENT_ACCESS_INDEX_SCHEMA_VERSION, + }) + + has_backlog = indexing.has_document_access_index_repair_backlog() + + assert has_backlog is True + + +def test_repair_doc_write_failure_keeps_backlog_uncleared(): + """Repair-count refresh should not clear backlog state after repair tracking fails.""" + redis_client = FailingIncrRedisClient() + settings_container = FailingRepairDocumentContainer() + + with _load_document_access_index_module( + redis_client, + settings_container=settings_container, + ) as (indexing, index_container, loaded_settings_container): + _prepare_ready_projection(indexing, index_container, loaded_settings_container) + + redis_client.fail_incr = True + settings_container.fail_repair_upserts = True + update_result = indexing.sync_document_access_index_for_document_fail_open( + _document("doc-1", "owner-1", title="Repair Tracking Failed"), + operation="test_repair_doc_write_failure", + settings=_settings(), + ) + repair_count = indexing.count_document_access_index_repair_documents() + repair_result = indexing.reconcile_document_access_index_repair_documents( + settings=_settings(), + force=True, + ) + has_backlog = indexing.has_document_access_index_repair_backlog() + state = loaded_settings_container.read_item( + item=indexing.DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, + partition_key=indexing.DOCUMENT_ACCESS_REPAIR_BACKLOG_STATE_DOC_ID, + ) + blocked_result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=_settings(), + ) + + assert update_result["success"] is False + assert update_result["status"] == "repair_required" + assert repair_count is None + assert repair_result["success"] is True + assert repair_result["repairs_processed"] == 0 + assert has_backlog is True + assert state["has_repair_backlog"] is True + assert state["repair_tracking_failed"] is True + assert blocked_result["success"] is False + assert blocked_result["status"] == "repair_backlog_present" + + +def test_tag_and_legacy_count_cache_hits_use_separate_keys(): + """Tag and legacy count reads should cache independently from document lists.""" + redis_client = FakeRedisClient() + document = _document("doc-legacy", "owner-1", percentage_complete=None) + with _load_document_access_index_module(redis_client) as (indexing, index_container, settings_container): + _prepare_ready_projection(indexing, index_container, settings_container, document=document) + + first_tags = indexing.query_document_access_index_tag_counts( + source_scope="personal", + user_id="owner-1", + settings=_settings(), + ) + second_tags = indexing.query_document_access_index_tag_counts( + source_scope="personal", + user_id="owner-1", + settings=_settings(), + ) + first_legacy = indexing.query_document_access_index_legacy_count( + source_scope="personal", + user_id="owner-1", + settings=_settings(), + ) + second_legacy = indexing.query_document_access_index_legacy_count( + source_scope="personal", + user_id="owner-1", + settings=_settings(), + ) + + assert first_tags["status"] == "served_from_index" + assert second_tags["status"] == "served_from_cache" + assert second_tags["tag_counts"] == {"cosmos": 1, "wave6": 1} + assert first_legacy["status"] == "served_from_index" + assert second_legacy["status"] == "served_from_cache" + assert second_legacy["legacy_count"] == 1 + + +def test_redis_failures_bypass_cache_without_forcing_source_fallback(): + """Redis cache failures should still allow direct DAI reads to succeed.""" + redis_client = FailingRedisClient() + with _load_document_access_index_module(redis_client) as (indexing, index_container, settings_container): + _prepare_ready_projection(indexing, index_container, settings_container) + + result = indexing.query_document_access_index_documents( + source_scope="personal", + user_id="owner-1", + settings=_settings(), + ) + cache_metrics = indexing.get_document_access_index_cache_metrics() + + assert result["success"] is True + assert result["status"] == "served_from_index" + assert result["documents"][0]["id"] == "doc-1" + assert cache_metrics["windows"]["15m"]["error_count"] >= 1 + + +if __name__ == "__main__": + tests = [ + test_document_list_cache_hit_and_scope_version_invalidation, + test_document_access_cache_version_marker_ttl_hygiene_refreshes_existing_markers, + test_document_access_cache_version_hash_resolution_returns_safe_scope_metadata, + test_invalidation_runs_when_cache_setting_is_disabled, + test_failed_invalidation_marks_repair_and_blocks_cached_reads, + test_missing_redis_client_when_configured_marks_repair, + test_repair_invalidates_removed_share_scope_after_invalidation_failure, + test_partial_projection_failure_preserves_removed_share_scope_for_repair, + test_stale_false_repair_backlog_state_is_verified_against_repair_docs, + test_repair_doc_write_failure_keeps_backlog_uncleared, + test_tag_and_legacy_count_cache_hits_use_separate_keys, + test_redis_failures_bypass_cache_without_forcing_source_fallback, + ] + results = [] + for test in tests: + print(f"\nRunning {test.__name__}...") + try: + test() + print("PASS") + results.append(True) + except Exception as exc: + print(f"FAIL: {exc}") + import traceback + traceback.print_exc() + results.append(False) + + passed = sum(results) + print(f"\nResults: {passed}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_document_metrics.py b/functional_tests/test_document_metrics.py index 476f41b7..c1ea3a4a 100644 --- a/functional_tests/test_document_metrics.py +++ b/functional_tests/test_document_metrics.py @@ -1,14 +1,15 @@ #!/usr/bin/env python3 """ Test improved document metrics implementation. -Version: 0.230.024 +Version: 0.250.047 Implemented in: 0.230.024 +CosmosClient module import cleanup updated in: 0.250.047 This script tests the improved document metrics for Control Center. """ import os -from azure.cosmos import CosmosClient +import azure.cosmos as azure_cosmos from datetime import datetime, timezone, timedelta def test_improved_document_metrics(): @@ -18,7 +19,7 @@ def test_improved_document_metrics(): endpoint = os.getenv('AZURE_COSMOS_ENDPOINT', '') key = os.getenv('AZURE_COSMOS_KEY', '') - client = CosmosClient(endpoint, key, consistency_level="Session") + client = azure_cosmos.CosmosClient(endpoint, key, consistency_level="Session") database = client.get_database_client("SimpleChat") # Get containers diff --git a/functional_tests/test_document_metrics_database_queries.py b/functional_tests/test_document_metrics_database_queries.py index b5fba265..9e82f1db 100644 --- a/functional_tests/test_document_metrics_database_queries.py +++ b/functional_tests/test_document_metrics_database_queries.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 """ Functional test for Enhanced Document Metrics database queries. -Version: 0.230.024 +Version: 0.250.047 Implemented in: 0.230.024 +CosmosClient module import cleanup updated in: 0.250.047 This test validates the document metrics calculation logic by directly testing the database queries and calculations without requiring a running Flask app. @@ -11,7 +12,7 @@ import sys import os from datetime import datetime -from azure.cosmos import CosmosClient +import azure.cosmos as azure_cosmos def test_document_metrics_queries(): """Test the document metrics database queries directly.""" @@ -22,7 +23,7 @@ def test_document_metrics_queries(): endpoint = os.getenv('AZURE_COSMOS_ENDPOINT', '') key = os.getenv('AZURE_COSMOS_KEY', '') - client = CosmosClient(endpoint, key, consistency_level="Session") + client = azure_cosmos.CosmosClient(endpoint, key, consistency_level="Session") # Test user ID test_user_id = "07e61033-ea1a-4472-a1e7-6b9ac874984a" diff --git a/functional_tests/test_fixed_queries.py b/functional_tests/test_fixed_queries.py index 382ccb47..169ed922 100644 --- a/functional_tests/test_fixed_queries.py +++ b/functional_tests/test_fixed_queries.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 """ Test the fixed query approach - separate count and size queries. +Version: 0.250.047 + +CosmosClient module import cleanup updated in: 0.250.047 """ import os import sys -from azure.cosmos import CosmosClient +import azure.cosmos as azure_cosmos def test_fixed_queries(): """Test the fixed query approach with separate count and size queries.""" @@ -14,7 +17,7 @@ def test_fixed_queries(): endpoint = os.getenv('AZURE_COSMOS_ENDPOINT', '') key = os.getenv('AZURE_COSMOS_KEY', '') - client = CosmosClient(endpoint, key, consistency_level="Session") + client = azure_cosmos.CosmosClient(endpoint, key, consistency_level="Session") database = client.get_database_client("SimpleChat") conversations_container = database.get_container_client("conversations") diff --git a/functional_tests/test_latest_release_docs_structure.py b/functional_tests/test_latest_release_docs_structure.py index 2f9da46b..997b2a68 100644 --- a/functional_tests/test_latest_release_docs_structure.py +++ b/functional_tests/test_latest_release_docs_structure.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """ Functional test for latest-release documentation structure. -Version: 0.250.034 -Implemented in: 0.241.002; 0.241.003; 0.241.164; 0.241.165; 0.241.166; 0.241.167; 0.241.183; 0.241.184; 0.250.001; 0.250.034 +Version: 0.250.047 +Implemented in: 0.241.002; 0.241.003; 0.241.164; 0.241.165; 0.241.166; 0.241.167; 0.241.183; 0.241.184; 0.250.001; 0.250.034; 0.250.035; 0.250.036; 0.250.041; 0.250.042; 0.250.043; 0.250.044; 0.250.045; 0.250.046; 0.250.047 This test ensures the docs/latest-release landing page is driven by the latest release YAML data, exposes current, previous, and earlier release sections, and @@ -109,7 +109,7 @@ def test_latest_release_docs_structure() -> bool: index_content = read_text(LATEST_RELEASE_INDEX) release_data = yaml.safe_load(read_text(LATEST_RELEASE_DATA)) - assert 'VERSION = "0.250.034"' in config_content, "Config version marker is not current." + assert 'VERSION = "0.250.047"' in config_content, "Config version marker is not current." required_index_markers = [ 'layout: latest-release-index', diff --git a/functional_tests/test_login_activity.py b/functional_tests/test_login_activity.py index 73591ddf..72e066cc 100644 --- a/functional_tests/test_login_activity.py +++ b/functional_tests/test_login_activity.py @@ -1,12 +1,15 @@ #!/usr/bin/env python3 """ Test script to debug login activity data for Control Center +Version: 0.250.047 + +CosmosClient module import cleanup updated in: 0.250.047 """ import sys import os from datetime import datetime, timedelta -from azure.cosmos import CosmosClient +import azure.cosmos as azure_cosmos def test_login_activity(): """Test what login activity data is available""" @@ -26,7 +29,7 @@ def test_login_activity(): endpoint = os.getenv('AZURE_COSMOS_ENDPOINT', '') key = os.getenv('AZURE_COSMOS_KEY', '') - client = CosmosClient(endpoint, key, consistency_level="Session") + client = azure_cosmos.CosmosClient(endpoint, key, consistency_level="Session") sample_records = list(client.get_database_client("SimpleChat").get_container_client("activity_logs").query_items( query=sample_query, diff --git a/functional_tests/test_public_workspace_tag_color_xss_fix.py b/functional_tests/test_public_workspace_tag_color_xss_fix.py index 1a6da159..a75e4c3d 100644 --- a/functional_tests/test_public_workspace_tag_color_xss_fix.py +++ b/functional_tests/test_public_workspace_tag_color_xss_fix.py @@ -1,7 +1,7 @@ # test_public_workspace_tag_color_xss_fix.py """ Functional test for public workspace tag color stored XSS hardening. -Version: 0.241.022 +Version: 0.250.031 Implemented in: 0.241.022 This test ensures tag colors are validated on write, repaired on read, @@ -184,7 +184,7 @@ def test_fix_artifacts_and_version_are_in_sync(): """Verify versioned regression artifacts landed for this fix.""" print("🔍 Testing fix artifacts and version alignment...") - assert read_config_version() == "0.241.022" + assert read_config_version() >= "0.241.022" assert os.path.exists(FIX_DOC), f"Expected fix documentation at {FIX_DOC}" assert os.path.exists(UI_TEST), f"Expected UI regression test at {UI_TEST}" diff --git a/functional_tests/test_tableau_action_plugin.py b/functional_tests/test_tableau_action_plugin.py index 53041109..09b21f5d 100644 --- a/functional_tests/test_tableau_action_plugin.py +++ b/functional_tests/test_tableau_action_plugin.py @@ -2,7 +2,7 @@ #!/usr/bin/env python3 """ Functional test for Tableau action plugin configuration. -Version: 0.250.030 +Version: 0.250.047 Implemented in: 0.241.210 This test ensures the Tableau action factory, plugin, manifest validation, @@ -37,16 +37,66 @@ def decorator(function): plugin_invocation_logger_stub.plugin_function_logger = plugin_function_logger sys.modules.setdefault("semantic_kernel_plugins.plugin_invocation_logger", plugin_invocation_logger_stub) -from functions_tableau_operations import ( # noqa: E402 - TABLEAU_AUTH_METHOD_PAT, - TABLEAU_AUTH_METHOD_USERNAME_PASSWORD, - TABLEAU_PLUGIN_TYPE, - normalize_tableau_additional_fields, - normalize_tableau_server_url, -) -from semantic_kernel_plugins.plugin_health_checker import PluginHealthChecker # noqa: E402 -from semantic_kernel_plugins.tableau_plugin_factory import TableauPluginFactory # noqa: E402 -import semantic_kernel_plugins.tableau_plugin as tableau_plugin_module # noqa: E402 + +class FakeConfigCosmosContainer: + """Minimal Cosmos container stand-in for importing app config in tests.""" + + def __init__(self): + self.items = {} + + def read_item(self, item, partition_key=None): + if item in self.items: + return self.items[item] + if item == "app_settings": + return {"id": "app_settings", "settings": {}} + raise KeyError(item) + + def upsert_item(self, item): + self.items[item["id"]] = item + return item + + def query_items(self, *args, **kwargs): + return [] + + +class FakeConfigCosmosDatabase: + """Minimal Cosmos database stand-in for importing config.py without live I/O.""" + + def __init__(self): + self.containers = {} + + def create_container_if_not_exists(self, id, **kwargs): + self.containers.setdefault(id, FakeConfigCosmosContainer()) + return self.containers[id] + + +class FakeConfigCosmosClient: + """Minimal Cosmos client stand-in for config.py import-time container setup.""" + + def __init__(self, *args, **kwargs): + self.database = FakeConfigCosmosDatabase() + + def create_database_if_not_exists(self, *args, **kwargs): + return self.database + + +import azure.cosmos as azure_cosmos # noqa: E402 + +original_cosmos_client = azure_cosmos.CosmosClient +azure_cosmos.CosmosClient = FakeConfigCosmosClient +try: + from functions_tableau_operations import ( # noqa: E402 + TABLEAU_AUTH_METHOD_PAT, + TABLEAU_AUTH_METHOD_USERNAME_PASSWORD, + TABLEAU_PLUGIN_TYPE, + normalize_tableau_additional_fields, + normalize_tableau_server_url, + ) + from semantic_kernel_plugins.plugin_health_checker import PluginHealthChecker # noqa: E402 + from semantic_kernel_plugins.tableau_plugin_factory import TableauPluginFactory # noqa: E402 + import semantic_kernel_plugins.tableau_plugin as tableau_plugin_module # noqa: E402 +finally: + azure_cosmos.CosmosClient = original_cosmos_client class FakeTableauItem: @@ -325,7 +375,7 @@ def test_tableau_identity_and_modal_contract(): assert "toggleTableauAuthFields" in modal_source assert "populateTableauSummary" in modal_source assert "tableauserverclient==0.40" in requirements_source - assert 'VERSION = "0.250.030"' in config_source + assert 'VERSION = "0.250.047"' in config_source print("Tableau identity and modal source contract verified.") return True diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..87d8b948 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,10 @@ +# requirements-dev.txt +# Development and test-only Python packages not already pinned by +# application/single_app/requirements.txt. +# +# Install after the base app requirements: +# pip install -r application/single_app/requirements.txt -r requirements-dev.txt + +pytest==9.0.3 +pytest-playwright==0.7.2 +azure-mgmt-playwright==1.0.0 diff --git a/ui_tests/test_admin_cosmos_throughput_settings_ui.py b/ui_tests/test_admin_cosmos_throughput_settings_ui.py index a1d6c656..b23e2e19 100644 --- a/ui_tests/test_admin_cosmos_throughput_settings_ui.py +++ b/ui_tests/test_admin_cosmos_throughput_settings_ui.py @@ -2,7 +2,7 @@ """ UI test for Admin Settings Cosmos throughput controls. -Version: 0.241.199 +Version: 0.250.045 Implemented in: 0.241.147 This test ensures the Scale tab exposes Cosmos throughput monitoring and @@ -21,6 +21,7 @@ Version 0.241.183 adds explicit setup guidance and detailed Validate Access diagnostics coverage. Version 0.241.184 adds neutral informational copy for normal container-targeted throughput mode. Version 0.241.199 adds SimpleChat's 10,000 RU/s scale support ceiling, monitor-only indicators, and container policy modal filtering coverage. +Version 0.250.045 adds a stable left-nav anchor for the Cosmos metrics table. """ import re @@ -76,6 +77,7 @@ def test_admin_cosmos_throughput_controls_render_from_template(): "cosmos-throughput-scale-down-btn", "cosmos-throughput-container-filter", "cosmos-throughput-container-filter-count", + "cosmos-throughput-metrics-table-section", "cosmos-throughput-container-policy-filter", "cosmos-throughput-container-policy-filter-count", "cosmos-throughput-refresh-table-btn", diff --git a/ui_tests/test_admin_document_access_index_settings_ui.py b/ui_tests/test_admin_document_access_index_settings_ui.py new file mode 100644 index 00000000..2c07580f --- /dev/null +++ b/ui_tests/test_admin_document_access_index_settings_ui.py @@ -0,0 +1,469 @@ +# test_admin_document_access_index_settings_ui.py +""" +UI test for Admin Settings document access index operations. + +Version: 0.250.046 +Implemented in: 0.250.011 +Default read enablement updated in: 0.250.027 +Redis DAI cache dashboard updated in: 0.250.029 +DAI debug UI cleanup updated in: 0.250.031 +Conversation cache metrics updated in: 0.250.034 +Cosmos maintenance cleanup updated in: 0.250.038 +Cosmos index apply action updated in: 0.250.039 +Scale left navigation updated in: 0.250.045 + +This test ensures the Scale tab exposes the DAI operations dashboard, +automatic maintenance status, production read metrics, default-hidden debug +controls, optional shadow validation, Wave 6 Redis DAI cache metrics, and +conversation cache metrics. +""" + +import re +from pathlib import Path + +import pytest + +try: + from playwright.sync_api import expect, sync_playwright +except ModuleNotFoundError: + expect = None + sync_playwright = None + + +REPO_ROOT = Path(__file__).resolve().parents[1] +ADMIN_TEMPLATE = REPO_ROOT / "application" / "single_app" / "templates" / "admin_settings.html" +ADMIN_JS = REPO_ROOT / "application" / "single_app" / "static" / "js" / "admin" / "admin_settings.js" +SIDEBAR_TEMPLATE = REPO_ROOT / "application" / "single_app" / "templates" / "_sidebar_nav.html" +ADMIN_SIDEBAR_JS = REPO_ROOT / "application" / "single_app" / "static" / "js" / "admin" / "admin_sidebar_nav.js" + + +def _extract_document_access_index_markup(template): + start = template.index('
    {card_html}") + section = page.locator("#conversation-cache-section") + expect(section).to_be_visible() + expect(section.get_by_role("heading", name="Conversation Cache")).to_be_visible() + expect(page.get_by_role("button", name="Refresh Metrics")).to_be_visible() + expect(page.get_by_label("Enable conversation cache")).to_be_checked() + expect(page.get_by_label("Cache TTL Seconds")).to_have_value("120") + expect(section.get_by_text("Runtime Status")).to_be_visible() + expect(section.get_by_text("15m Cache Hit Rate")).to_be_visible() + expect(section.get_by_text("15m Cache Hits / Misses")).to_be_visible() + expect(section.get_by_text("15m Cache Bypasses / Errors")).to_be_visible() + expect(section.get_by_text("15m Writes / Invalidations")).to_be_visible() + expect(section.get_by_text("15m Operation Mix")).to_be_visible() + expect(section.get_by_text("Last Cache Event")).to_be_visible() + expect(section.get_by_text("Last Invalidation")).to_be_visible() + expect(section.get_by_text("Phase 4")).to_have_count(0) + finally: + browser.close() + playwright_context.stop() + + +@pytest.mark.ui +def test_admin_document_access_index_dashboard_renders_safe_controls(): + """Validate the document access index dashboard markup and accessible controls.""" + if sync_playwright is None or expect is None: + pytest.skip("Install playwright to run this UI test.") + + template = ADMIN_TEMPLATE.read_text(encoding="utf-8") + card_html = _render_document_access_index_markup(template, enable_dai_debug=False) + + playwright_context = sync_playwright().start() + browser = playwright_context.chromium.launch() + page = browser.new_page(viewport={"width": 1280, "height": 900}) + + try: + page.set_content(f"
    {card_html}
    ") + section = page.locator("#document-access-index-section") + expect(section).to_be_visible() + expect(section.get_by_text("Cosmos Document Access Index")).to_be_visible() + expect(section.get_by_role("button", name="Refresh Status")).to_be_visible() + expect(page.get_by_role("button", name="Run One Backfill Batch")).to_have_count(0) + expect(page.get_by_role("button", name="Reset Checkpoint")).to_have_count(0) + expect(page.get_by_label("Write-through projection")).to_have_count(0) + expect(page.get_by_label("Automatic repair/backfill")).to_have_count(0) + expect(page.get_by_label("Enable shadow validation")).to_have_count(0) + expect(page.get_by_label("Backfill Batch Size")).to_have_count(0) + expect(page.get_by_label("Repair Batch Size")).to_have_count(0) + expect(section.get_by_text("Auto Maintenance")).to_be_visible() + expect(section.get_by_text("Next Maintenance Action")).to_be_visible() + expect(section.get_by_text("Redis List Cache")).to_be_visible() + expect(section.get_by_text("15m DAI Read Attempts")).to_be_visible() + expect(section.get_by_text("15m Redis Cache Hit Rate")).to_be_visible() + expect(section.get_by_text("15m Cache Hits / Misses")).to_be_visible() + expect(section.get_by_text("15m Cache Bypasses / Errors")).to_be_visible() + expect(section.get_by_text("15m Cache Invalidations")).to_be_visible() + expect(section.get_by_text("15m Source Fallbacks")).to_be_visible() + expect(section.get_by_text("15m Fallback Rate")).to_be_visible() + expect(section.get_by_text("Last Fallback Reason")).to_be_visible() + expect(section.get_by_text("Validation Index RU")).to_have_count(0) + expect(section.get_by_text("Candidate Read RU")).to_have_count(0) + expect(section.get_by_text("Estimated Wave 5 Savings")).to_have_count(0) + expect(section.get_by_text("Estimated Wave 5 Latency")).to_have_count(0) + expect(section.get_by_text("5m Source / Candidate RU")).to_have_count(0) + expect(section.get_by_text("15m Source / Candidate RU")).to_have_count(0) + expect(section.get_by_text("15m Shadow Samples")).to_have_count(0) + expect(page.locator("#enable_document_access_index_reads")).to_have_count(0) + expect(page.locator("#enable_document_access_index_shadow_validation")).to_have_count(0) + expect(page.get_by_label("Redis document list cache")).to_have_count(0) + expect(page.get_by_label("Cache TTL Seconds")).to_have_count(0) + expect(page.locator("#documentAccessIndexResetModal")).to_have_count(0) + finally: + browser.close() + playwright_context.stop() + + +@pytest.mark.ui +def test_admin_document_access_index_debug_mode_renders_hidden_controls(): + """Validate the Cosmos-edited DAI debug flag reveals support-only controls.""" + if sync_playwright is None or expect is None: + pytest.skip("Install playwright to run this UI test.") + + template = ADMIN_TEMPLATE.read_text(encoding="utf-8") + card_html = _render_document_access_index_markup(template, enable_dai_debug=True) + + playwright_context = sync_playwright().start() + browser = playwright_context.chromium.launch() + page = browser.new_page(viewport={"width": 1280, "height": 900}) + + try: + page.set_content(f"
    {card_html}
    ") + section = page.locator("#document-access-index-section") + expect(section).to_be_visible() + expect(page.get_by_role("button", name="Run One Backfill Batch")).to_be_visible() + expect(page.get_by_role("button", name="Reset Checkpoint")).to_be_visible() + expect(page.get_by_label("Write-through projection")).to_be_visible() + expect(page.get_by_label("Automatic repair/backfill")).to_be_visible() + expect(page.get_by_label("Enable shadow validation")).to_be_visible() + expect(page.locator("#enable_document_access_index_reads")).to_be_disabled() + expect(page.get_by_label("Redis document list cache")).to_be_visible() + expect(page.get_by_label("Cache TTL Seconds")).to_have_value("900") + expect(section.get_by_text("Validation Index RU")).to_be_visible() + expect(section.get_by_text("15m Shadow Samples")).to_be_visible() + expect(page.locator("#documentAccessIndexResetModal")).to_be_attached() + expect(page.locator("#document-access-index-reset-confirm-btn")).to_be_attached() + finally: + browser.close() + playwright_context.stop() + + +@pytest.mark.ui +def test_admin_cosmos_maintenance_dashboard_renders_safe_controls(): + """Validate the Cosmos maintenance dashboard markup and cleanup actions.""" + if sync_playwright is None or expect is None: + pytest.skip("Install playwright to run this UI test.") + + template = ADMIN_TEMPLATE.read_text(encoding="utf-8") + card_html = _render_cosmos_maintenance_markup(template) + + playwright_context = sync_playwright().start() + browser = playwright_context.chromium.launch() + page = browser.new_page(viewport={"width": 1280, "height": 900}) + + try: + page.set_content(f"
    {card_html}
    ") + section = page.locator("#cosmos-maintenance-section") + expect(section).to_be_visible() + expect(section.get_by_text("Cosmos Maintenance")).to_be_visible() + expect(page.get_by_role("button", name="Refresh Status")).to_be_visible() + expect(page.get_by_role("button", name="Apply Missing Indexes")).to_be_visible() + expect(page.get_by_role("button", name="Dry Run Cleanup")).to_be_visible() + expect(page.get_by_role("button", name="Delete Stale Cache Docs")).to_be_visible() + expect(section.get_by_text("Indexing Policy Status")).to_be_visible() + expect(section.get_by_text("Missing Expected Indexes")).to_be_visible() + expect(section.get_by_text("Composite indexes can increase write-index overhead")).to_be_visible() + expect(section.get_by_text("Stale Cleanup Status")).to_be_visible() + expect(section.get_by_text("Cleanup Categories")).to_be_visible() + expect(page.locator("#cosmosIndexingPolicyApplyModal")).to_be_attached() + expect(page.locator("#cosmos-indexing-policy-apply-confirm-btn")).to_be_attached() + expect(page.locator("#staleCacheCleanupApplyModal")).to_be_attached() + expect(page.locator("#stale-cache-cleanup-apply-confirm-btn")).to_be_attached() + finally: + browser.close() + playwright_context.stop() + + +def test_admin_document_access_index_dashboard_wiring_contract(): + """Validate static ids, endpoint wiring, and Wave 5B field exposure.""" + template = ADMIN_TEMPLATE.read_text(encoding="utf-8") + source = ADMIN_JS.read_text(encoding="utf-8") + + required_ids = [ + "conversation-cache-section", + "conversation-cache-refresh-btn", + "conversation-cache-runtime-status", + "conversation-cache-15m-hit-rate", + "conversation-cache-15m-hits-misses", + "conversation-cache-15m-bypasses-errors", + "conversation-cache-15m-writes-invalidations", + "conversation-cache-15m-operation-counts", + "conversation-cache-last-event", + "conversation-cache-last-invalidation", + "cosmos-maintenance-section", + "cosmos-maintenance-refresh-btn", + "cosmos-maintenance-message", + "cosmos-indexing-policy-status", + "cosmos-indexing-policy-mode", + "cosmos-indexing-policy-container-count", + "cosmos-indexing-policy-missing-count", + "cosmos-indexing-policy-updated-count", + "cosmos-indexing-policy-failed-count", + "cosmos-indexing-policy-last-evaluated", + "cosmos-indexing-policy-apply-btn", + "cosmosIndexingPolicyApplyModal", + "cosmos-indexing-policy-apply-confirm-btn", + "stale-cache-cleanup-dry-run-btn", + "stale-cache-cleanup-apply-btn", + "stale-cache-cleanup-status", + "stale-cache-cleanup-mode", + "stale-cache-cleanup-candidates", + "stale-cache-cleanup-deleted", + "stale-cache-cleanup-failed", + "stale-cache-cleanup-more-candidates", + "stale-cache-cleanup-categories", + "stale-cache-cleanup-last-evaluated", + "staleCacheCleanupApplyModal", + "stale-cache-cleanup-apply-confirm-btn", + "document-access-index-section", + "document-access-index-refresh-btn", + "document-access-index-run-batch-btn", + "document-access-index-reset-btn", + "documentAccessIndexResetModal", + "document-access-index-reset-confirm-btn", + "document-access-index-backfill-status", + "document-access-index-repair-count", + "document-access-index-maintenance-mode", + "document-access-index-maintenance-next-action", + "document-access-index-maintenance-more-work", + "document-access-index-maintenance-active-interval", + "document-access-index-cache-status", + "document-access-index-read-15m-attempts", + "document-access-index-cache-15m-hit-rate", + "document-access-index-cache-15m-hits-misses", + "document-access-index-cache-15m-bypasses-errors", + "document-access-index-cache-15m-invalidations", + "document-access-index-read-15m-served", + "document-access-index-read-15m-fallbacks", + "document-access-index-read-15m-fallback-rate", + "document-access-index-read-15m-ru", + "document-access-index-read-15m-latency", + "document-access-index-read-last-fallback", + "document-access-index-read-last-sample", + "document-access-index-cache-last-event", + "document-access-index-shadow-last-status", + "document-access-index-shadow-mismatches", + "document-access-index-shadow-ru-comparison", + "document-access-index-shadow-validation-ru", + "document-access-index-shadow-candidate-ru", + "document-access-index-shadow-wave5-ru-savings", + "document-access-index-shadow-ms-comparison", + "document-access-index-shadow-ms-savings", + "document-access-index-rolling-5m-ru-comparison", + "document-access-index-rolling-5m-wave5-ru-savings", + "document-access-index-rolling-15m-ru-comparison", + "document-access-index-rolling-15m-wave5-ru-savings", + "document-access-index-rolling-15m-validation-overhead", + "document-access-index-rolling-15m-samples", + "document-access-index-total-processed", + "document-access-index-last-error", + ] + + for element_id in required_ids: + assert f'id="{element_id}"' in template + + rendered_conversation_cache = _render_conversation_cache_markup(template) + assert "Phase 4" not in rendered_conversation_cache + assert "Conversation cache metrics are lightweight in-process counters" in rendered_conversation_cache + rendered_default = _render_document_access_index_markup(template, enable_dai_debug=False) + assert 'id="document-access-index-run-batch-btn"' not in rendered_default + assert 'id="enable_document_access_index_shadow_validation"' not in rendered_default + assert 'id="documentAccessIndexResetModal"' not in rendered_default + assert 'Validation Index RU' not in rendered_default + assert '15m Shadow Samples' not in rendered_default + rendered_debug = _render_document_access_index_markup(template, enable_dai_debug=True) + assert 'id="document-access-index-run-batch-btn"' in rendered_debug + assert 'id="enable_document_access_index_shadow_validation"' in rendered_debug + assert 'id="documentAccessIndexResetModal"' in rendered_debug + assert 'value="900"' in rendered_debug + assert "Default 900 seconds" in rendered_debug + assert "enable_dai_debug" in template + assert 'name="enable_dai_debug"' not in template + assert 'name="enable_document_access_index_write_through"' in template + assert 'name="enable_startup_document_access_index_backfill"' in template + assert 'name="enable_document_access_index_shadow_validation"' in template + assert 'name="document_access_index_backfill_batch_size"' in template + assert 'name="document_access_index_repair_batch_size"' in template + assert 'name="enable_document_access_index_reads"' in template + assert 'name="enable_document_access_index_cache"' in template + assert 'name="document_access_index_cache_ttl_seconds"' in template + assert "Wave 5B default" in template + assert "Wave 6" in template + assert "enable_document_access_index_shadow_validation_preview" not in template + assert "setupDocumentAccessIndexControls();" in source + assert "loadDocumentAccessIndexStatus(null, { showLoading: false });" in source + assert "function getDocumentAccessIndexReadWindow" in source + assert "function getDocumentAccessIndexCacheWindow" in source + assert "formatDocumentAccessIndexCacheEvent" in source + assert "function renderConversationCacheStatus" in source + assert "function renderCosmosMaintenanceStatus" in source + assert "function runCosmosIndexingPolicyApply" in source + assert "getCosmosIndexingPolicyStatusFromRunResult" in source + assert "function runStaleCacheCleanup" in source + assert "setupCosmosMaintenanceControls();" in source + assert "function formatConversationCacheOperationCounts" in source + assert "conversation-cache-refresh-btn" in source + assert "conversation-cache-15m-writes-invalidations" in source + assert "run_stale_cache_cleanup: true" in source + assert "apply_stale_cache_cleanup: applyChanges" in source + assert "apply_cosmos_indexing_policies: true" in source + assert "run_stale_cache_cleanup: false" in source + assert "Index transformation may continue asynchronously" in source + assert "formatDocumentAccessIndexLatencyWindow" in source + assert "'/api/admin/settings/app-maintenance/status'" in source + assert "'/api/admin/settings/app-maintenance/run'" in source + assert "apply_cosmos_indexing_policies: false" in source + assert "getNormalizedDocumentAccessIndexStatus(status) === 'running'" in source + assert "function isDocumentAccessIndexBackfillInProgress" in source + assert "function isDocumentAccessIndexBackfillActive" not in source + assert "return ['running', 'in_progress'].includes" not in source + + +def test_admin_scale_sidebar_metric_links_are_wired(): + """Validate left-nav metric shortcuts for DAI, Cosmos, and Redis sections.""" + sidebar_template = SIDEBAR_TEMPLATE.read_text(encoding="utf-8") + sidebar_source = ADMIN_SIDEBAR_JS.read_text(encoding="utf-8") + + expected_links = [ + ("Redis Cache", "redis-cache-section"), + ("Redis Metrics", "redis-monitoring-section"), + ("DAI Metrics", "document-access-index-section"), + ("Cosmos Maintenance", "cosmos-maintenance-section"), + ("Cosmos DB Throughput", "cosmos-throughput-section"), + ("Cosmos Metrics", "cosmos-throughput-metrics-table-section"), + ("Azure Front Door", "front-door-section"), + ] + for label, section_id in expected_links: + assert label in sidebar_template + assert f'data-section="{section_id}"' in sidebar_template + assert f"'{section_id}': '{section_id}'" in sidebar_source + + scale_submenu = sidebar_template[ + sidebar_template.index('id="scale-submenu"'): + sidebar_template.index('', sidebar_template.index('id="scale-submenu"')) + ] + ordered_labels = [ + "Redis Cache", + "Redis Metrics", + "DAI Metrics", + "Cosmos Maintenance", + "Cosmos DB Throughput", + "Cosmos Metrics", + "Azure Front Door", + ] + ordered_positions = [scale_submenu.index(label) for label in ordered_labels] + assert ordered_positions == sorted(ordered_positions) diff --git a/ui_tests/test_admin_redis_monitoring_settings_ui.py b/ui_tests/test_admin_redis_monitoring_settings_ui.py new file mode 100644 index 00000000..73a32b08 --- /dev/null +++ b/ui_tests/test_admin_redis_monitoring_settings_ui.py @@ -0,0 +1,176 @@ +# test_admin_redis_monitoring_settings_ui.py +""" +UI test for Admin Settings Redis monitoring. + +Version: 0.250.046 +Implemented in: 0.250.026 +Redis Explorer implemented in: 0.250.040 +Redis Explorer DAI resolution implemented in: 0.250.043 + +This test ensures the Scale tab exposes Redis health, capacity, hit-rate, +eviction, DAI hygiene, runtime monitoring, and read-only Redis Explorer +controls before Redis-backed DAI caching is enabled. +""" + +import re +from pathlib import Path + +import pytest + +try: + from playwright.sync_api import expect, sync_playwright +except ModuleNotFoundError: + expect = None + sync_playwright = None + + +REPO_ROOT = Path(__file__).resolve().parents[1] +ADMIN_TEMPLATE = REPO_ROOT / "application" / "single_app" / "templates" / "admin_settings.html" +ADMIN_JS = REPO_ROOT / "application" / "single_app" / "static" / "js" / "admin" / "admin_settings.js" + + +def _extract_redis_markup(template): + start = template.index('
    {card_html}") + section = page.locator("#redis-monitoring-section") + expect(section).to_be_visible() + expect(section.get_by_text("Redis Monitoring")).to_be_visible() + expect(page.get_by_role("button", name="Refresh Redis Status")).to_be_visible() + expect(page.get_by_role("button", name="Redis Explorer")).to_be_visible() + expect(section.get_by_text("Configuration")).to_be_visible() + expect(section.get_by_text("Health")).to_be_visible() + expect(section.get_by_text("App Cache Runtime")).to_be_visible() + expect(section.get_by_text("Session Runtime")).to_be_visible() + expect(section.get_by_text("Memory Usage")).to_be_visible() + expect(section.get_by_text("Keyspace Hit Rate")).to_be_visible() + expect(section.get_by_text("DAI Version Markers")).to_be_visible() + expect(section.get_by_text("DAI Cache Payloads")).to_be_visible() + expect(section.get_by_text("Expired / Evicted Keys")).to_be_visible() + expect(section.get_by_text("Rejected Connections")).to_be_visible() + expect(page.locator("#redisExplorerModal")).to_be_attached() + expect(page.get_by_label("Key Filter")).to_be_attached() + expect(page.get_by_text("Filters are case sensitive.")).to_be_attached() + expect(page.get_by_label("Page Size")).to_be_attached() + expect(page.locator("#redis-explorer-browse-all-btn")).to_be_attached() + expect(page.locator("#redis-explorer-search-btn")).to_be_attached() + expect(page.locator("#redis-explorer-prev-btn")).to_be_attached() + expect(page.locator("#redis-explorer-next-btn")).to_be_attached() + expect(page.get_by_text("SimpleChat Resolution")).to_be_attached() + finally: + browser.close() + playwright_context.stop() + + +def test_admin_redis_monitoring_dashboard_wiring_contract(): + """Validate static ids, endpoint wiring, and safe Redis rendering helpers.""" + template = ADMIN_TEMPLATE.read_text(encoding="utf-8") + source = ADMIN_JS.read_text(encoding="utf-8") + + required_ids = [ + "redis-monitoring-section", + "redis-monitoring-refresh-btn", + "redis-monitoring-message", + "redis-monitoring-config-status", + "redis-monitoring-health-status", + "redis-monitoring-app-cache-status", + "redis-monitoring-session-status", + "redis-monitoring-source", + "redis-monitoring-ping-latency", + "redis-monitoring-memory-usage", + "redis-monitoring-memory-policy", + "redis-monitoring-connected-clients", + "redis-monitoring-ops-per-sec", + "redis-monitoring-hit-rate", + "redis-monitoring-key-count", + "redis-monitoring-dai-version-markers", + "redis-monitoring-dai-version-marker-expiry", + "redis-monitoring-dai-payload-keys", + "redis-monitoring-dai-version-ttl-policy", + "redis-monitoring-expired-keys", + "redis-monitoring-evicted-keys", + "redis-monitoring-fragmentation", + "redis-monitoring-errors", + "redis-monitoring-rejected-connections", + "redis-monitoring-version", + "redis-monitoring-checked-at", + "redis-monitoring-last-error", + "redis-explorer-open-btn", + "redisExplorerModal", + "redis-explorer-filter", + "redis-explorer-page-size", + "redis-explorer-browse-all-btn", + "redis-explorer-search-btn", + "redis-explorer-refresh-btn", + "redis-explorer-prev-btn", + "redis-explorer-next-btn", + "redis-explorer-key-list", + "redis-explorer-key-count", + "redis-explorer-message", + "redis-explorer-preview-empty", + "redis-explorer-preview-panel", + "redis-explorer-preview-key", + "redis-explorer-preview-type", + "redis-explorer-preview-ttl", + "redis-explorer-preview-memory", + "redis-explorer-preview-redacted", + "redis-explorer-resolution-card", + "redis-explorer-resolution-kind", + "redis-explorer-resolution-entity", + "redis-explorer-resolution-scope", + "redis-explorer-resolution-note", + "redis-explorer-preview-content", + ] + + for element_id in required_ids: + assert f'id="{element_id}"' in template + + redis_markup = _extract_redis_markup(template) + assert 'style="display: none;"' not in redis_markup + assert 'style="max-height:' not in redis_markup + assert "Leave the filter blank" in redis_markup + assert "Filters are case sensitive." in redis_markup + assert "APP_SETTINGS_CACHE" in redis_markup + assert "SimpleChat Resolution" in redis_markup + assert "modal-dialog-scrollable redis-explorer-dialog" in redis_markup + assert ".redis-explorer-body" in template + assert "overflow-y: auto;" in template + assert ".redis-explorer-key-list" in template + assert "height: clamp(16rem, 36vh, 24rem);" in template + assert "overscroll-behavior: contain;" in template + assert "redis-explorer-key-list\" style=" not in redis_markup + assert "setupRedisMonitoringControls();" in source + assert "setupRedisExplorerControls();" in source + assert "function renderRedisMonitoringStatus" in source + assert "function loadRedisMonitoringStatus" in source + assert "function loadRedisExplorerKeys" in source + assert "function loadRedisExplorerValue" in source + assert "function getRedisExplorerScopeLabel" in source + assert "function renderRedisExplorerResolution" in source + assert "statusPayload?.dai_cache" in source + assert "redis-explorer-browse-all-btn" in source + assert "'/api/admin/settings/redis-monitoring/status'" in source + assert "'/api/admin/settings/redis-explorer/value'" in source + assert "/api/admin/settings/redis-explorer/keys" in source + assert "setRedisTestResult(resultDiv, data.message || 'Redis connection successful.', 'success');" in source + assert "redisSettingsDiv.style.display" not in source + assert "innerHTML" not in source[source.index("function renderRedisExplorerKeys"):source.index("function getRedisExplorerRequestState")]