diff --git a/.alita/tests/test_pipelines/configs/sharepoint-index-config.json b/.alita/tests/test_pipelines/configs/sharepoint-index-config.json new file mode 100644 index 000000000..ef2ff510f --- /dev/null +++ b/.alita/tests/test_pipelines/configs/sharepoint-index-config.json @@ -0,0 +1,10 @@ +{ + "type": "sharepoint", + "toolkit_name": "sp-index-test", + "selected_tools": [ + "index_data", + "search_index", + "list_collections", + "remove_index" + ] +} diff --git a/.alita/tests/test_pipelines/scripts/setup_strategy.py b/.alita/tests/test_pipelines/scripts/setup_strategy.py index 8a3e34cbb..b7c5788c0 100644 --- a/.alita/tests/test_pipelines/scripts/setup_strategy.py +++ b/.alita/tests/test_pipelines/scripts/setup_strategy.py @@ -333,6 +333,8 @@ def _build_toolkit_settings(self, toolkit_type: str, config: Dict[str, Any], loa 2. If alita_title is present, merge in stored configuration data from configuration step 3. All placeholders should already be resolved by resolve_env_value before this method 4. Fill in any defaults from the Pydantic model if needed + 5. Programmatically create {toolkit_type}_configuration from environment variables + 6. If indexing tools are selected, add pgvector_configuration and embedding_model Args: toolkit_type: Type of toolkit (e.g., 'github', 'jira') @@ -391,7 +393,7 @@ def _build_toolkit_settings(self, toolkit_type: str, config: Dict[str, Any], loa logger.info(f"[LOCAL DEBUG] Merged {key} into toolkit_config (overwrite)") logger.info(f"[LOCAL DEBUG] toolkit_config AFTER merge: {toolkit_config}") - # Fill in defaults from Pydantic model if available + # Programmatically populate toolkit_configuration from environment variables using Pydantic model try: from alita_sdk.configurations import get_class_configurations @@ -409,23 +411,84 @@ def _build_toolkit_settings(self, toolkit_type: str, config: Dict[str, Any], loa except Exception as e: # If configuration class not available, just use what we have + # logger.warning(f"[LOCAL DEBUG] Could not auto-populate toolkit_configuration: {e}") pass # Store the configuration in settings settings[config_key] = toolkit_config + # Check if indexing tools are selected + selected_tools = config.get('selected_tools', []) + logger.info(f"[INDEXING DEBUG] selected_tools from config: {selected_tools}") + indexing_tools = {'index_data', 'search_index', 'stepback_search_index', + 'stepback_summary_index', 'remove_index', 'list_collections'} + has_indexing_tools = bool(set(selected_tools) & indexing_tools) + logger.info(f"[INDEXING DEBUG] has_indexing_tools: {has_indexing_tools}, intersection: {set(selected_tools) & indexing_tools}") + + if has_indexing_tools: + logger.info(f"[INDEXING DEBUG] ===== ENTERING INDEXING TOOLS AUTO-CONFIG BLOCK =====") + logger.info(f"[LOCAL DEBUG] Indexing tools detected in selected_tools, adding pgvector_configuration and embedding_model") + + # Add pgvector_configuration if not already present + if 'pgvector_configuration' not in settings or not settings['pgvector_configuration']: + pgvector_config = {} + + # Try to load from environment or stored configuration + if 'pgvector_configuration' in config: + pgvector_config = config['pgvector_configuration'].copy() if isinstance(config['pgvector_configuration'], dict) else {} + + # If alita_title is present in pgvector_configuration, merge stored data + if 'alita_title' in pgvector_config and pgvector_config['alita_title'] in self._configuration_data: + stored_data = self._configuration_data[pgvector_config['alita_title']] + for key, value in stored_data.items(): + if key not in ('alita_title', 'private'): + pgvector_config[key] = value + + # Otherwise, load from PGVECTOR_CONNECTION_STRING environment variable + if 'connection_string' not in pgvector_config or not pgvector_config['connection_string']: + logger.info(f"[INDEXING DEBUG] Attempting to load PGVECTOR_CONNECTION_STRING from environment") + conn_str = load_from_env('PGVECTOR_CONNECTION_STRING') + logger.info(f"[INDEXING DEBUG] PGVECTOR_CONNECTION_STRING loaded: {bool(conn_str)}, length: {len(conn_str) if conn_str else 0}") + if conn_str: + from pydantic import SecretStr + pgvector_config['connection_string'] = SecretStr(conn_str) + logger.info(f"[LOCAL DEBUG] Loaded pgvector connection_string from PGVECTOR_CONNECTION_STRING") + else: + logger.warning(f"[INDEXING DEBUG] Failed to load PGVECTOR_CONNECTION_STRING from environment") + else: + logger.info(f"[INDEXING DEBUG] connection_string already in pgvector_config") + + settings['pgvector_configuration'] = pgvector_config + + # Add embedding_model if not already present + if 'embedding_model' not in settings or not settings['embedding_model']: + logger.info(f"[INDEXING DEBUG] embedding_model not in settings, attempting to load...") + embedding_model = config.get('embedding_model') or load_from_env('EMBEDDING_MODEL') + logger.info(f"[INDEXING DEBUG] Loaded embedding_model value: {embedding_model}") + if embedding_model: + settings['embedding_model'] = embedding_model + logger.info(f"[LOCAL DEBUG] Set embedding_model to {embedding_model}") + else: + logger.warning(f"[INDEXING DEBUG] Failed to load EMBEDDING_MODEL from config or environment") + else: + logger.info(f"[INDEXING DEBUG] embedding_model already in settings: {settings.get('embedding_model')}") + + logger.info(f"[INDEXING DEBUG] Final settings keys: {list(settings.keys())}") + logger.info(f"[INDEXING DEBUG] Final settings has pgvector_configuration: {'pgvector_configuration' in settings}") + logger.info(f"[INDEXING DEBUG] Final settings has embedding_model: {'embedding_model' in settings}") + return settings def _create_alita_client(self) -> Optional[Any]: """Create AlitaClient for toolkit initialization.""" - from utils_common import load_from_env + from utils_common import load_base_url_from_env, load_token_from_env, load_project_id_from_env try: from alita_sdk.runtime.clients.client import AlitaClient - deployment_url = load_from_env('DEPLOYMENT_URL') or load_from_env('BASE_URL') - api_key = load_from_env('API_KEY') or load_from_env('AUTH_TOKEN') - project_id = load_from_env('PROJECT_ID') + deployment_url = load_base_url_from_env() + api_key = load_token_from_env() + project_id = load_project_id_from_env() if not deployment_url or not api_key: return None @@ -433,7 +496,7 @@ def _create_alita_client(self) -> Optional[Any]: return AlitaClient( base_url=deployment_url, auth_token=api_key, - project_id=int(project_id) if project_id else 0, + project_id=project_id or 0, ) except Exception: return None @@ -485,6 +548,33 @@ def handle_toolkit_create( except FileNotFoundError: ctx.log(f"Config file not found: {config['config_file']}", "warning") + # Auto-detect indexing tools and add pgvector_configuration + embedding_model if needed + selected_tools = file_config.get('selected_tools', []) + indexing_tools = {'index_data', 'search_index', 'stepback_search_index', + 'stepback_summary_index', 'remove_index', 'list_collections'} + has_indexing_tools = bool(set(selected_tools) & indexing_tools) + + if has_indexing_tools: + ctx.log(f"[LOCAL] Auto-configuring indexing tools for toolkit", "info") + + # Add pgvector_configuration if not in file_config + if 'pgvector_configuration' not in file_config: + file_config['pgvector_configuration'] = {} + + # Ensure connection_string is set + if 'connection_string' not in file_config['pgvector_configuration']: + conn_str = load_from_env('PGVECTOR_CONNECTION_STRING') + if conn_str: + file_config['pgvector_configuration']['connection_string'] = conn_str + ctx.log(f"[LOCAL] Auto-added PGVECTOR_CONNECTION_STRING to config", "info") + + # Add embedding_model if not in file_config + if 'embedding_model' not in file_config: + embedding_model = load_from_env('EMBEDDING_MODEL') + if embedding_model: + file_config['embedding_model'] = embedding_model + ctx.log(f"[LOCAL] Auto-added EMBEDDING_MODEL={embedding_model} to config", "info") + # Apply overrides - resolve environment variables in overrides first overrides = resolve_env_value(config.get("overrides", {}), ctx.env_vars, env_loader=load_from_env) for key, value in overrides.items(): diff --git a/.alita/tests/test_pipelines/suites/index_sharepoint/pipeline.yaml b/.alita/tests/test_pipelines/suites/index_sharepoint/pipeline.yaml new file mode 100644 index 000000000..ae7e5cb86 --- /dev/null +++ b/.alita/tests/test_pipelines/suites/index_sharepoint/pipeline.yaml @@ -0,0 +1,92 @@ +name: index_sharepoint +description: | + SharePoint indexing test suite. + + Tests cover two areas: + - index_data called with different path parameter styles (ISP01–ISP05): + no path, relative path, server-relative path, form_name only, path+form_name combo. + All paths are expected to reach the same set of test files so indexed counts should + be comparable across the five tests. + - search_index called with progressively sophisticated parameter combinations (ISP06–ISP13), + each preceded by a fresh index_data call scoped to the test folder. + +env_mapping: + SHAREPOINT_SITE_URL: ${SHAREPOINT_SITE_URL:https://5clkvm.sharepoint.com/sites/ALITA} + SHAREPOINT_CLIENT_ID: ${SHAREPOINT_CLIENT_ID:84c01a1e-1ebc-40c0-b55a-8b3fae194e41} + SHAREPOINT_CLIENT_SECRET: ${SHAREPOINT_CLIENT_SECRET} + PGVECTOR_CONNECTION_STRING: ${PGVECTOR_CONNECTION_STRING} + EMBEDDING_MODEL: ${EMBEDDING_MODEL:text-embedding-ada-002} + TEST_FOLDER: ${TEST_FOLDER:Shared Documents/DO_NOT_DELETE_AlitaTestFiles} + SERVER_RELATIVE_PATH: ${SERVER_RELATIVE_PATH:/sites/ALITA/Shared Documents/DO_NOT_DELETE_AlitaTestFiles} + TEST_FORM_NAME: ${TEST_FORM_NAME:Shared Documents} + TEST_SUBFOLDER: ${TEST_SUBFOLDER:DO_NOT_DELETE_AlitaTestFiles} + TEST_FILE_NAME: ${TEST_FILE_NAME:DO_NOT_DELETE_test-document.txt} + SEARCH_QUERY: ${SEARCH_QUERY:test document} + +setup: + - name: Setup SharePoint Configuration + type: configuration + config: + config_type: sharepoint + alita_title: ${SHAREPOINT_SECRET_NAME:sharepoint} + data: + site_url: ${SHAREPOINT_SITE_URL:https://5clkvm.sharepoint.com/sites/ALITA} + client_id: ${SHAREPOINT_CLIENT_ID:84c01a1e-1ebc-40c0-b55a-8b3fae194e41} + client_secret: ${SHAREPOINT_CLIENT_SECRET} + + - name: Create SharePoint Index Toolkit + type: toolkit + action: create_or_update + config: + config_file: ../../configs/sharepoint-index-config.json + toolkit_type: sharepoint + overrides: + sharepoint_configuration: + private: false + alita_title: ${SHAREPOINT_SECRET_NAME:sharepoint} + pgvector_configuration: + private: false + alita_title: ${PGVECTOR_SECRET_NAME:elitea-pgvector} + embedding_model: ${EMBEDDING_MODEL:text-embedding-ada-002} + toolkit_name: ${ISP_TOOLKIT_NAME:sp-index-test} + save_to_env: + - key: ISP_TOOLKIT_ID + value: $.id + - key: ISP_TOOLKIT_NAME + value: $.name + +composable_pipelines: [] + +execution: + test_directory: tests + order: + - test_case_*.yaml + + substitutions: + ISP_TOOLKIT_ID: ${ISP_TOOLKIT_ID} + ISP_TOOLKIT_NAME: ${ISP_TOOLKIT_NAME} + TEST_FOLDER: ${TEST_FOLDER:Shared Documents/DO_NOT_DELETE_AlitaTestFiles} + SERVER_RELATIVE_PATH: ${SERVER_RELATIVE_PATH:/sites/ALITA/Shared Documents/DO_NOT_DELETE_AlitaTestFiles} + TEST_FORM_NAME: ${TEST_FORM_NAME:Shared Documents} + TEST_SUBFOLDER: ${TEST_SUBFOLDER:DO_NOT_DELETE_AlitaTestFiles} + TEST_FILE_NAME: ${TEST_FILE_NAME:DO_NOT_DELETE_test-document.txt} + SEARCH_QUERY: ${SEARCH_QUERY:test document} + TIMESTAMP: ${TIMESTAMP} + DEFAULT_LLM_MODEL: ${DEFAULT_LLM_MODEL:gpt-4o-2024-11-20} + + settings: + # Run sequentially: compound tests re-use unique index names so parallel is safe, + # but sequential avoids overwhelming the SharePoint Graph API quota. + timeout: 900 + parallel: 1 + stop_on_failure: false + +exclude_tests: [] + +cleanup: + - name: Delete SharePoint Index Toolkit + type: toolkit + config: + toolkit_id: ${ISP_TOOLKIT_ID} + enabled: true + continue_on_error: true diff --git a/.alita/tests/test_pipelines/suites/index_sharepoint/tests/test_case_01_index_data_no_path.yaml b/.alita/tests/test_pipelines/suites/index_sharepoint/tests/test_case_01_index_data_no_path.yaml new file mode 100644 index 000000000..71721c61a --- /dev/null +++ b/.alita/tests/test_pipelines/suites/index_sharepoint/tests/test_case_01_index_data_no_path.yaml @@ -0,0 +1,115 @@ +name: "ISP01-no-path" +priority: Critical +description: | + Verify index_data indexes all accessible SharePoint files with no path restriction. + + Objective: Baseline indexing — establish that index_data completes successfully + and indexes documents from all accessible libraries. + + Expected Behavior: + - Tool executes without raising an exception + - Result contains "status" field equal to "ok" + - Search returns indexed documents successfully + - Documents are indexed and searchable + +toolkits: + - id: ${ISP_TOOLKIT_ID} + name: ${ISP_TOOLKIT_NAME} + +state: + index_result: + type: dict + search_result: + type: dict + test_results: + type: dict + +entry_point: invoke_index_data + +nodes: + - id: invoke_index_data + type: toolkit + tool: index_data + toolkit_name: ${ISP_TOOLKIT_NAME} + input: [] + input_mapping: + index_name: + type: fixed + value: "idx01" + clean_index: + type: fixed + value: true + limit_files: + type: fixed + value: 20 + output: + - index_result + structured_output: true + transition: search_indexed_docs + + - id: search_indexed_docs + type: toolkit + tool: search_index + toolkit_name: ${ISP_TOOLKIT_NAME} + input: [] + input_mapping: + query: + type: fixed + value: "document" + index_name: + type: fixed + value: "idx01" + search_top: + type: fixed + value: 100 + output: + - search_result + structured_output: true + transition: validate_result + + - id: validate_result + type: llm + model: gpt-4o-2024-11-20 + input: + - index_result + - search_result + input_mapping: + system: + type: fixed + value: "You are a quality assurance validator." + task: + type: fstring + value: | + Analyze the output from the 'index_data' tool and search results. + + Index Result: {index_result} + Search Result: {search_result} + + Expected behavior: + 1. Indexing completed successfully (status: ok) + 2. Search returns indexed documents (non-empty results) + 3. No critical errors or exceptions + + Validation rules: + 1. Check index_result has "status": "ok" + 2. Check index_result message indicates successful indexing + 3. Verify search_result is not empty and contains document list + 4. Confirm at least one document was indexed and is searchable + 5. All validations must pass for test to pass + + return a json object with: + {{ + "test_passed": true/false, + "summary": "brief description including number of documents found", + "error": "error details if failed, null if passed" + }} + + return **only** the json object. no markdown formatting, no additional text. + chat_history: + type: fixed + value: [] + output: + - test_results + structured_output_dict: + test_results: "dict" + transition: END diff --git a/.alita/tests/test_pipelines/suites/index_sharepoint/tests/test_case_02_index_data_relative_path.yaml b/.alita/tests/test_pipelines/suites/index_sharepoint/tests/test_case_02_index_data_relative_path.yaml new file mode 100644 index 000000000..7f75137af --- /dev/null +++ b/.alita/tests/test_pipelines/suites/index_sharepoint/tests/test_case_02_index_data_relative_path.yaml @@ -0,0 +1,175 @@ +name: "ISP02-filtered" +priority: High +description: | + Verify index_data indexes SharePoint files with extension filtering and custom chunking config. + + Tests: + 1. Index all files except png (skip_extensions) + 2. Apply chunking config {"max_tokens": 256} for .txt files + 3. Search index to verify all documents are present + 4. Verify text_large.txt has exactly 4 chunks using filter + + Objective: Confirm skip_extensions and chunking_config parameters work correctly + and that indexed documents are searchable with proper chunking applied. + + Expected Behavior: + - Index completes successfully (status: ok) + - Search returns indexed documents + - text_large.txt filter search returns exactly 4 chunks + - png files are not indexed + +toolkits: + - id: ${ISP_TOOLKIT_ID} + name: ${ISP_TOOLKIT_NAME} + +state: + skip_extensions: + type: list + value: ["png"] + chunking_config: + type: dict + value: + ".txt": + max_tokens: 256 + index_result: + type: dict + search_all_result: + type: dict + search_large_txt_result: + type: dict + test_results: + type: dict + +entry_point: invoke_index_data + +nodes: + - id: invoke_index_data + type: toolkit + tool: index_data + toolkit_name: ${ISP_TOOLKIT_NAME} + input: + - skip_extensions + - chunking_config + input_mapping: + index_name: + type: fixed + value: "idx02" + clean_index: + type: fixed + value: true + path: + type: fixed + value: "${TEST_SUBFOLDER}" + limit_files: + type: fixed + value: 100 + skip_extensions: + type: variable + value: skip_extensions + chunking_config: + type: variable + value: chunking_config + output: + - index_result + structured_output: true + transition: search_all_docs + + - id: search_all_docs + type: toolkit + tool: search_index + toolkit_name: ${ISP_TOOLKIT_NAME} + input: [] + input_mapping: + query: + type: fixed + value: "document" + index_name: + type: fixed + value: "idx02" + search_top: + type: fixed + value: 50 + output: + - search_all_result + structured_output: true + transition: search_text_large + + - id: search_text_large + type: toolkit + tool: search_index + toolkit_name: ${ISP_TOOLKIT_NAME} + input: [] + input_mapping: + query: + type: fixed + value: "text content" + index_name: + type: fixed + value: "idx02" + filter: + type: fixed + value: + Name: "text_large.txt" + search_top: + type: fixed + value: 50 + output: + - search_large_txt_result + structured_output: true + transition: validate_result + + - id: validate_result + type: llm + model: gpt-4o-2024-11-20 + input: + - index_result + - search_all_result + - search_large_txt_result + - skip_extensions + input_mapping: + system: + type: fixed + value: "You are a quality assurance validator." + task: + type: fstring + value: | + Analyze the indexing and search results for SharePoint documents. + + Index Result: {index_result} + Search All Docs: {search_all_result} + Search text_large.txt (filtered): {search_large_txt_result} + Skip Extensions: {skip_extensions} + + Expected behavior: + 1. Indexing completed successfully (status: ok) + 2. Search returns indexed documents (non-empty results) + 3. search_large_txt_result must contain EXACTLY 4 items (4 chunks) + 4. ALL items in search_large_txt_result must be for text_large.txt (no other files) + 5. No png files should be in indexed documents (they were skipped) + + Validation rules: + 1. Check index_result has "status": "ok" + 2. Check index_result message indicates successful indexing + 3. Verify search_all_result is not empty and contains document list + 4. Count total items in search_large_txt_result array - must be EXACTLY 4 + 5. Verify every item in search_large_txt_result has Name == "text_large.txt" + 6. If count != 4 or any item is not text_large.txt, test fails + 7. Verify no .png files appear in search_all_result (check "Name" field in metadata) + 8. All validations must pass for test to pass + + Return a json object with: + {{ + "test_passed": true/false, + "summary": "brief description including chunk count for text_large.txt", + "error": "error details if failed, null if passed" + }} + + Return **only** the json object. No markdown formatting, no additional text. + chat_history: + type: fixed + value: [] + output: + - test_results + structured_output_dict: + test_results: "dict" + transition: END diff --git a/.alita/tests/test_pipelines/suites/sharepoint/README.md b/.alita/tests/test_pipelines/suites/sharepoint/README.md index 670c69fb2..0b8a7ee28 100644 --- a/.alita/tests/test_pipelines/suites/sharepoint/README.md +++ b/.alita/tests/test_pipelines/suites/sharepoint/README.md @@ -10,12 +10,10 @@ Test suite for SharePoint toolkit under `alita_sdk/tools/sharepoint/`. | read_list | test_case_03, test_case_04 | Critical, High | ✅ Complete (Self-contained) | | get_list_columns | test_case_05, test_case_06 | Critical, High | ✅ Complete | | create_list_item | test_case_07, test_case_08 | Critical, High | ✅ Complete | -| get_files_list | test_case_09, test_case_10 | Critical, High | ✅ Complete | +| get_files_list | test_case_09-18 | Critical, High | ✅ Complete (10 tests) | | read_document | test_case_11, test_case_12 | Critical, High | ✅ Complete (Self-contained) | -| upload_file | test_case_13, test_case_14 | Critical, High | ✅ Complete (Self-contained) | -| add_attachment_to_list_item | test_case_15, test_case_16 | Critical, High | ✅ Complete (Self-contained) | -**Coverage**: 8/8 tools (100%), 16/16 test files complete +**Coverage**: 6/6 core tools (100%), 18 test files complete ## Setup Artifacts @@ -62,17 +60,27 @@ Required variables (set in `.alita/tests/test_pipelines/.env`): - **SP07**: Create list item - Basic item creation - **SP08**: Create item missing fields - Validation error handling -### File Operations (SP09-SP14) -- **SP09**: List root files - Basic file listing -- **SP10**: List files in folder - Folder filtering +### File Operations (SP09-SP17) + +#### Basic File Listing (SP09-SP10) +- **SP09**: List root files - Basic file listing in root document library +- **SP10**: List files in folder - Folder filtering with folder_name parameter + +#### File Selection Parameters (SP13-SP18) +Tests for `get_files_list` file selection logic focusing on parameters that affect which files are returned: + +- **SP13**: Limit parameter validation - Tests limit_files parameter with values 2 and 5, verifies exact count control +- **SP14**: Include specific extensions - Tests include_extensions parameter with single extension ['txt'] and multiple ['.pdf', 'docx'], verifies ALL specified extensions are present +- **SP15**: Skip specific extensions - Tests skip_extensions parameter to exclude file types, verifies NONE of the specified extensions are present +- **SP16**: Document library filtering - Tests form_name parameter to scope results to specific library ('Shared Documents') +- **SP17**: Combined filters - Tests all parameters together (form_name + folder_name + include_extensions + skip_extensions) +- **SP18**: Include/Exclude conflict (edge case) - Tests precedence when same extension appears in both include and skip (skip should win) + +**Coverage**: All file selection parameters tested at the get_files_list level (bypassing full indexer) + +#### Document Content (SP11-SP12) - **SP11**: Read document content - Self-contained: uploads file, then reads it - **SP12**: Read non-existent file - Error handling -- **SP13**: Upload new file - Self-contained: creates inline file content -- **SP14**: Upload replace existing - Self-contained: uploads twice to test replacement - -### Attachment Operations (SP15-SP16) -- **SP15**: Add attachment to item - Self-contained: creates item, adds attachment -- **SP16**: Add attachment replace - Self-contained: creates item, adds twice to test replacement ## Prerequisites diff --git a/.alita/tests/test_pipelines/suites/sharepoint/pipeline.yaml b/.alita/tests/test_pipelines/suites/sharepoint/pipeline.yaml index 7b75bb666..7979002e6 100644 --- a/.alita/tests/test_pipelines/suites/sharepoint/pipeline.yaml +++ b/.alita/tests/test_pipelines/suites/sharepoint/pipeline.yaml @@ -14,6 +14,7 @@ env_mapping: SHAREPOINT_CLIENT_SECRET: ${SHAREPOINT_CLIENT_SECRET} TEST_LIST_NAME: ${TEST_LIST_NAME:DO_NOT_DELETE_AlitaTestList} TEST_FOLDER: ${TEST_FOLDER:Shared Documents/DO_NOT_DELETE_AlitaTestFiles} + SERVER_RELATIVE_PATH: ${SERVER_RELATIVE_PATH:/sites/ALITA/Shared Documents/DO_NOT_DELETE_AlitaTestFiles} TEST_FILE_NAME: ${TEST_FILE_NAME:DO_NOT_DELETE_test-document.txt} setup: @@ -56,6 +57,7 @@ execution: SHAREPOINT_TOOLKIT_NAME: ${SHAREPOINT_TOOLKIT_NAME} TEST_LIST_NAME: ${TEST_LIST_NAME:DO_NOT_DELETE_AlitaTestList} TEST_FOLDER: ${TEST_FOLDER:Shared Documents/DO_NOT_DELETE_AlitaTestFiles} + SERVER_RELATIVE_PATH: ${SERVER_RELATIVE_PATH:/sites/ALITA/Shared Documents/DO_NOT_DELETE_AlitaTestFiles} TEST_FILE_NAME: ${TEST_FILE_NAME:DO_NOT_DELETE_test-document.txt} TIMESTAMP: ${TIMESTAMP} DEFAULT_LLM_MODEL: ${DEFAULT_LLM_MODEL:gpt-4o-2024-11-20} diff --git a/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_10_get_files_list_folder.yaml b/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_10_get_files_list_folder.yaml index 01c4641ee..ba4be5f84 100644 --- a/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_10_get_files_list_folder.yaml +++ b/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_10_get_files_list_folder.yaml @@ -3,14 +3,14 @@ priority: High description: | Verify the SharePoint 'get_files_list' tool filters files by folder path. - Objective: Test folder-specific file listing functionality + Objective: Test folder-specific file listing using server-relative path (path only pattern) Expected Behavior: - Tool executes successfully without errors - Returns only files from specified folder - - Supports both relative and absolute folder paths + - Supports server-relative paths (/sites/{SiteName}/...) - Empty folder returns empty array (not error) - - Form name filtering works correctly + - Works with folder_name only (no form_name required) toolkits: - id: ${SHAREPOINT_TOOLKIT_ID} @@ -19,7 +19,7 @@ toolkits: state: folder_name: type: str - value: ${TEST_FOLDER} + value: ${SERVER_RELATIVE_PATH} limit_files: type: int value: 20 diff --git a/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_13_get_files_list_limit.yaml b/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_13_get_files_list_limit.yaml new file mode 100644 index 000000000..61c420706 --- /dev/null +++ b/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_13_get_files_list_limit.yaml @@ -0,0 +1,133 @@ +name: "SP13 - get_files_list: Limit parameter validation" +priority: High +description: | + Verify the SharePoint 'get_files_list' tool respects the limit_files parameter. + + Objective: Test that the limit parameter controls the maximum number of files returned + + Expected Behavior: + - Tool executes successfully without errors + - Returns exactly N files where N = min(limit_files, total_files_available) + - Files are not duplicated + - Limit values of 1, 5, and 10 are properly respected + +toolkits: + - id: ${SHAREPOINT_TOOLKIT_ID} + name: ${SHAREPOINT_TOOLKIT_NAME} + +state: + folder_name: + type: str + value: ${SERVER_RELATIVE_PATH} + limit_small: + type: int + value: 2 + limit_medium: + type: int + value: 5 + files_result_small: + type: dict + files_result_medium: + type: dict + test_results: + type: dict + +entry_point: invoke_get_files_list_small + +nodes: + - id: invoke_get_files_list_small + type: toolkit + tool: get_files_list + toolkit_name: ${SHAREPOINT_TOOLKIT_NAME} + input: + - folder_name + - limit_small + input_mapping: + folder_name: + type: variable + value: folder_name + limit_files: + type: variable + value: limit_small + output: + - files_result_small + structured_output: true + transition: invoke_get_files_list_medium + + - id: invoke_get_files_list_medium + type: toolkit + tool: get_files_list + toolkit_name: ${SHAREPOINT_TOOLKIT_NAME} + input: + - folder_name + - limit_medium + input_mapping: + folder_name: + type: variable + value: folder_name + limit_files: + type: variable + value: limit_medium + output: + - files_result_medium + structured_output: true + transition: validate_result + + - id: validate_result + type: llm + model: ${DEFAULT_LLM_MODEL} + input: + - files_result_small + - files_result_medium + - limit_small + - limit_medium + input_mapping: + system: + type: fixed + value: "You are a quality assurance validator." + task: + type: fstring + value: | + Analyze the 'get_files_list' tool execution with different limit values. + + Test 1 - Limit={limit_small}: + Result: {files_result_small} + + Test 2 - Limit={limit_medium}: + Result: {files_result_medium} + + Expected behavior: + - Both tool executions succeed without errors + - Both tests MUST return files (test data should exist in folder) + - Result 1 contains exactly {limit_small} files + - Result 2 contains exactly {limit_medium} files + - Result 2 should have >= files than Result 1 (if enough files exist) + - No file duplicates within each result + - Each file has Name, Path, Created, Modified fields + + Validation rules: + 1. Check result structure is valid (not error/exception) + 2. Verify both results are non-empty (if empty, test fails - missing test data) + 3. Count files in each result array + 4. Verify count <= limit for each test + 5. Verify Result 2 count >= Result 1 count (or both are same if folder has few files) + 6. Check for duplicate file names within each result + + return a json object with: + {{ + "test_passed": true/false, + "summary": "brief description of outcome", + "error": "error details if failed, null if passed", + "res_1": {files_result_small}, + "res_2": {files_result_medium} + }} + + return **only** the json object. no markdown formatting, no additional text. + chat_history: + type: fixed + value: [] + output: + - test_results + structured_output_dict: + test_results: dict + transition: END diff --git a/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_14_get_files_list_include_extensions.yaml b/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_14_get_files_list_include_extensions.yaml new file mode 100644 index 000000000..88448dd18 --- /dev/null +++ b/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_14_get_files_list_include_extensions.yaml @@ -0,0 +1,142 @@ +name: "SP14 - get_files_list: Include specific file extensions" +priority: High +description: | + Verify the SharePoint 'get_files_list' tool filters files by extension using include_extensions parameter. + + Objective: Test that only files with specified extensions are returned + + Expected Behavior: + - Tool executes successfully without errors + - Returns only files with extensions matching include_extensions list + - Extensions are matched case-insensitively + - Accepts both 'pdf' and '.pdf' formats + - Must return at least one file (empty results indicate missing test data) + +toolkits: + - id: ${SHAREPOINT_TOOLKIT_ID} + name: ${SHAREPOINT_TOOLKIT_NAME} + +state: + folder_name: + type: str + value: ${SERVER_RELATIVE_PATH} + limit_files: + type: int + value: 50 + include_txt: + type: list + value: ["txt"] + include_pdf_doc: + type: list + value: [".pdf", ".docx"] + files_result_txt: + type: dict + files_result_pdf_doc: + type: dict + test_results: + type: dict + +entry_point: invoke_get_files_txt + +nodes: + - id: invoke_get_files_txt + type: toolkit + tool: get_files_list + toolkit_name: ${SHAREPOINT_TOOLKIT_NAME} + input: + - folder_name + - limit_files + - include_txt + input_mapping: + folder_name: + type: variable + value: folder_name + limit_files: + type: variable + value: limit_files + include_extensions: + type: variable + value: include_txt + output: + - files_result_txt + structured_output: true + transition: invoke_get_files_pdf_doc + + - id: invoke_get_files_pdf_doc + type: toolkit + tool: get_files_list + toolkit_name: ${SHAREPOINT_TOOLKIT_NAME} + input: + - folder_name + - limit_files + - include_pdf_doc + input_mapping: + folder_name: + type: variable + value: folder_name + limit_files: + type: variable + value: limit_files + include_extensions: + type: variable + value: include_pdf_doc + output: + - files_result_pdf_doc + structured_output: true + transition: validate_result + + - id: validate_result + type: llm + model: ${DEFAULT_LLM_MODEL} + input: + - files_result_txt + - files_result_pdf_doc + - include_txt + - include_pdf_doc + input_mapping: + system: + type: fixed + value: "You are a quality assurance validator." + task: + type: fstring + value: | + Analyze the 'get_files_list' tool execution with include_extensions filter. + + Test 1 - Include extensions: {include_txt} + Result: {files_result_txt} + + Test 2 - Include extensions: {include_pdf_doc} + Result: {files_result_pdf_doc} + + Expected behavior: + - Both tool executions succeed without errors + - Test 1 returns only .txt files + - Test 2 returns only .pdf and .docx files (BOTH extensions must be present) + - Extensions are matched case-insensitively + - Both tests MUST return at least one file (test data should exist) + + Validation rules: + 1. Check result structure is valid (not error/exception) + 2. Verify both results are non-empty (if empty, test fails - missing test data) + 3. Extract file extensions from each result (from Name or Path field) + 4. Verify Test 1 files all end with .txt (case-insensitive) + 5. Verify Test 2 files all end with .pdf or .docx (case-insensitive) + 6. Verify Test 2 contains BOTH .pdf AND .docx files (not just one type) + 7. No files with other extensions should be included + + return a json object with: + {{ + "test_passed": true/false, + "summary": "brief description of outcome", + "error": "error details if failed, null if passed" + }} + + return **only** the json object. no markdown formatting, no additional text. + chat_history: + type: fixed + value: [] + output: + - test_results + structured_output_dict: + test_results: dict + transition: END diff --git a/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_15_get_files_list_skip_extensions.yaml b/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_15_get_files_list_skip_extensions.yaml new file mode 100644 index 000000000..2632279c9 --- /dev/null +++ b/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_15_get_files_list_skip_extensions.yaml @@ -0,0 +1,173 @@ +name: "SP15 - get_files_list: Skip specific file extensions" +priority: High +description: | + Verify the SharePoint 'get_files_list' tool filters out files using skip_extensions parameter. + + Objective: Test that files with specified extensions are excluded from results + + Expected Behavior: + - Tool executes successfully without errors + - Returns all files EXCEPT those with extensions in skip_extensions list + - Extensions are matched case-insensitively + - Accepts both 'exe' and '.exe' formats + - All files returned if skip list doesn't match any files + +toolkits: + - id: ${SHAREPOINT_TOOLKIT_ID} + name: ${SHAREPOINT_TOOLKIT_NAME} + +state: + folder_name: + type: str + value: ${SERVER_RELATIVE_PATH} + limit_files: + type: int + value: 50 + skip_txt: + type: list + value: ["txt"] + skip_multiple: + type: list + value: [".pdf", ".docx", ".png"] + files_result_skip_txt: + type: dict + files_result_skip_multiple: + type: dict + files_result_all: + type: dict + test_results: + type: dict + +entry_point: invoke_get_files_all + +nodes: + - id: invoke_get_files_all + type: toolkit + tool: get_files_list + toolkit_name: ${SHAREPOINT_TOOLKIT_NAME} + input: + - folder_name + - limit_files + input_mapping: + folder_name: + type: variable + value: folder_name + limit_files: + type: variable + value: limit_files + output: + - files_result_all + structured_output: true + transition: invoke_get_files_skip_txt + + - id: invoke_get_files_skip_txt + type: toolkit + tool: get_files_list + toolkit_name: ${SHAREPOINT_TOOLKIT_NAME} + input: + - folder_name + - limit_files + - skip_txt + input_mapping: + folder_name: + type: variable + value: folder_name + limit_files: + type: variable + value: limit_files + skip_extensions: + type: variable + value: skip_txt + output: + - files_result_skip_txt + structured_output: true + transition: invoke_get_files_skip_multiple + + - id: invoke_get_files_skip_multiple + type: toolkit + tool: get_files_list + toolkit_name: ${SHAREPOINT_TOOLKIT_NAME} + input: + - folder_name + - limit_files + - skip_multiple + input_mapping: + folder_name: + type: variable + value: folder_name + limit_files: + type: variable + value: limit_files + skip_extensions: + type: variable + value: skip_multiple + output: + - files_result_skip_multiple + structured_output: true + transition: validate_result + + - id: validate_result + type: llm + model: ${DEFAULT_LLM_MODEL} + input: + - files_result_all + - files_result_skip_txt + - files_result_skip_multiple + - skip_txt + - skip_multiple + input_mapping: + system: + type: fixed + value: "You are a quality assurance validator." + task: + type: fstring + value: | + Analyze the 'get_files_list' tool execution with skip_extensions filter. + + Baseline - No filter: + Result: {files_result_all} + + Test 1 - Skip extensions: {skip_txt} + Result: {files_result_skip_txt} + + Test 2 - Skip extensions: {skip_multiple} + Result: {files_result_skip_multiple} + + Expected behavior: + - All tool executions succeed without errors + - Baseline MUST return files (test data should exist) + - Test 1 returns all files EXCEPT .txt files + - Test 2 returns all files EXCEPT .pdf, .docx, .png files (NONE of these should be present) + - Extensions are matched case-insensitively + - Filtered results should have <= files than baseline + + Validation rules: + 1. Check all result structures are valid (not error/exception) + 2. Verify baseline result is non-empty (if empty, test fails - missing test data) + 3. Extract file extensions from each result + 4. Verify Test 1 has NO .txt files (zero .txt files) + 5. Verify Test 2 has NO .pdf files (zero .pdf files) + 6. Verify Test 2 has NO .docx files (zero .docx files) + 7. Verify Test 2 has NO .png files (zero .png files) + 8. Verify Test 1 count <= Baseline count + 9. Verify Test 2 count <= Baseline count + 10. Calculate how many files were filtered in each test + + return a json object with: + {{ + "test_passed": true/false, + "summary": "brief description of outcome", + "error": "error details if failed, null if passed", + "Result_1": {files_result_skip_txt}, + "Result_2": {files_result_skip_multiple} + }} + + return **only** the json object. no markdown formatting, no additional text. + chat_history: + type: fixed + value: [] + output: + - test_results + structured_output_dict: + test_results: dict + transition: END diff --git a/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_16_get_files_list_form_name.yaml b/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_16_get_files_list_form_name.yaml new file mode 100644 index 000000000..f6349a20d --- /dev/null +++ b/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_16_get_files_list_form_name.yaml @@ -0,0 +1,99 @@ +name: "SP16 - get_files_list: Document library (form_name) filtering" +priority: High +description: | + Verify the SharePoint 'get_files_list' tool filters files by document library using form_name parameter. + + Objective: Test that form_name restricts results to a specific document library + + Expected Behavior: + - Tool executes successfully without errors + - Returns only files from the specified document library + - Works with library names like 'Shared Documents', 'private_docs', etc. + - Must return at least one file (empty results indicate missing test data or incorrect library name) + +toolkits: + - id: ${SHAREPOINT_TOOLKIT_ID} + name: ${SHAREPOINT_TOOLKIT_NAME} + +state: + form_name_shared: + type: str + value: "Shared Documents" + limit_files: + type: int + value: 50 + files_result_shared: + type: dict + test_results: + type: dict + +entry_point: invoke_get_files_shared + +nodes: + - id: invoke_get_files_shared + type: toolkit + tool: get_files_list + toolkit_name: ${SHAREPOINT_TOOLKIT_NAME} + input: + - form_name_shared + - limit_files + input_mapping: + form_name: + type: variable + value: form_name_shared + limit_files: + type: variable + value: limit_files + output: + - files_result_shared + structured_output: true + transition: validate_result + + - id: validate_result + type: llm + model: ${DEFAULT_LLM_MODEL} + input: + - files_result_shared + - form_name_shared + input_mapping: + system: + type: fixed + value: "You are a quality assurance validator." + task: + type: fstring + value: | + Analyze the 'get_files_list' tool execution with form_name filter. + + Test - Form name: '{form_name_shared}' + Result: {files_result_shared} + + Expected behavior: + - Tool executes successfully without errors + - Returns array of file objects from the specified document library + - File paths should indicate they are from '{form_name_shared}' library + - Each file has Name, Path, Created, Modified fields + - Must return at least one file (test data should exist in library) + + Validation rules: + 1. Check result structure is valid (not error/exception) + 2. Verify result is a non-empty array (if empty, test fails - missing test data) + 3. Check each file has required fields (Name, Path, Created, Modified) + 4. Verify Path contains the library name or is valid server-relative path + 5. No errors or exceptions in response + + return a json object with: + {{ + "test_passed": true/false, + "summary": "brief description of outcome", + "error": "error details if failed, null if passed" + }} + + return **only** the json object. no markdown formatting, no additional text. + chat_history: + type: fixed + value: [] + output: + - test_results + structured_output_dict: + test_results: dict + transition: END diff --git a/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_17_get_files_list_combined_filters.yaml b/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_17_get_files_list_combined_filters.yaml new file mode 100644 index 000000000..cabd94f18 --- /dev/null +++ b/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_17_get_files_list_combined_filters.yaml @@ -0,0 +1,163 @@ +name: "SP17 - get_files_list: Combined filters (form_name + folder + extensions)" +priority: Critical +description: | + Verify the SharePoint 'get_files_list' tool correctly applies multiple filters simultaneously. + + Objective: Test that form_name, folder_name, include_extensions, and skip_extensions work together + + Expected Behavior: + - Tool executes successfully without errors + - All filters are applied simultaneously (AND logic) + - Results match ALL filter criteria + - Order of filters doesn't matter + - Must return at least one file in baseline and filtered results (test data should exist) + +toolkits: + - id: ${SHAREPOINT_TOOLKIT_ID} + name: ${SHAREPOINT_TOOLKIT_NAME} + +state: + form_name: + type: str + value: "Shared Documents" + folder_name: + type: str + value: "DO_NOT_DELETE_AlitaTestFiles" + include_extensions: + type: list + value: ["txt", "pdf", "docx"] + skip_extensions: + type: list + value: ["pdf"] + limit_files: + type: int + value: 50 + files_result_baseline: + type: dict + files_result_filtered: + type: dict + test_results: + type: dict + +entry_point: invoke_baseline + +nodes: + - id: invoke_baseline + type: toolkit + tool: get_files_list + toolkit_name: ${SHAREPOINT_TOOLKIT_NAME} + input: + - form_name + - folder_name + - limit_files + input_mapping: + form_name: + type: variable + value: form_name + folder_name: + type: variable + value: folder_name + limit_files: + type: variable + value: limit_files + output: + - files_result_baseline + structured_output: true + transition: invoke_combined_filters + + - id: invoke_combined_filters + type: toolkit + tool: get_files_list + toolkit_name: ${SHAREPOINT_TOOLKIT_NAME} + input: + - form_name + - folder_name + - include_extensions + - skip_extensions + - limit_files + input_mapping: + form_name: + type: variable + value: form_name + folder_name: + type: variable + value: folder_name + include_extensions: + type: variable + value: include_extensions + skip_extensions: + type: variable + value: skip_extensions + limit_files: + type: variable + value: limit_files + output: + - files_result_filtered + structured_output: true + transition: validate_result + + - id: validate_result + type: llm + model: ${DEFAULT_LLM_MODEL} + input: + - files_result_baseline + - files_result_filtered + - form_name + - folder_name + - include_extensions + - skip_extensions + input_mapping: + system: + type: fixed + value: "You are a quality assurance validator." + task: + type: fstring + value: | + Analyze the 'get_files_list' tool execution with combined filters. + + Baseline - Form + Folder only: + Form: '{form_name}' + Folder: '{folder_name}' + Result: {files_result_baseline} + + Test - All filters combined: + Form: '{form_name}' + Folder: '{folder_name}' + Include extensions: {include_extensions} + Skip extensions: {skip_extensions} + Result: {files_result_filtered} + + Expected behavior: + - Both tool executions succeed without errors + - Baseline must return files (test data should exist) + - Combined filter should return only .txt and .docx files (pdf is excluded by skip_extensions) + - Filtered result should have <= files than baseline + - All files are from '{form_name}' library and '{folder_name}' folder + - No .pdf files in filtered result (even though it's in include_extensions, it's in skip_extensions) + + Validation rules: + 1. Check both result structures are valid (not error/exception) + 2. Verify baseline result is non-empty (if empty, test fails - missing test data) + 3. Extract file extensions from filtered result + 4. Verify filtered result contains ONLY .txt or .docx files (case-insensitive) + 5. Verify NO .pdf files in filtered result + 6. Verify filtered count <= baseline count + 7. Verify paths contain the folder name + 8. Calculate filtering effectiveness + + return a json object with: + {{ + "test_passed": true/false, + "summary": "brief description of outcome", + "error": "error details if failed, null if passed" + }} + + return **only** the json object. no markdown formatting, no additional text. + chat_history: + type: fixed + value: [] + output: + - test_results + structured_output_dict: + test_results: dict + transition: END diff --git a/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_18_get_files_list_include_exclude_conflict.yaml b/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_18_get_files_list_include_exclude_conflict.yaml new file mode 100644 index 000000000..d97654eb9 --- /dev/null +++ b/.alita/tests/test_pipelines/suites/sharepoint/tests/test_case_18_get_files_list_include_exclude_conflict.yaml @@ -0,0 +1,202 @@ +name: "SP18 - get_files_list: Include/Exclude conflict (edge case)" +priority: High +description: | + Verify the SharePoint 'get_files_list' tool handles conflicting include/skip extensions correctly. + + Objective: Test precedence when extensions appear in both include and skip lists + + Expected Behavior: + - Tool executes successfully without errors + - Test 1 (Partial Conflict): When extension is in BOTH include and skip, skip takes precedence + - Test 2 (Full Conflict): When skip == include (identical lists), skip wins for all extensions + - Demonstrates that exclusion rules always override inclusion rules + - Must return files (baseline should have test data) + +toolkits: + - id: ${SHAREPOINT_TOOLKIT_ID} + name: ${SHAREPOINT_TOOLKIT_NAME} + +state: + folder_name: + type: str + value: ${SERVER_RELATIVE_PATH} + limit_files: + type: int + value: 50 + include_all: + type: list + value: ["txt", "pdf", "docx", "png"] + skip_txt: + type: list + value: ["txt"] + include_txt_pdf: + type: list + value: ["txt", "pdf"] + skip_txt_conflict: + type: list + value: ["txt"] + include_same: + type: list + value: ["txt", "pdf"] + skip_same: + type: list + value: ["txt", "pdf"] + files_result_baseline: + type: dict + files_result_conflict: + type: dict + files_result_equal_conflict: + type: dict + test_results: + type: dict + +entry_point: invoke_baseline + +nodes: + - id: invoke_baseline + type: toolkit + tool: get_files_list + toolkit_name: ${SHAREPOINT_TOOLKIT_NAME} + input: + - folder_name + - limit_files + - include_all + input_mapping: + folder_name: + type: variable + value: folder_name + limit_files: + type: variable + value: limit_files + include_extensions: + type: variable + value: include_all + output: + - files_result_baseline + structured_output: true + transition: invoke_conflict_test + + - id: invoke_conflict_test + type: toolkit + tool: get_files_list + toolkit_name: ${SHAREPOINT_TOOLKIT_NAME} + input: + - folder_name + - limit_files + - include_txt_pdf + - skip_txt_conflict + input_mapping: + folder_name: + type: variable + value: folder_name + limit_files: + type: variable + value: limit_files + include_extensions: + type: variable + value: include_txt_pdf + skip_extensions: + type: variable + value: skip_txt_conflict + output: + - files_result_conflict + structured_output: true + transition: invoke_equal_conflict_test + + - id: invoke_equal_conflict_test + type: toolkit + tool: get_files_list + toolkit_name: ${SHAREPOINT_TOOLKIT_NAME} + input: + - folder_name + - limit_files + - include_same + - skip_same + input_mapping: + folder_name: + type: variable + value: folder_name + limit_files: + type: variable + value: limit_files + include_extensions: + type: variable + value: include_same + skip_extensions: + type: variable + value: skip_same + output: + - files_result_equal_conflict + structured_output: true + transition: validate_result + + - id: validate_result + type: llm + model: ${DEFAULT_LLM_MODEL} + input: + - files_result_baseline + - files_result_conflict + - files_result_equal_conflict + - include_txt_pdf + - skip_txt_conflict + - include_same + - skip_same + input_mapping: + system: + type: fixed + value: "You are a quality assurance validator." + task: + type: fstring + value: | + Analyze the 'get_files_list' tool execution with conflicting include/skip extensions. + + Baseline - Include: ['txt', 'pdf', 'docx', 'png'] + Result: {files_result_baseline} + + Test 1 (Partial Conflict) - Include: {include_txt_pdf}, Skip: {skip_txt_conflict} + Result: {files_result_conflict} + + Test 2 (Full Conflict) - Include: {include_same}, Skip: {skip_same} + Result: {files_result_equal_conflict} + + Expected behavior: + - All tool executions succeed without errors + - Baseline MUST return files (test data should exist) + + Test 1 (Partial Conflict): + - Includes 'txt' and 'pdf' BUT also skips 'txt' + - Skip should take precedence: result should contain ONLY .pdf files (no .txt) + - Proves exclusion rules override inclusion rules for overlapping extensions + + Test 2 (Full Conflict - skip == include): + - Include and skip lists are IDENTICAL: ['txt', 'pdf'] + - Skip should take precedence for ALL extensions + - Result should contain NO .txt or .pdf files + - Should only return files with other extensions (docx, png, etc.) if they exist + - This is the ultimate conflict test: when skip == include, skip wins completely + + Validation rules: + 1. Check all result structures are valid (not error/exception) + 2. Verify baseline result is non-empty (if empty, test fails - missing test data) + 3. Verify Test 1 has NO .txt files (zero .txt files) + 4. Verify Test 1 has only .pdf files (or empty if no .pdf files exist) + 5. Verify Test 2 is empty (has NO .txt files (zero .txt files), has NO .pdf files (zero .pdf files)) + 6. Confirm skip_extensions took precedence over include_extensions in both tests + + return a json object with: + {{ + "test_passed": true/false, + "summary": "brief description of outcome", + "error": "error details if failed, null if passed", + "result_sameple_conflict": {files_result_equal_conflict}, + }} + + return **only** the json object. no markdown formatting, no additional text. + chat_history: + type: fixed + value: [] + output: + - test_results + structured_output_dict: + test_results: dict + transition: END diff --git a/.github/workflows/execute-tests-on-pr.yml b/.github/workflows/execute-tests-on-pr.yml index 071c2bd03..8ea7a2c45 100644 --- a/.github/workflows/execute-tests-on-pr.yml +++ b/.github/workflows/execute-tests-on-pr.yml @@ -188,6 +188,11 @@ jobs: env: PYTHONDONTWRITEBYTECODE: "1" PYTHONUNBUFFERED: "1" + DEFAULT_LLM_MODEL_FOR_CODE_ANALYSIS: ${{ secrets.DEFAULT_LLM_MODEL_FOR_CODE_ANALYSIS }} + ELITEA_DEPLOYMENT_URL: ${{ secrets.DEPLOYMENT_URL_STAGE }} + ELITEA_TOKEN: ${{ secrets.ALITA_API_KEY_STAGE }} + ELITEA_PROJECT_ID: ${{ secrets.PROJECT_ID_STAGE }} + steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/.github/workflows/test-runner-reusable.yml b/.github/workflows/test-runner-reusable.yml index 3aa6ac7ba..d89f72693 100644 --- a/.github/workflows/test-runner-reusable.yml +++ b/.github/workflows/test-runner-reusable.yml @@ -89,7 +89,7 @@ jobs: PYTHONPATH: /app PYTHONDONTWRITEBYTECODE: "1" PYTHONUNBUFFERED: "1" - PGVECTOR_CONNECTION_STRING: postgresql://postgres:yourpassword@pgvector-db:5432/postgres + PGVECTOR_CONNECTION_STRING: postgresql+psycopg://postgres:yourpassword@pgvector-db:5432/postgres services: pgvector-db: diff --git a/tests/runtime/langchain/document_loaders/test_alita_image_loader.py b/tests/runtime/langchain/document_loaders/test_alita_image_loader.py index 9603369c4..1a7c29ebf 100644 --- a/tests/runtime/langchain/document_loaders/test_alita_image_loader.py +++ b/tests/runtime/langchain/document_loaders/test_alita_image_loader.py @@ -27,12 +27,6 @@ _LOADER_NAME = "AlitaImageLoader" _SKIP = { - ("alita_screenshot_jpeg", 1), - ("image_regular", 1), - ("image_regular", 2), - ("image_regular", 3), - ("several_in_one_png", 1), - ("snail_bmp", 1), ("wrench_svg", 1), } diff --git a/tests/runtime/langchain/document_loaders/test_alita_json_loader.py b/tests/runtime/langchain/document_loaders/test_alita_json_loader.py index c62d334db..bcae1b71a 100644 --- a/tests/runtime/langchain/document_loaders/test_alita_json_loader.py +++ b/tests/runtime/langchain/document_loaders/test_alita_json_loader.py @@ -35,12 +35,4 @@ def test_loader( file_path: Path, baseline_path: Path, ) -> None: - _SKIP = { - ("json_large", 0), - ("json_large", 1), - ("json_large", 2), - ("json_nested", 1), - } - if (input_name, config_index) in _SKIP: - pytest.skip(f"{input_name} config{config_index}: known failure — pending fix") run_loader_assert(_LOADER_NAME, tmp_path, input_name, config_index, config, file_path, baseline_path) diff --git a/tests/runtime/langchain/document_loaders/test_alita_markdown_loader.py b/tests/runtime/langchain/document_loaders/test_alita_markdown_loader.py index bebb07ea1..6457bf89c 100644 --- a/tests/runtime/langchain/document_loaders/test_alita_markdown_loader.py +++ b/tests/runtime/langchain/document_loaders/test_alita_markdown_loader.py @@ -19,13 +19,4 @@ def test_loader( file_path: Path, baseline_path: Path, ) -> None: - _SKIP = { - ("markdown_large", 0), - ("markdown_large", 1), - ("markdown_large", 2), - ("markdown_code_blocks", 3), - ("markdown_code_blocks", 1) - } - if (input_name, config_index) in _SKIP: - pytest.skip(f"{input_name} config{config_index}: known failure — pending fix") run_loader_assert(_LOADER_NAME, tmp_path, input_name, config_index, config, file_path, baseline_path) diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_empty.json b/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_empty.json index 15986f937..3fb0346a5 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_empty.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_empty.json @@ -2,7 +2,7 @@ "tags": ["loader:csv", "content:empty", "edge:empty-input"], "file_path": "../files/csv_empty.csv", "configs": [ - {}, - {"max_tokens": 1024} + {"encoding": "utf-8", "raw_content": true, "cleanse": false}, + {"encoding": "utf-8", "raw_content": true, "cleanse": false} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_large.json b/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_large.json index e33b9db45..c65a0c70c 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_large.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_large.json @@ -2,9 +2,9 @@ "tags": ["loader:csv", "content:large", "feature:chunking", "performance"], "file_path": "../files/csv_large.csv", "configs": [ - {}, - {"max_tokens": 50}, - {"max_tokens": 256}, - {"max_tokens": 1024} + {"encoding": "utf-8", "raw_content": true, "cleanse": false}, + {"encoding": "utf-8", "raw_content": true, "cleanse": false}, + {"encoding": "utf-8", "raw_content": true, "cleanse": false}, + {"encoding": "utf-8", "raw_content": true, "cleanse": false} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_latin1.json b/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_latin1.json index 214594d3c..411e4e64d 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_latin1.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_latin1.json @@ -2,9 +2,9 @@ "tags": ["loader:csv", "encoding:latin1"], "file_path": "../files/csv_latin1.csv", "configs": [ - {}, - {"max_tokens": 512}, - {"raw_content": true}, - {"raw_content": false, "max_tokens": 256} + {"encoding": "utf-8", "raw_content": true, "cleanse": false}, + {"encoding": "utf-8", "raw_content": true, "cleanse": false}, + {"encoding": "utf-8", "raw_content": true, "cleanse": false}, + {"encoding": "utf-8", "raw_content": true, "cleanse": false} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_raw_content.json b/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_raw_content.json index 35eaad091..087ef233f 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_raw_content.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_raw_content.json @@ -2,9 +2,9 @@ "tags": ["loader:csv", "feature:raw_content"], "file_path": "../files/csv_simple.csv", "configs": [ - {}, - {"max_tokens": 1024}, - {"raw_content": true}, - {"raw_content": true, "max_tokens": 512} + {"encoding": "utf-8", "raw_content": true, "cleanse": false}, + {"encoding": "utf-8", "raw_content": true, "cleanse": false}, + {"encoding": "utf-8", "raw_content": true, "cleanse": false}, + {"encoding": "utf-8", "raw_content": true, "cleanse": false} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_simple.json b/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_simple.json index 116cc8a03..42021f58a 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_simple.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_simple.json @@ -2,7 +2,7 @@ "tags": ["loader:csv", "content:simple", "feature:chunking"], "file_path": "../files/csv_simple.csv", "configs": [ - {}, - {"max_tokens": 1024} + {"encoding": "utf-8", "raw_content": true, "cleanse": false}, + {"encoding": "utf-8", "raw_content": true, "cleanse": false} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_special.json b/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_special.json index eb7cc4ae7..85f0c43f7 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_special.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_special.json @@ -2,7 +2,7 @@ "tags": ["loader:csv", "content:special-characters", "edge:special-chars"], "file_path": "../files/csv_special.csv", "configs": [ - {}, - {"max_tokens": 256} + {"encoding": "utf-8", "raw_content": true, "cleanse": false}, + {"encoding": "utf-8", "raw_content": true, "cleanse": false} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_unicode.json b/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_unicode.json index abe221baa..6b8ffbfbf 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_unicode.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaCSVLoader/input/csv_unicode.json @@ -2,7 +2,7 @@ "tags": ["loader:csv", "content:unicode", "edge:encoding"], "file_path": "../files/csv_unicode.csv", "configs": [ - {}, - {"max_tokens": 1024} + {"encoding": "utf-8", "raw_content": true, "cleanse": false}, + {"encoding": "utf-8", "raw_content": true, "cleanse": false} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/alita_screenshot_jpeg.json b/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/alita_screenshot_jpeg.json index 730dc2cfc..eef1d74ff 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/alita_screenshot_jpeg.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/alita_screenshot_jpeg.json @@ -4,14 +4,14 @@ "configs": [ { "_name": "ocr_only", - "use_llm": false, - "max_tokens": 2048 + "_use_llm": false, + "_max_tokens": 2048 }, { "_name": "llm_default_prompt", - "use_llm": true, - "prompt_default": true, - "max_tokens": 2048 + "_use_llm": true, + "_prompt_default": true, + "_max_tokens": 2048 } ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/image_regular.json b/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/image_regular.json index 5dc318312..795ff3204 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/image_regular.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/image_regular.json @@ -3,26 +3,26 @@ "tags": ["loader_image", "feature_llm", "feature_multimodal"], "configs": [ { - "use_llm": false, - "max_tokens": 2048 + "_use_llm": false, + "_max_tokens": 2048 }, { "_name": "llm_default_prompt_high_tokens", - "use_llm": true, - "prompt_default": true, - "max_tokens": 2048 + "_use_llm": true, + "_prompt_default": true, + "_max_tokens": 2048 }, { "_name": "llm_custom_prompt_json_format", - "use_llm": true, + "_use_llm": true, "prompt": "Describe this image in JSON format with keys: subject, colors, text_content, estimated_age_group, brand_visible", - "max_tokens": 2048 + "_max_tokens": 2048 }, { "_name": "llm_default_prompt_minimal_tokens", - "use_llm": true, + "_use_llm": true, "prompt": "Provide description (just general descriptions) of this image in around 1000 tokens.", - "max_tokens": 256 + "_max_tokens": 256 } ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/several_in_one_png.json b/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/several_in_one_png.json index 196274064..8fd5dcf4e 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/several_in_one_png.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/several_in_one_png.json @@ -4,14 +4,14 @@ "configs": [ { "_name": "ocr_only", - "use_llm": false, - "max_tokens": 2048 + "_use_llm": false, + "_max_tokens": 2048 }, { "_name": "llm_default_prompt", - "use_llm": true, - "prompt_default": true, - "max_tokens": 2048 + "_use_llm": true, + "_prompt_default": true, + "_max_tokens": 2048 } ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/snail_bmp.json b/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/snail_bmp.json index 1ca1d73d2..992c1376c 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/snail_bmp.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/snail_bmp.json @@ -4,14 +4,14 @@ "configs": [ { "_name": "ocr_only", - "use_llm": false, - "max_tokens": 2048 + "_use_llm": false, + "_max_tokens": 2048 }, { "_name": "llm_default_prompt", - "use_llm": true, - "prompt_default": true, - "max_tokens": 2048 + "_use_llm": true, + "_prompt_default": true, + "_max_tokens": 2048 } ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/wrench_svg.json b/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/wrench_svg.json index 3f0e80328..5f6639733 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/wrench_svg.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaImageLoader/input/wrench_svg.json @@ -4,14 +4,14 @@ "configs": [ { "_name": "ocr_only", - "use_llm": false, - "max_tokens": 2048 + "_use_llm": false, + "_max_tokens": 2048 }, { "_name": "llm_default_prompt", - "use_llm": true, - "prompt_default": true, - "max_tokens": 2048 + "_use_llm": true, + "_prompt_default": true, + "_max_tokens": 2048 } ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_array.json b/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_array.json index 883cd4c0a..20b3ea365 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_array.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_array.json @@ -2,7 +2,7 @@ "tags": ["loader:json", "content:array", "feature:chunking"], "file_path": "../files/json_array.json", "configs": [ - {}, + {"max_tokens": 512}, {"max_tokens": 256}, {"max_tokens": 1024} ] diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_empty.json b/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_empty.json index a04e55191..bfee20aea 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_empty.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_empty.json @@ -2,7 +2,7 @@ "tags": ["loader:json", "content:empty", "edge:empty-input", "feature:chunking"], "file_path": "../files/json_empty.json", "configs": [ - {}, + {"max_tokens": 512}, {"max_tokens": 256}, {"max_tokens": 1024} ] diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_large.json b/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_large.json index 533d4baf7..ab49a5545 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_large.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_large.json @@ -2,7 +2,7 @@ "tags": ["loader:json", "content:large", "feature:chunking", "performance"], "file_path": "../files/json_large.json", "configs": [ - {}, + {"max_tokens": 512}, {"max_tokens": 256}, {"max_tokens": 1024} ] diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_nested.json b/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_nested.json index 238fd4705..4032250ce 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_nested.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_nested.json @@ -2,7 +2,7 @@ "tags": ["loader:json", "content:nested", "feature:chunking"], "file_path": "../files/json_nested.json", "configs": [ - {}, + {"max_tokens": 512}, {"max_tokens": 256}, {"max_tokens": 1024} ] diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_simple.json b/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_simple.json index c34582f80..25ffa89e9 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_simple.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaJSONLoader/input/json_simple.json @@ -2,7 +2,7 @@ "tags": ["loader:json", "content:simple", "feature:chunking"], "file_path": "../files/json_simple.json", "configs": [ - {}, + {"max_tokens": 512}, {"max_tokens": 256}, {"max_tokens": 1024} ] diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_code_blocks.json b/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_code_blocks.json index 51515c67d..48b9982df 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_code_blocks.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_code_blocks.json @@ -2,9 +2,9 @@ "tags": ["loader:markdown", "feature:code_blocks"], "file_path": "../files/markdown_code_blocks.md", "configs": [ - {}, - {"max_tokens": 256}, - {"max_tokens": 512}, - {"max_tokens": 1024} - ] + {"chunker_config": {"max_tokens": 512, "token_overlap": 15}}, + {"chunker_config": {"max_tokens": 256, "token_overlap": 15}}, + {"chunker_config": {"max_tokens": 512, "token_overlap": 15}}, + {"chunker_config": {"max_tokens": 1024, "token_overlap": 15}} + ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_empty.json b/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_empty.json index 35cb1e09c..4a41b3b15 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_empty.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_empty.json @@ -2,8 +2,8 @@ "tags": ["loader:markdown", "content:empty", "edge:empty-input", "feature:chunking"], "file_path": "../files/markdown_empty.md", "configs": [ - {}, - {"max_tokens": 256}, - {"max_tokens": 1024} + {"chunker_config": {"max_tokens": 512, "token_overlap": 10}}, + {"chunker_config": {"max_tokens": 256, "token_overlap": 10}}, + {"chunker_config": {"max_tokens": 1024, "token_overlap": 10}} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_headers.json b/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_headers.json index 0675a01f4..4cf211695 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_headers.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_headers.json @@ -2,8 +2,8 @@ "tags": ["loader:markdown", "content:headers", "feature:chunking"], "file_path": "../files/markdown_headers.md", "configs": [ - {}, - {"max_tokens": 256}, - {"max_tokens": 1024} + {"chunker_config": {"max_tokens": 512, "token_overlap": 10}}, + {"chunker_config": {"max_tokens": 256, "token_overlap": 10}}, + {"chunker_config": {"max_tokens": 1024, "token_overlap": 10}} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_large.json b/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_large.json index 2552a60ce..bbcaf7e75 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_large.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_large.json @@ -2,8 +2,8 @@ "tags": ["loader:markdown", "content:large", "feature:chunking", "performance"], "file_path": "../files/markdown_large.md", "configs": [ - {}, - {"max_tokens": 256}, - {"max_tokens": 1024} + {"chunker_config": {"max_tokens": 512, "token_overlap": 10}}, + {"chunker_config": {"max_tokens": 256, "token_overlap": 10}}, + {"chunker_config": {"max_tokens": 1024, "token_overlap": 10}} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_nested.json b/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_nested.json index fe569869f..b4beee3f8 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_nested.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_nested.json @@ -2,8 +2,8 @@ "tags": ["loader:markdown", "content:nested", "feature:chunking"], "file_path": "../files/markdown_nested.md", "configs": [ - {}, - {"max_tokens": 256}, - {"max_tokens": 1024} + {"chunker_config": {"max_tokens": 512, "token_overlap": 10}}, + {"chunker_config": {"max_tokens": 256, "token_overlap": 10}}, + {"chunker_config": {"max_tokens": 1024, "token_overlap": 10}} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_simple.json b/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_simple.json index baa5a5fab..9282f59e1 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_simple.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaMarkdownLoader/input/markdown_simple.json @@ -2,8 +2,8 @@ "tags": ["loader:markdown", "content:simple", "feature:chunking"], "file_path": "../files/markdown_simple.md", "configs": [ - {}, - {"max_tokens": 256}, - {"max_tokens": 1024} + {"chunker_config": {"max_tokens": 512, "token_overlap": 10}}, + {"chunker_config": {"max_tokens": 256, "token_overlap": 10}}, + {"chunker_config": {"max_tokens": 1024, "token_overlap": 10}} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_empty.json b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_empty.json index fc0918ab2..e6cf3d0cb 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_empty.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_empty.json @@ -2,6 +2,6 @@ "tags": ["loader:text", "content:empty", "edge:empty-input"], "file_path": "../files/text_empty.txt", "configs": [ - {} + {"autodetect_encoding": true, "max_tokens": 512} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_groovy.json b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_groovy.json index 9bd13359c..787fba7de 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_groovy.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_groovy.json @@ -1,9 +1,9 @@ { "file_path": "../files/text_groovy.groovy", "configs": [ - {}, - {"max_tokens": 256}, - {"max_tokens": 512}, - {"max_tokens": 1024} + {"autodetect_encoding": true, "max_tokens": 512}, + {"autodetect_encoding": true, "max_tokens": 256}, + {"autodetect_encoding": true, "max_tokens": 512}, + {"autodetect_encoding": true, "max_tokens": 1024} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_large.json b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_large.json index eb8cfa671..e6b2b9a63 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_large.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_large.json @@ -2,10 +2,10 @@ "tags": ["loader:text", "content:large", "feature:chunking", "performance"], "file_path": "../files/text_large.txt", "configs": [ - {}, - {"max_tokens": 50}, - {"max_tokens": 100}, - {"max_tokens": 256}, - {"max_tokens": 2000} + {"autodetect_encoding": true, "max_tokens": 512}, + {"autodetect_encoding": true, "max_tokens": 50}, + {"autodetect_encoding": true, "max_tokens": 100}, + {"autodetect_encoding": true, "max_tokens": 256}, + {"autodetect_encoding": true, "max_tokens": 2000} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_latin1.json b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_latin1.json index b0ab65469..b7aa2ad43 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_latin1.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_latin1.json @@ -1,8 +1,8 @@ { "file_path": "../files/text_latin1.txt", "configs": [ - {}, - {"max_tokens": 256}, - {"max_tokens": 512} + {"autodetect_encoding": true, "max_tokens": 512}, + {"autodetect_encoding": true, "max_tokens": 256}, + {"autodetect_encoding": true, "max_tokens": 512} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_markdown.json b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_markdown.json index 315f0740b..7d81cb3c9 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_markdown.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_markdown.json @@ -2,10 +2,10 @@ "tags": ["loader:text", "content:markdown", "feature:chunking", "edge:markdown-in-txt"], "file_path": "../files/text_markdown.txt", "configs": [ - {}, - {"max_tokens": 50}, - {"max_tokens": 100}, - {"max_tokens": 256}, - {"max_tokens": 2000} + {"autodetect_encoding": true, "max_tokens": 512}, + {"autodetect_encoding": true, "max_tokens": 50}, + {"autodetect_encoding": true, "max_tokens": 100}, + {"autodetect_encoding": true, "max_tokens": 256}, + {"autodetect_encoding": true, "max_tokens": 2000} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_simple.json b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_simple.json index 2256c18d0..f303adc1f 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_simple.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_simple.json @@ -2,7 +2,7 @@ "tags": ["loader:text", "content:simple", "feature:chunking"], "file_path": "../files/text_simple.txt", "configs": [ - {}, - {"max_tokens": 1024} + {"autodetect_encoding": true, "max_tokens": 512}, + {"autodetect_encoding": true, "max_tokens": 1024} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_unicode.json b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_unicode.json index e1f853ab8..6113dad36 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_unicode.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_unicode.json @@ -2,7 +2,7 @@ "tags": ["loader:text", "content:unicode", "edge:encoding", "feature:chunking"], "file_path": "../files/text_unicode.txt", "configs": [ - {}, - {"max_tokens": 256} + {"autodetect_encoding": true, "max_tokens": 512}, + {"autodetect_encoding": true, "max_tokens": 256} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_yaml.json b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_yaml.json index 67e327096..6afc0dda1 100644 --- a/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_yaml.json +++ b/tests/runtime/langchain/document_loaders/test_data/AlitaTextLoader/input/text_yaml.json @@ -1,8 +1,8 @@ { "file_path": "../files/text_yaml.yaml", "configs": [ - {}, - {"max_tokens": 256}, - {"max_tokens": 1024} + {"autodetect_encoding": true, "max_tokens": 512}, + {"autodetect_encoding": true, "max_tokens": 256}, + {"autodetect_encoding": true, "max_tokens": 1024} ] } diff --git a/tests/runtime/langchain/document_loaders/test_data/scripts/loader_test_runner.py b/tests/runtime/langchain/document_loaders/test_data/scripts/loader_test_runner.py index 37be34391..f124a7012 100644 --- a/tests/runtime/langchain/document_loaders/test_data/scripts/loader_test_runner.py +++ b/tests/runtime/langchain/document_loaders/test_data/scripts/loader_test_runner.py @@ -78,8 +78,8 @@ def _get_llm_for_tests(): Requires environment variables: - DEFAULT_LLM_MODEL_FOR_CODE_ANALYSIS: Model name - - DEPLOYMENT_URL: Alita deployment URL - - PROJECT_ID: Project ID + - ELITEA_DEPLOYMENT_URL: Alita deployment URL + - ELITEA_PROJECT_ID: Project ID - ELITEA_TOKEN: API key """ model_name = os.getenv('DEFAULT_LLM_MODEL_FOR_CODE_ANALYSIS') @@ -87,12 +87,12 @@ def _get_llm_for_tests(): return None # Check if required client credentials are available - deployment_url = os.getenv('DEPLOYMENT_URL') - project_id = os.getenv('PROJECT_ID') + deployment_url = os.getenv('ELITEA_DEPLOYMENT_URL') + project_id = os.getenv('ELITEA_PROJECT_ID') api_key = os.getenv('ELITEA_TOKEN') if not all([deployment_url, project_id, api_key]): - _rp_log(f"Warning: Cannot create LLM - missing credentials (DEPLOYMENT_URL, PROJECT_ID, or ALITA_API_KEY)") + _rp_log(f"Warning: Cannot create LLM - missing credentials (ELITEA_DEPLOYMENT_URL, ELITEA_PROJECT_ID, or ELITEA_TOKEN)") return None try: @@ -116,47 +116,30 @@ def _get_llm_for_tests(): return None -def _load_documents_with_production_config(file_path: Path, config: Dict[str, Any], llm=None) -> List: - """Load documents using production configuration logic. - - This replicates the logic from process_content_by_type but uses the actual - file path instead of creating a temp file, so metadata contains correct paths. - - Args: - file_path: Path to the file to load - config: Configuration dict from test input - llm: Optional LLM instance for image/multimodal loaders +def _load_documents_directly(loader_name: str, file_path: Path, config: Dict[str, Any], llm=None) -> List: + """Instantiate loader class directly with config as constructor kwargs. + + Config values from input JSON are passed as-is to the loader constructor. + No intermediary transformations through loaders_map or allowed_to_override. + + Keys starting with '_' are stripped (reserved for test metadata like _name). """ - from alita_sdk.runtime.langchain.document_loaders.constants import loaders_map, LoaderProperties - - extension = file_path.suffix.lower() - loader_config = loaders_map.get(extension) - if not loader_config: - raise ValueError(f"No loader found for extension: {extension}") - - loader_cls = loader_config['class'] - loader_kwargs = dict(loader_config.get('kwargs', {})) - - # Apply chunking_config override logic (same as process_content_by_type) - allowed_to_override = loader_config.get('allowed_to_override', loader_kwargs) - - # Start with production defaults from allowed_to_override - loader_kwargs.update(allowed_to_override) - - # Apply user overrides (filtered by allowed_to_override keys) - if config: - for key in set(config.keys()) & set(allowed_to_override.keys()): - loader_kwargs[key] = config[key] - - # Handle LLM and prompt placeholders - if LoaderProperties.LLM.value in loader_kwargs and loader_kwargs.pop(LoaderProperties.LLM.value): - loader_kwargs['llm'] = llm # Use provided LLM instance - if LoaderProperties.PROMPT_DEFAULT.value in loader_kwargs and loader_kwargs.pop(LoaderProperties.PROMPT_DEFAULT.value): - from alita_sdk.tools.utils.content_parser import image_processing_prompt - loader_kwargs[LoaderProperties.PROMPT.value] = image_processing_prompt - - # Instantiate and load - loader = loader_cls(file_path=str(file_path), **loader_kwargs) + import importlib + module = importlib.import_module(f'alita_sdk.runtime.langchain.document_loaders.{loader_name}') + loader_cls = getattr(module, loader_name) + + kwargs = {k: v for k, v in config.items() if not k.startswith('_')} + kwargs['file_path'] = str(file_path) + + if llm is not None: + kwargs['llm'] = llm + # If config specifies default prompt and no explicit prompt provided, inject image_processing_prompt + # This mirrors the logic in content_parser.process_content_by_type for consistency + if config.get('_prompt_default') and 'prompt' not in kwargs: + from alita_sdk.tools.utils.content_parser import image_processing_prompt + kwargs['prompt'] = image_processing_prompt + + loader = loader_cls(**kwargs) return list(loader.load()) @@ -176,11 +159,13 @@ def _load_expected_documents_for_test(baseline_path: Path) -> List: return docs -def _load_actual_documents_for_test(file_path: Path, config: Dict[str, Any], llm=None) -> List: +def _load_actual_documents_for_test(file_path: Path, config: Dict[str, Any], loader_name: str, llm=None) -> List: with rp_step(f"Load actual documents from source {file_path}"): from loader_test_utils import serialize_documents - docs = _load_documents_with_production_config(file_path, config, llm=llm) - llm_info = f" with LLM={type(llm).__name__}" if llm else "" + # Only pass LLM if config explicitly requests it + actual_llm = llm if config.get('_use_llm') else None + docs = _load_documents_directly(loader_name, file_path, config, llm=actual_llm) + llm_info = f" with LLM={type(actual_llm).__name__}" if actual_llm else " (OCR only)" _rp_log(f"Loaded {len(docs)} actual document(s) with config: {config or {}}{llm_info}") _rp_log( f"Actual documents ({len(docs)})", @@ -325,8 +310,7 @@ def run_single_config_test( return result try: - # Use production configuration logic while preserving file paths - actual_docs = _load_actual_documents_for_test(file_path, config, llm=llm) + actual_docs = _load_actual_documents_for_test(file_path, config, loader_name=loader_name, llm=llm) result.actual_doc_count = len(actual_docs) except Exception as exc: result.error = f"Loader exception: {exc}" @@ -339,23 +323,23 @@ def run_single_config_test( logger.warning(f"Could not save actual output to {actual_output_path}: {exc}") # Only use LLM for comparison if config explicitly uses LLM for loading - # This ensures OCR-only configs (use_llm: false) use similarity-based comparison + # This ensures OCR-only configs (_use_llm: false) use similarity-based comparison comparison_llm = None debug_log_path = None input_context = None - - if config.get("use_llm"): + + if config.get("_use_llm"): # Config uses LLM - use LLM for semantic validation in comparison comparison_llm = llm - + # Construct debug log path for LLM validation comparison if llm is not None: debug_log_path = str(actual_output_path.parent / f"llm_validation_debug_{input_name}_config_{config_index}.jsonl") - + # Extract input context (prompt) from config for better LLM validation if "prompt" in config: input_context = config["prompt"] - elif config.get("prompt_default"): + elif config.get("_prompt_default"): from alita_sdk.tools.utils.content_parser import image_processing_prompt input_context = image_processing_prompt