At index sharepoint

.alita/tests/test_pipelines/configs/sharepoint-index-config.json

@@ -0,0 +1,10 @@
+{
+  "type": "sharepoint",
+  "toolkit_name": "sp-index-test",
+  "selected_tools": [
+    "index_data",
+    "search_index",
+    "list_collections",
+    "remove_index"
+  ]
+}

.alita/tests/test_pipelines/scripts/setup_strategy.py

@@ -333,6 +333,8 @@ def _build_toolkit_settings(self, toolkit_type: str, config: Dict[str, Any], loa
         2. If alita_title is present, merge in stored configuration data from configuration step
         3. All placeholders should already be resolved by resolve_env_value before this method
         4. Fill in any defaults from the Pydantic model if needed
+        5. Programmatically create {toolkit_type}_configuration from environment variables
+        6. If indexing tools are selected, add pgvector_configuration and embedding_model
 
         Args:
             toolkit_type: Type of toolkit (e.g., 'github', 'jira')
@@ -391,7 +393,7 @@ def _build_toolkit_settings(self, toolkit_type: str, config: Dict[str, Any], loa
                 logger.info(f"[LOCAL DEBUG] Merged {key} into toolkit_config (overwrite)")
             logger.info(f"[LOCAL DEBUG] toolkit_config AFTER merge: {toolkit_config}")
 
-        # Fill in defaults from Pydantic model if available
+        # Programmatically populate toolkit_configuration from environment variables using Pydantic model
         try:
             from alita_sdk.configurations import get_class_configurations
 
@@ -409,31 +411,92 @@ def _build_toolkit_settings(self, toolkit_type: str, config: Dict[str, Any], loa
 
         except Exception as e:
             # If configuration class not available, just use what we have
+            # logger.warning(f"[LOCAL DEBUG] Could not auto-populate toolkit_configuration: {e}")
             pass
 
         # Store the configuration in settings
         settings[config_key] = toolkit_config
 
+        # Check if indexing tools are selected
+        selected_tools = config.get('selected_tools', [])
+        logger.info(f"[INDEXING DEBUG] selected_tools from config: {selected_tools}")
+        indexing_tools = {'index_data', 'search_index', 'stepback_search_index', 
+                          'stepback_summary_index', 'remove_index', 'list_collections'}
+        has_indexing_tools = bool(set(selected_tools) & indexing_tools)
+        logger.info(f"[INDEXING DEBUG] has_indexing_tools: {has_indexing_tools}, intersection: {set(selected_tools) & indexing_tools}")
+
+        if has_indexing_tools:
+            logger.info(f"[INDEXING DEBUG] ===== ENTERING INDEXING TOOLS AUTO-CONFIG BLOCK =====")
+            logger.info(f"[LOCAL DEBUG] Indexing tools detected in selected_tools, adding pgvector_configuration and embedding_model")
+
+            # Add pgvector_configuration if not already present
+            if 'pgvector_configuration' not in settings or not settings['pgvector_configuration']:
+                pgvector_config = {}
+
+                # Try to load from environment or stored configuration
+                if 'pgvector_configuration' in config:
+                    pgvector_config = config['pgvector_configuration'].copy() if isinstance(config['pgvector_configuration'], dict) else {}
+
+                # If alita_title is present in pgvector_configuration, merge stored data
+                if 'alita_title' in pgvector_config and pgvector_config['alita_title'] in self._configuration_data:
+                    stored_data = self._configuration_data[pgvector_config['alita_title']]
+                    for key, value in stored_data.items():
+                        if key not in ('alita_title', 'private'):
+                            pgvector_config[key] = value
+
+                # Otherwise, load from PGVECTOR_CONNECTION_STRING environment variable
+                if 'connection_string' not in pgvector_config or not pgvector_config['connection_string']:
+                    logger.info(f"[INDEXING DEBUG] Attempting to load PGVECTOR_CONNECTION_STRING from environment")
+                    conn_str = load_from_env('PGVECTOR_CONNECTION_STRING')
+                    logger.info(f"[INDEXING DEBUG] PGVECTOR_CONNECTION_STRING loaded: {bool(conn_str)}, length: {len(conn_str) if conn_str else 0}")
+                    if conn_str:
+                        from pydantic import SecretStr
+                        pgvector_config['connection_string'] = SecretStr(conn_str)
+                        logger.info(f"[LOCAL DEBUG] Loaded pgvector connection_string from PGVECTOR_CONNECTION_STRING")
+                    else:
+                        logger.warning(f"[INDEXING DEBUG] Failed to load PGVECTOR_CONNECTION_STRING from environment")
+                else:
+                    logger.info(f"[INDEXING DEBUG] connection_string already in pgvector_config")
+
+                settings['pgvector_configuration'] = pgvector_config
+
+            # Add embedding_model if not already present
+            if 'embedding_model' not in settings or not settings['embedding_model']:
+                logger.info(f"[INDEXING DEBUG] embedding_model not in settings, attempting to load...")
+                embedding_model = config.get('embedding_model') or load_from_env('EMBEDDING_MODEL')
+                logger.info(f"[INDEXING DEBUG] Loaded embedding_model value: {embedding_model}")
+                if embedding_model:
+                    settings['embedding_model'] = embedding_model
+                    logger.info(f"[LOCAL DEBUG] Set embedding_model to {embedding_model}")
+                else:
+                    logger.warning(f"[INDEXING DEBUG] Failed to load EMBEDDING_MODEL from config or environment")
+            else:
+                logger.info(f"[INDEXING DEBUG] embedding_model already in settings: {settings.get('embedding_model')}")
+
+        logger.info(f"[INDEXING DEBUG] Final settings keys: {list(settings.keys())}")
+        logger.info(f"[INDEXING DEBUG] Final settings has pgvector_configuration: {'pgvector_configuration' in settings}")
+        logger.info(f"[INDEXING DEBUG] Final settings has embedding_model: {'embedding_model' in settings}")
+
         return settings
 
     def _create_alita_client(self) -> Optional[Any]:
         """Create AlitaClient for toolkit initialization."""
-        from utils_common import load_from_env
+        from utils_common import load_base_url_from_env, load_token_from_env, load_project_id_from_env
 
         try:
             from alita_sdk.runtime.clients.client import AlitaClient
 
-            deployment_url = load_from_env('DEPLOYMENT_URL') or load_from_env('BASE_URL')
-            api_key = load_from_env('API_KEY') or load_from_env('AUTH_TOKEN')
-            project_id = load_from_env('PROJECT_ID')
+            deployment_url = load_base_url_from_env()
+            api_key = load_token_from_env()
+            project_id = load_project_id_from_env()
 
             if not deployment_url or not api_key:
                 return None
 
             return AlitaClient(
                 base_url=deployment_url,
                 auth_token=api_key,
-                project_id=int(project_id) if project_id else 0,
+                project_id=project_id or 0,
             )
         except Exception:
             return None
@@ -485,6 +548,33 @@ def handle_toolkit_create(
             except FileNotFoundError:
                 ctx.log(f"Config file not found: {config['config_file']}", "warning")
 
+        # Auto-detect indexing tools and add pgvector_configuration + embedding_model if needed
+        selected_tools = file_config.get('selected_tools', [])
+        indexing_tools = {'index_data', 'search_index', 'stepback_search_index', 
+                          'stepback_summary_index', 'remove_index', 'list_collections'}
+        has_indexing_tools = bool(set(selected_tools) & indexing_tools)
+
+        if has_indexing_tools:
+            ctx.log(f"[LOCAL] Auto-configuring indexing tools for toolkit", "info")
+
+            # Add pgvector_configuration if not in file_config
+            if 'pgvector_configuration' not in file_config:
+                file_config['pgvector_configuration'] = {}
+
+            # Ensure connection_string is set
+            if 'connection_string' not in file_config['pgvector_configuration']:
+                conn_str = load_from_env('PGVECTOR_CONNECTION_STRING')
+                if conn_str:
+                    file_config['pgvector_configuration']['connection_string'] = conn_str
+                    ctx.log(f"[LOCAL] Auto-added PGVECTOR_CONNECTION_STRING to config", "info")
+
+            # Add embedding_model if not in file_config
+            if 'embedding_model' not in file_config:
+                embedding_model = load_from_env('EMBEDDING_MODEL')
+                if embedding_model:
+                    file_config['embedding_model'] = embedding_model
+                    ctx.log(f"[LOCAL] Auto-added EMBEDDING_MODEL={embedding_model} to config", "info")
+
         # Apply overrides - resolve environment variables in overrides first
         overrides = resolve_env_value(config.get("overrides", {}), ctx.env_vars, env_loader=load_from_env)
         for key, value in overrides.items():

.alita/tests/test_pipelines/suites/index_sharepoint/pipeline.yaml

@@ -0,0 +1,92 @@
+name: index_sharepoint
+description: |
+  SharePoint indexing test suite.
+
+  Tests cover two areas:
+  - index_data called with different path parameter styles (ISP01–ISP05):
+      no path, relative path, server-relative path, form_name only, path+form_name combo.
+      All paths are expected to reach the same set of test files so indexed counts should
+      be comparable across the five tests.
+  - search_index called with progressively sophisticated parameter combinations (ISP06–ISP13),
+      each preceded by a fresh index_data call scoped to the test folder.
+
+env_mapping:
+  SHAREPOINT_SITE_URL: ${SHAREPOINT_SITE_URL:https://5clkvm.sharepoint.com/sites/ALITA}
+  SHAREPOINT_CLIENT_ID: ${SHAREPOINT_CLIENT_ID:84c01a1e-1ebc-40c0-b55a-8b3fae194e41}
+  SHAREPOINT_CLIENT_SECRET: ${SHAREPOINT_CLIENT_SECRET}
+  PGVECTOR_CONNECTION_STRING: ${PGVECTOR_CONNECTION_STRING}
+  EMBEDDING_MODEL: ${EMBEDDING_MODEL:text-embedding-ada-002}
+  TEST_FOLDER: ${TEST_FOLDER:Shared Documents/DO_NOT_DELETE_AlitaTestFiles}
+  SERVER_RELATIVE_PATH: ${SERVER_RELATIVE_PATH:/sites/ALITA/Shared Documents/DO_NOT_DELETE_AlitaTestFiles}
+  TEST_FORM_NAME: ${TEST_FORM_NAME:Shared Documents}
+  TEST_SUBFOLDER: ${TEST_SUBFOLDER:DO_NOT_DELETE_AlitaTestFiles}
+  TEST_FILE_NAME: ${TEST_FILE_NAME:DO_NOT_DELETE_test-document.txt}
+  SEARCH_QUERY: ${SEARCH_QUERY:test document}
+
+setup:
+  - name: Setup SharePoint Configuration
+    type: configuration
+    config:
+      config_type: sharepoint
+      alita_title: ${SHAREPOINT_SECRET_NAME:sharepoint}
+      data:
+        site_url: ${SHAREPOINT_SITE_URL:https://5clkvm.sharepoint.com/sites/ALITA}
+        client_id: ${SHAREPOINT_CLIENT_ID:84c01a1e-1ebc-40c0-b55a-8b3fae194e41}
+        client_secret: ${SHAREPOINT_CLIENT_SECRET}
+
+  - name: Create SharePoint Index Toolkit
+    type: toolkit
+    action: create_or_update
+    config:
+      config_file: ../../configs/sharepoint-index-config.json
+      toolkit_type: sharepoint
+      overrides:
+        sharepoint_configuration:
+          private: false
+          alita_title: ${SHAREPOINT_SECRET_NAME:sharepoint}
+        pgvector_configuration:
+          private: false
+          alita_title: ${PGVECTOR_SECRET_NAME:elitea-pgvector}
+        embedding_model: ${EMBEDDING_MODEL:text-embedding-ada-002}
+      toolkit_name: ${ISP_TOOLKIT_NAME:sp-index-test}
+    save_to_env:
+      - key: ISP_TOOLKIT_ID
+        value: $.id
+      - key: ISP_TOOLKIT_NAME
+        value: $.name
+
+composable_pipelines: []
+
+execution:
+  test_directory: tests
+  order:
+    - test_case_*.yaml
+
+  substitutions:
+    ISP_TOOLKIT_ID: ${ISP_TOOLKIT_ID}
+    ISP_TOOLKIT_NAME: ${ISP_TOOLKIT_NAME}
+    TEST_FOLDER: ${TEST_FOLDER:Shared Documents/DO_NOT_DELETE_AlitaTestFiles}
+    SERVER_RELATIVE_PATH: ${SERVER_RELATIVE_PATH:/sites/ALITA/Shared Documents/DO_NOT_DELETE_AlitaTestFiles}
+    TEST_FORM_NAME: ${TEST_FORM_NAME:Shared Documents}
+    TEST_SUBFOLDER: ${TEST_SUBFOLDER:DO_NOT_DELETE_AlitaTestFiles}
+    TEST_FILE_NAME: ${TEST_FILE_NAME:DO_NOT_DELETE_test-document.txt}
+    SEARCH_QUERY: ${SEARCH_QUERY:test document}
+    TIMESTAMP: ${TIMESTAMP}
+    DEFAULT_LLM_MODEL: ${DEFAULT_LLM_MODEL:gpt-4o-2024-11-20}
+
+  settings:
+    # Run sequentially: compound tests re-use unique index names so parallel is safe,
+    # but sequential avoids overwhelming the SharePoint Graph API quota.
+    timeout: 900
+    parallel: 1
+    stop_on_failure: false
+
+exclude_tests: []
+
+cleanup:
+  - name: Delete SharePoint Index Toolkit
+    type: toolkit
+    config:
+      toolkit_id: ${ISP_TOOLKIT_ID}
+    enabled: true
+    continue_on_error: true

.alita/tests/test_pipelines/suites/index_sharepoint/tests/test_case_01_index_data_no_path.yaml

@@ -0,0 +1,115 @@
+name: "ISP01-no-path"
+priority: Critical
+description: |
+  Verify index_data indexes all accessible SharePoint files with no path restriction.
+
+  Objective: Baseline indexing — establish that index_data completes successfully
+  and indexes documents from all accessible libraries.
+
+  Expected Behavior:
+  - Tool executes without raising an exception
+  - Result contains "status" field equal to "ok"
+  - Search returns indexed documents successfully
+  - Documents are indexed and searchable
+
+toolkits:
+  - id: ${ISP_TOOLKIT_ID}
+    name: ${ISP_TOOLKIT_NAME}
+
+state:
+  index_result:
+    type: dict
+  search_result:
+    type: dict
+  test_results:
+    type: dict
+
+entry_point: invoke_index_data
+
+nodes:
+  - id: invoke_index_data
+    type: toolkit
+    tool: index_data
+    toolkit_name: ${ISP_TOOLKIT_NAME}
+    input: []
+    input_mapping:
+      index_name:
+        type: fixed
+        value: "idx01"
+      clean_index:
+        type: fixed
+        value: true
+      limit_files:
+        type: fixed
+        value: 20
+    output:
+      - index_result
+    structured_output: true
+    transition: search_indexed_docs
+
+  - id: search_indexed_docs
+    type: toolkit
+    tool: search_index
+    toolkit_name: ${ISP_TOOLKIT_NAME}
+    input: []
+    input_mapping:
+      query:
+        type: fixed
+        value: "document"
+      index_name:
+        type: fixed
+        value: "idx01"
+      search_top:
+        type: fixed
+        value: 100
+    output:
+      - search_result
+    structured_output: true
+    transition: validate_result
+
+  - id: validate_result
+    type: llm
+    model: gpt-4o-2024-11-20
+    input:
+      - index_result
+      - search_result
+    input_mapping:
+      system:
+        type: fixed
+        value: "You are a quality assurance validator."
+      task:
+        type: fstring
+        value: |
+          Analyze the output from the 'index_data' tool and search results.
+
+          Index Result: {index_result}
+          Search Result: {search_result}
+
+          Expected behavior:
+          1. Indexing completed successfully (status: ok)
+          2. Search returns indexed documents (non-empty results)
+          3. No critical errors or exceptions
+
+          Validation rules:
+          1. Check index_result has "status": "ok"
+          2. Check index_result message indicates successful indexing
+          3. Verify search_result is not empty and contains document list
+          4. Confirm at least one document was indexed and is searchable
+          5. All validations must pass for test to pass
+
+          return a json object with:
+          {{
+            "test_passed": true/false,
+            "summary": "brief description including number of documents found",
+            "error": "error details if failed, null if passed"
+          }}
+
+          return **only** the json object. no markdown formatting, no additional text.
+      chat_history:
+        type: fixed
+        value: []
+    output:
+      - test_results
+    structured_output_dict:
+      test_results: "dict"
+    transition: END