fixed add-docs for documents and website

cmd/wizard.go

@@ -172,6 +172,8 @@ This makes it easy to set up a new RAG without remembering all command options.`
 		var excludePaths []string
 		var useWebCrawler bool
 		var useSitemap bool
+		var crawlSingleURL bool
+		var crawlURLsList []string
 
 		if sourceChoice == "2" {
 			// Website crawler option
@@ -223,6 +225,44 @@ This makes it easy to set up a new RAG without remembering all command options.`
 			if err != nil {
 				return err
 			}
+
+			// Display crawling configuration
+			fmt.Println("\nCrawling configuration:")
+			fmt.Printf("- URL: %s\n", websiteURL)
+			fmt.Printf("- Max depth: %d\n", maxDepth)
+			fmt.Printf("- Concurrency: %d\n", concurrency)
+			if len(excludePaths) > 0 {
+				fmt.Printf("- Excluded paths: %s\n", strings.Join(excludePaths, ", "))
+			}
+			fmt.Printf("- Use sitemap: %t\n", useSitemap)
+			fmt.Printf("- Single URL mode: %t\n", crawlSingleURL)
+			if len(crawlURLsList) > 0 {
+				fmt.Printf("- Specific URLs: %s\n", strings.Join(crawlURLsList, ", "))
+			}
+
+			// Ask if the user wants to use single URL mode
+			useSingleURLPrompt := &survey.Confirm{
+				Message: "Use single URL mode (only crawl the specified URL without following links)?",
+				Default: false,
+			}
+			err = survey.AskOne(useSingleURLPrompt, &crawlSingleURL)
+			if err != nil {
+				return err
+			}
+
+			// Ask for a list of specific URLs to crawl
+			if !crawlSingleURL {
+				fmt.Print("Enter specific URLs to crawl (comma-separated, leave empty to crawl all): ")
+				urlsListStr, _ := reader.ReadString('\n')
+				urlsListStr = strings.TrimSpace(urlsListStr)
+				if urlsListStr != "" {
+					urlsList := strings.Split(urlsListStr, ",")
+					for i := range urlsList {
+						urlsList[i] = strings.TrimSpace(urlsList[i])
+					}
+					crawlURLsList = urlsList
+				}
+			}
 		} else {
 			// Option local folder (existing code)
 			useWebCrawler = false
@@ -316,9 +356,14 @@ This makes it easy to set up a new RAG without remembering all command options.`
 			fmt.Printf("- Source: Website - %s\n", websiteURL)
 			fmt.Printf("- Crawl depth: %d\n", maxDepth)
 			fmt.Printf("- Concurrency: %d\n", concurrency)
+			fmt.Printf("- Use sitemap: %t\n", useSitemap)
+			fmt.Printf("- Single URL mode: %t\n", crawlSingleURL)
 			if len(excludePaths) > 0 {
 				fmt.Printf("- Exclude paths: %s\n", strings.Join(excludePaths, ", "))
 			}
+			if len(crawlURLsList) > 0 {
+				fmt.Printf("- Specific URLs: %s\n", strings.Join(crawlURLsList, ", "))
+			}
 		} else {
 			fmt.Printf("- Source: Local folder - %s\n", folderPath)
 			if len(excludeDirs) > 0 {
@@ -372,9 +417,18 @@ This makes it easy to set up a new RAG without remembering all command options.`
 				return fmt.Errorf("error initializing web crawler: %w", err)
 			}
 
+			// Set the single URL option
+			webCrawler.SetSingleURLMode(crawlSingleURL)
+
 			// Set the sitemap option
 			webCrawler.SetUseSitemap(useSitemap)
 
+			// Set specific URLs list if provided
+			if len(crawlURLsList) > 0 {
+				webCrawler.SetURLsList(crawlURLsList)
+				fmt.Printf("Using list of %d specific URLs for crawling\n", len(crawlURLsList))
+			}
+
 			// Start the crawling
 			documents, err := webCrawler.CrawlWebsite()
 			if err != nil {
@@ -433,6 +487,98 @@ This makes it easy to set up a new RAG without remembering all command options.`
 		fmt.Println("\n🎉 RAG created successfully! 🎉")
 		fmt.Printf("\nYou can now use your RAG with: rlama run %s\n", ragName)
 
+		// Ask if the user wants to set up directory watching
+		if !useWebCrawler {
+			// For local folder RAGs, offer directory watching
+			setupDirWatchPrompt := &survey.Confirm{
+				Message: "Do you want to set up directory watching to automatically update the RAG when files change?",
+				Default: false,
+			}
+
+			var setupDirWatch bool
+			if err := survey.AskOne(setupDirWatchPrompt, &setupDirWatch); err == nil && setupDirWatch {
+				// Ask for the interval
+				var watchInterval int
+				intervalPrompt := &survey.Input{
+					Message: "Check interval in minutes (0 = only check when RAG is used):",
+					Default: "60",
+				}
+
+				var intervalStr string
+				if err := survey.AskOne(intervalPrompt, &intervalStr); err == nil {
+					fmt.Sscanf(intervalStr, "%d", &watchInterval)
+
+					// Set up directory watching with appropriate options
+					watchLoaderOptions := service.DocumentLoaderOptions{
+						ExcludeDirs:      excludeDirs,
+						ExcludeExts:      excludeExts,
+						ProcessExts:      processExts,
+						ChunkSize:        chunkSize,
+						ChunkOverlap:     overlap,
+						ChunkingStrategy: chunkingStrategy,
+					}
+
+					// Set up directory watching
+					err := ragService.SetupDirectoryWatching(ragName, folderPath, watchInterval, watchLoaderOptions)
+					if err != nil {
+						fmt.Printf("Warning: Failed to set up directory watching: %v\n", err)
+					} else {
+						if watchInterval == 0 {
+							fmt.Printf("Directory watching set up for '%s'. Directory '%s' will be checked each time the RAG is used.\n",
+								ragName, folderPath)
+						} else {
+							fmt.Printf("Directory watching set up for '%s'. Directory '%s' will be checked every %d minutes.\n",
+								ragName, folderPath, watchInterval)
+						}
+					}
+				}
+			}
+		} else {
+			// For web crawler RAGs, offer web watching
+			setupWebWatchPrompt := &survey.Confirm{
+				Message: "Do you want to set up website watching to automatically update the RAG when website content changes?",
+				Default: false,
+			}
+
+			var setupWebWatch bool
+			if err := survey.AskOne(setupWebWatchPrompt, &setupWebWatch); err == nil && setupWebWatch {
+				// Ask for the interval
+				var watchInterval int
+				intervalPrompt := &survey.Input{
+					Message: "Check interval in minutes (0 = only check when RAG is used):",
+					Default: "60",
+				}
+
+				var intervalStr string
+				if err := survey.AskOne(intervalPrompt, &intervalStr); err == nil {
+					fmt.Sscanf(intervalStr, "%d", &watchInterval)
+
+					// Set up web watching options
+					webWatchOptions := domain.WebWatchOptions{
+						MaxDepth:     maxDepth,
+						Concurrency:  concurrency,
+						ExcludePaths: excludePaths,
+						ChunkSize:    chunkSize,
+						ChunkOverlap: overlap,
+					}
+
+					// Set up web watching
+					err := ragService.SetupWebWatching(ragName, websiteURL, watchInterval, webWatchOptions)
+					if err != nil {
+						fmt.Printf("Warning: Failed to set up website watching: %v\n", err)
+					} else {
+						if watchInterval == 0 {
+							fmt.Printf("Website watching set up for '%s'. Website '%s' will be checked each time the RAG is used.\n",
+								ragName, websiteURL)
+						} else {
+							fmt.Printf("Website watching set up for '%s'. Website '%s' will be checked every %d minutes.\n",
+								ragName, websiteURL, watchInterval)
+						}
+					}
+				}
+			}
+		}
+
 		return nil
 	},
 }

internal/crawler/crawler.go

@@ -203,7 +203,15 @@ func (wc *WebCrawler) crawlStandard() ([]domain.Document, error) {
 		}
 
 		// Don't crawl deeper if we've reached the maximum depth
-		urlDepth := strings.Count(url[len(wc.baseURL.String()):], "/")
+		urlStr := url
+		baseURLStr := wc.baseURL.String()
+
+		// Prevent panic when url length is less than baseURL length
+		urlDepth := 0
+		if len(urlStr) > len(baseURLStr) {
+			urlDepth = strings.Count(urlStr[len(baseURLStr):], "/")
+		}
+
 		if urlDepth >= wc.maxDepth {
 			continue
 		}