diff --git a/docs/115-indexing-api.md b/docs/115-indexing-api.md new file mode 100644 index 00000000..6315463e --- /dev/null +++ b/docs/115-indexing-api.md @@ -0,0 +1,121 @@ +# 115 Cloud Storage Indexing API + +This document describes the HTTP API endpoints for the 115 cloud storage indexing feature. + +## Endpoints + +### 1. Import Batch - POST /api/fs/115/import-batch + +Import multiple 115 cloud storage nodes into the search index. + +**Authentication:** Admin only + +**Request Body:** +```json +{ + "nodes": [ + { + "path": "/movies/action/movie1.mp4", + "name": "movie1.mp4", + "size": 1073741824, + "is_dir": false, + "modified": "2024-01-01T00:00:00Z", + "parent_path": "/movies/action", + "depth": 2, + "child_count": 0 + } + ] +} +``` + +**Response:** +```json +{ + "success": true, + "imported_count": 1, + "failed_count": 0, + "message": "Successfully imported 1 nodes" +} +``` + +### 2. Search - GET/POST /api/fs/115/search + +Search indexed 115 cloud storage nodes. + +**Authentication:** Authenticated users + +**Request Body:** +```json +{ + "query": "movie", + "page": 1, + "per_page": 20, + "scope": 0 +} +``` + +**Parameters:** +- `query` (string, required): Search query +- `page` (int, optional): Page number (default: 1) +- `per_page` (int, optional): Results per page (default: 20, max: 100) +- `scope` (int, optional): Search scope + - `0`: All (files and folders) - default + - `1`: Folders only + - `2`: Files only + +**Response:** +```json +{ + "query": "movie", + "total": 100, + "results": [ + { + "path": "/movies/action/movie1.mp4", + "name": "movie1.mp4", + "size": 1073741824, + "is_dir": false + } + ] +} +``` + +### 3. Clear Index - DELETE /api/fs/115/clear + +Clear the entire 115 index. + +**Authentication:** Admin only + +**Response:** +```json +{ + "message": "Successfully cleared 115 index" +} +``` + +## Implementation Details + +- Index location: `{dataDir}/indexes/115` +- Batch processing: Maximum 10,000 nodes per batch +- Path mapping: Emoji prefixes are automatically removed +- Document ID: Generated from SHA-256 hash of path (idempotent updates) +- Search sorting: Results sorted by indexed_at descending (most recent first) + +## Usage Example + +```bash +# Import nodes +curl -X POST http://localhost:5244/api/fs/115/import-batch \ + -H "Authorization: Bearer YOUR_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"nodes":[{"path":"/test.txt","name":"test.txt","size":100,"is_dir":false}]}' + +# Search +curl -X POST http://localhost:5244/api/fs/115/search \ + -H "Authorization: Bearer YOUR_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"query":"test","page":1,"per_page":20,"scope":0}' + +# Clear index +curl -X DELETE http://localhost:5244/api/fs/115/clear \ + -H "Authorization: Bearer YOUR_TOKEN" +``` diff --git a/internal/bootstrap/alist115.go b/internal/bootstrap/alist115.go new file mode 100644 index 00000000..d7c9b477 --- /dev/null +++ b/internal/bootstrap/alist115.go @@ -0,0 +1,21 @@ +package bootstrap + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/server/handles" + log "github.com/sirupsen/logrus" +) + +func InitAlist115() { + dataDir := conf.Conf.Database.DBFile + if dataDir == "" { + dataDir = "data" + } + + if err := handles.InitAlist115(dataDir); err != nil { + log.Errorf("init alist115 error: %+v", err) + return + } + + log.Info("alist115 service initialized successfully") +} diff --git a/internal/bootstrap/run.go b/internal/bootstrap/run.go index 6740dba6..73b121e4 100644 --- a/internal/bootstrap/run.go +++ b/internal/bootstrap/run.go @@ -36,6 +36,7 @@ func Init() { data.InitData() InitStreamLimit() InitIndex() + InitAlist115() InitUpgradePatch() } diff --git a/internal/search/alist115/model.go b/internal/search/alist115/model.go new file mode 100644 index 00000000..b5b33f97 --- /dev/null +++ b/internal/search/alist115/model.go @@ -0,0 +1,54 @@ +// Package alist115 provides data models and indexing support for 115 cloud storage. +// It enables full-text search across 115 cloud files and directories through +// bleve indexing and provides API structures for import and search operations. +package alist115 + +import "time" + +// IndexNode represents a file or directory node in the 115 index +type IndexNode struct { + Path string `json:"path"` // Full path + Name string `json:"name"` // File/directory name + Size int64 `json:"size"` // File size in bytes + IsDir bool `json:"is_dir"` // Whether this is a directory + Modified time.Time `json:"modified"` // Last modified time + ParentPath string `json:"parent_path"` // Parent directory path + Depth int `json:"depth"` // Directory depth (0 for root) + ChildCount int `json:"child_count"` // Number of children (for directories) +} + +// ImportBatchRequest represents a batch import request +type ImportBatchRequest struct { + Nodes []IndexNode `json:"nodes"` // Nodes to import +} + +// ImportBatchResponse represents a batch import response +type ImportBatchResponse struct { + Success bool `json:"success"` // Whether import succeeded + ImportedCount int `json:"imported_count"` // Number of nodes imported + FailedCount int `json:"failed_count"` // Number of nodes that failed + Message string `json:"message"` // Status message +} + +// SearchRequest represents a search request +type SearchRequest struct { + Query string `json:"query"` // Search query + Page int `json:"page"` // Page number (1-based) + PerPage int `json:"per_page"` // Results per page + Scope int `json:"scope"` // Search scope: 0=all, 1=folder only, 2=file only +} + +// SearchResponse represents a search response +type SearchResponse struct { + Query string `json:"query"` // Original query + Total int `json:"total"` // Total number of results + Results []SearchNode `json:"results"` // Search results +} + +// SearchNode represents a search result node +type SearchNode struct { + Path string `json:"path"` // Full path + Name string `json:"name"` // File/directory name + Size int64 `json:"size"` // File size in bytes + IsDir bool `json:"is_dir"` // Whether this is a directory +} diff --git a/internal/search/alist115/path_mapper.go b/internal/search/alist115/path_mapper.go new file mode 100644 index 00000000..39bb9fab --- /dev/null +++ b/internal/search/alist115/path_mapper.go @@ -0,0 +1,33 @@ +package alist115 + +import ( + "strings" +) + +// MapPath removes the specific emoji prefix added by webdavsim to 115 cloud storage paths. +// webdavsim prepends 🏷️ to mounted 115 share paths, and this function normalizes them +// for indexing and search. +// +// Supported transformation: +// +// /🏷️我的115分享/ → /我的115分享/ +// +// Note: Only the specific emoji 🏷️ from webdavsim is handled. Other emojis in paths +// are intentionally left unchanged as they may be legitimate user content. +// +// Example: +// +// MapPath("/🏷️我的115分享/folder/file.txt") → "/我的115分享/folder/file.txt" +// MapPath("/other-emoji😀/file.txt") → "/other-emoji😀/file.txt" (unchanged) +func MapPath(path string) string { + if path == "" || path == "/" { + return path + } + + // Remove the emoji prefix 🏷️ (keeping the leading /) + if strings.HasPrefix(path, "/🏷️") { + return "/" + strings.TrimPrefix(path, "/🏷️") + } + + return path +} diff --git a/internal/search/alist115/path_mapper_test.go b/internal/search/alist115/path_mapper_test.go new file mode 100644 index 00000000..276a59a9 --- /dev/null +++ b/internal/search/alist115/path_mapper_test.go @@ -0,0 +1,68 @@ +package alist115 + +import ( + "testing" +) + +func TestMapPath(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "removes emoji prefix", + input: "/🏷️我的115分享/folder/file.txt", + expected: "/我的115分享/folder/file.txt", + }, + { + name: "handles path without emoji", + input: "/我的115分享/folder/file.txt", + expected: "/我的115分享/folder/file.txt", + }, + { + name: "handles root path with emoji", + input: "/🏷️我的115分享/", + expected: "/我的115分享/", + }, + { + name: "handles root path without emoji", + input: "/我的115分享/", + expected: "/我的115分享/", + }, + { + name: "handles nested paths with emoji", + input: "/🏷️我的115分享/深层/目录/文件.mp4", + expected: "/我的115分享/深层/目录/文件.mp4", + }, + { + name: "handles empty path", + input: "", + expected: "", + }, + { + name: "handles single slash", + input: "/", + expected: "/", + }, + { + name: "other emojis pass through unchanged", + input: "/folder😀/file🎉.txt", + expected: "/folder😀/file🎉.txt", + }, + { + name: "other emoji at start passes through", + input: "/📁my-folder/file.txt", + expected: "/📁my-folder/file.txt", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := MapPath(tt.input) + if result != tt.expected { + t.Errorf("MapPath(%q) = %q, want %q", tt.input, result, tt.expected) + } + }) + } +} diff --git a/internal/search/alist115/service.go b/internal/search/alist115/service.go new file mode 100644 index 00000000..d9c6af54 --- /dev/null +++ b/internal/search/alist115/service.go @@ -0,0 +1,251 @@ +package alist115 + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" +) + +const ( + // maxBatchSize limits the number of documents indexed in a single batch + // to prevent memory exhaustion with very large datasets + maxBatchSize = 10000 +) + +// Service provides bleve-based indexing and search for 115 cloud storage +type Service struct { + index bleve.Index +} + +// NewService creates or opens a bleve index at dataDir/indexes/115 +func NewService(dataDir string) (*Service, error) { + indexPath := filepath.Join(dataDir, "indexes", "115") + + var index bleve.Index + var err error + + // Try to open existing index + index, err = bleve.Open(indexPath) + if err == bleve.ErrorIndexPathDoesNotExist { + // Create new index with default mapping + mapping := bleve.NewIndexMapping() + index, err = bleve.New(indexPath, mapping) + if err != nil { + return nil, fmt.Errorf("failed to create index: %w", err) + } + } else if err != nil { + return nil, fmt.Errorf("failed to open index: %w", err) + } + + return &Service{ + index: index, + }, nil +} + +// BatchIndex indexes multiple nodes in batch, applying path mapping. +// It processes nodes in chunks to prevent memory exhaustion with large datasets. +// Returns the total number of successfully indexed nodes. +func (s *Service) BatchIndex(nodes []IndexNode) (int, error) { + // Check if index is still open + if s.index == nil { + return 0, fmt.Errorf("index is closed") + } + + totalIndexed := 0 + + // Process in chunks to prevent memory exhaustion + for i := 0; i < len(nodes); i += maxBatchSize { + end := i + maxBatchSize + if end > len(nodes) { + end = len(nodes) + } + + chunk := nodes[i:end] + indexed, err := s.indexChunk(chunk) + totalIndexed += indexed + + if err != nil { + return totalIndexed, fmt.Errorf("failed to index chunk at offset %d: %w", i, err) + } + } + + return totalIndexed, nil +} + +// indexChunk indexes a single chunk of nodes +func (s *Service) indexChunk(nodes []IndexNode) (int, error) { + batch := s.index.NewBatch() + indexed := 0 + + for _, node := range nodes { + // Apply path mapping to remove emoji prefix + mappedPath := MapPath(node.Path) + + // Create document for indexing + doc := map[string]interface{}{ + "path": mappedPath, + "name": node.Name, + "size": node.Size, + "is_dir": node.IsDir, + "indexed_at": time.Now(), + } + + // Use deterministic ID based on path for idempotent indexing + docID := generateDocID(mappedPath) + + err := batch.Index(docID, doc) + if err != nil { + return indexed, fmt.Errorf("failed to add document to batch: %w", err) + } + indexed++ + } + + // Execute batch + if err := s.index.Batch(batch); err != nil { + // Don't count as indexed if batch execution fails + return 0, fmt.Errorf("failed to execute batch: %w", err) + } + + return indexed, nil +} + +// generateDocID creates a deterministic document ID from a path using SHA-256 hash. +// This ensures that re-importing the same file updates the existing document +// instead of creating duplicates. +func generateDocID(path string) string { + hash := sha256.Sum256([]byte(path)) + return hex.EncodeToString(hash[:]) +} + +// Search performs a search query on the indexed nodes. +// It searches the path field using a match query and returns paginated results. +func (s *Service) Search(req SearchRequest) (*SearchResponse, error) { + // Check if index is still open + if s.index == nil { + return nil, fmt.Errorf("index is closed") + } + + // Apply defaults and validate pagination + if req.Page <= 0 { + req.Page = 1 + } + if req.PerPage <= 0 { + req.PerPage = 20 + } + if req.PerPage > 100 { + req.PerPage = 100 + } + + // Calculate offset from page number (Page=1 is first page) + offset := (req.Page - 1) * req.PerPage + + // Build match query on path field + matchQuery := bleve.NewMatchQuery(req.Query) + matchQuery.SetField("path") + + // Apply scope filtering if specified + var q query.Query = matchQuery + if req.Scope == 1 { + // Scope 1: folders only (is_dir=true) + boolQuery := bleve.NewBooleanQuery() + boolQuery.AddMust(matchQuery) + isDirQuery := bleve.NewBoolFieldQuery(true) + isDirQuery.SetField("is_dir") + boolQuery.AddMust(isDirQuery) + q = boolQuery + } else if req.Scope == 2 { + // Scope 2: files only (is_dir=false) + boolQuery := bleve.NewBooleanQuery() + boolQuery.AddMust(matchQuery) + isDirQuery := bleve.NewBoolFieldQuery(false) + isDirQuery.SetField("is_dir") + boolQuery.AddMust(isDirQuery) + q = boolQuery + } + // Scope 0 or invalid: no filter (all results) + + searchRequest := bleve.NewSearchRequest(q) + searchRequest.From = offset + searchRequest.Size = req.PerPage + + // Sort by indexed_at descending (most recent first) + searchRequest.SortBy([]string{"-indexed_at"}) + + // Request specific fields to extract + searchRequest.Fields = []string{"path", "name", "size", "is_dir"} + + // Execute search + searchResults, err := s.index.Search(searchRequest) + if err != nil { + return nil, fmt.Errorf("search failed: %w", err) + } + + // Build result nodes + results := make([]SearchNode, 0, len(searchResults.Hits)) + for _, hit := range searchResults.Hits { + node := SearchNode{} + + // Extract fields from hit + if path, ok := hit.Fields["path"].(string); ok { + node.Path = path + } + if name, ok := hit.Fields["name"].(string); ok { + node.Name = name + } + if size, ok := hit.Fields["size"].(float64); ok { + node.Size = int64(size) + } + if isDir, ok := hit.Fields["is_dir"].(bool); ok { + node.IsDir = isDir + } + + results = append(results, node) + } + + return &SearchResponse{ + Query: req.Query, + Total: int(searchResults.Total), + Results: results, + }, nil +} + +// Clear closes the current index, deletes the index directory, and creates a new empty index. +func (s *Service) Clear(dataDir string) error { + // Close current index + if s.index != nil { + if err := s.index.Close(); err != nil { + return fmt.Errorf("failed to close index: %w", err) + } + s.index = nil + } + + // Delete index directory + indexPath := filepath.Join(dataDir, "indexes", "115") + if err := os.RemoveAll(indexPath); err != nil { + return fmt.Errorf("failed to delete index directory: %w", err) + } + + // Create new empty index + mapping := bleve.NewIndexMapping() + index, err := bleve.New(indexPath, mapping) + if err != nil { + return fmt.Errorf("failed to create new index: %w", err) + } + + s.index = index + return nil +} + +// Close closes the bleve index +func (s *Service) Close() error { + if s.index != nil { + return s.index.Close() + } + return nil +} diff --git a/internal/search/alist115/service_test.go b/internal/search/alist115/service_test.go new file mode 100644 index 00000000..139f7998 --- /dev/null +++ b/internal/search/alist115/service_test.go @@ -0,0 +1,550 @@ +package alist115 + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/blevesearch/bleve/v2" +) + +func TestServiceBatchIndex(t *testing.T) { + // Create temporary directory for test index + tempDir, err := os.MkdirTemp("", "alist115-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + t.Cleanup(func() { os.RemoveAll(tempDir) }) + + // Create service + service, err := NewService(tempDir) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer service.Close() + + // Verify index path exists + indexPath := filepath.Join(tempDir, "indexes", "115") + if _, err := os.Stat(indexPath); os.IsNotExist(err) { + t.Errorf("Index directory was not created at: %s", indexPath) + } + + // Create test nodes with emoji paths + nodes := []IndexNode{ + { + Path: "/🏷️我的115分享/测试文件夹/file1.txt", + Name: "file1.txt", + Size: 1024, + IsDir: false, + Modified: time.Now(), + }, + { + Path: "/🏷️我的115分享/测试文件夹", + Name: "测试文件夹", + Size: 0, + IsDir: true, + Modified: time.Now(), + }, + } + + // Index nodes + indexed, err := service.BatchIndex(nodes) + if err != nil { + t.Fatalf("BatchIndex failed: %v", err) + } + + // Verify indexed count + if indexed != 2 { + t.Errorf("Expected indexed=2, got %d", indexed) + } + + // Verify documents are in index by checking document count + docCount, err := service.index.DocCount() + if err != nil { + t.Fatalf("Failed to get document count: %v", err) + } + + if docCount != 2 { + t.Errorf("Expected 2 documents in index, got %d", docCount) + } + + // Verify path mapping was applied by searching for the mapped path + // (without emoji prefix) + query := bleve.NewMatchQuery("我的115分享") + search := bleve.NewSearchRequest(query) + searchResults, err := service.index.Search(search) + if err != nil { + t.Fatalf("Failed to search index: %v", err) + } + + if searchResults.Total != 2 { + t.Errorf("Expected to find 2 results with mapped path, got %d", searchResults.Total) + } +} + +func TestServiceClose(t *testing.T) { + // Create temporary directory for test index + tempDir, err := os.MkdirTemp("", "alist115-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + t.Cleanup(func() { os.RemoveAll(tempDir) }) + + // Create service + service, err := NewService(tempDir) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + + // Close service + err = service.Close() + if err != nil { + t.Errorf("Close failed: %v", err) + } + + // Verify we can reopen the index + service2, err := NewService(tempDir) + if err != nil { + t.Fatalf("Failed to reopen service: %v", err) + } + defer service2.Close() +} + +// TestBatchIndexAfterClose verifies that BatchIndex returns an error when called after Close +func TestBatchIndexAfterClose(t *testing.T) { + tempDir, err := os.MkdirTemp("", "alist115-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + t.Cleanup(func() { os.RemoveAll(tempDir) }) + + service, err := NewService(tempDir) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + + // Close the service + if err := service.Close(); err != nil { + t.Fatalf("Failed to close service: %v", err) + } + + // Try to index after close - should fail + nodes := []IndexNode{ + {Path: "/test/file.txt", Name: "file.txt", Size: 100, IsDir: false}, + } + + indexed, err := service.BatchIndex(nodes) + if err == nil { + t.Error("Expected error when indexing after close, got nil") + } + if indexed != 0 { + t.Errorf("Expected indexed=0 after close, got %d", indexed) + } +} + +// TestIdempotentIndexing verifies that re-indexing the same path updates instead of duplicating +func TestIdempotentIndexing(t *testing.T) { + tempDir, err := os.MkdirTemp("", "alist115-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + t.Cleanup(func() { os.RemoveAll(tempDir) }) + + service, err := NewService(tempDir) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer service.Close() + + // Index a node + nodes := []IndexNode{ + {Path: "/test/file.txt", Name: "file.txt", Size: 100, IsDir: false, Modified: time.Now()}, + } + + indexed, err := service.BatchIndex(nodes) + if err != nil { + t.Fatalf("First BatchIndex failed: %v", err) + } + if indexed != 1 { + t.Errorf("Expected indexed=1, got %d", indexed) + } + + // Verify document count + docCount, err := service.index.DocCount() + if err != nil { + t.Fatalf("Failed to get document count: %v", err) + } + if docCount != 1 { + t.Errorf("Expected 1 document after first index, got %d", docCount) + } + + // Re-index the same path with different size (simulating an update) + nodes[0].Size = 200 + indexed, err = service.BatchIndex(nodes) + if err != nil { + t.Fatalf("Second BatchIndex failed: %v", err) + } + if indexed != 1 { + t.Errorf("Expected indexed=1, got %d", indexed) + } + + // Verify document count is still 1 (updated, not duplicated) + docCount, err = service.index.DocCount() + if err != nil { + t.Fatalf("Failed to get document count: %v", err) + } + if docCount != 1 { + t.Errorf("Expected 1 document after re-index (no duplicate), got %d", docCount) + } +} + +// TestLargeBatchChunking verifies that large batches are processed in chunks +func TestLargeBatchChunking(t *testing.T) { + tempDir, err := os.MkdirTemp("", "alist115-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + t.Cleanup(func() { os.RemoveAll(tempDir) }) + + service, err := NewService(tempDir) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer service.Close() + + // Create 25,000 nodes (should be processed in 3 chunks of 10,000) + const nodeCount = 25000 + nodes := make([]IndexNode, nodeCount) + for i := 0; i < nodeCount; i++ { + nodes[i] = IndexNode{ + Path: fmt.Sprintf("/test/file%d.txt", i), + Name: fmt.Sprintf("file%d.txt", i), + Size: int64(i * 100), + IsDir: false, + Modified: time.Now(), + } + } + + // Index all nodes + indexed, err := service.BatchIndex(nodes) + if err != nil { + t.Fatalf("BatchIndex failed: %v", err) + } + + if indexed != nodeCount { + t.Errorf("Expected indexed=%d, got %d", nodeCount, indexed) + } + + // Verify document count + docCount, err := service.index.DocCount() + if err != nil { + t.Fatalf("Failed to get document count: %v", err) + } + + if docCount != nodeCount { + t.Errorf("Expected %d documents in index, got %d", nodeCount, docCount) + } +} + +// TestServiceSearch verifies that search returns matching files +func TestServiceSearch(t *testing.T) { + tempDir, err := os.MkdirTemp("", "alist115-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + t.Cleanup(func() { os.RemoveAll(tempDir) }) + + service, err := NewService(tempDir) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer service.Close() + + // Index 3 files: 2 with "侏罗纪", 1 with "other" + nodes := []IndexNode{ + { + Path: "/movies/侏罗纪公园.mp4", + Name: "侏罗纪公园.mp4", + Size: 1024000, + IsDir: false, + Modified: time.Now(), + }, + { + Path: "/movies/侏罗纪世界.mp4", + Name: "侏罗纪世界.mp4", + Size: 2048000, + IsDir: false, + Modified: time.Now(), + }, + { + Path: "/movies/other_movie.mp4", + Name: "other_movie.mp4", + Size: 512000, + IsDir: false, + Modified: time.Now(), + }, + } + + indexed, err := service.BatchIndex(nodes) + if err != nil { + t.Fatalf("BatchIndex failed: %v", err) + } + if indexed != 3 { + t.Fatalf("Expected 3 indexed nodes, got %d", indexed) + } + + // Search for "侏罗纪" + req := SearchRequest{ + Query: "侏罗纪", + Page: 1, + PerPage: 20, + } + + resp, err := service.Search(req) + if err != nil { + t.Fatalf("Search failed: %v", err) + } + + // Verify Total >= 2 + if resp.Total < 2 { + t.Errorf("Expected Total >= 2, got %d", resp.Total) + } + + // Verify query is echoed back + if resp.Query != "侏罗纪" { + t.Errorf("Expected Query='侏罗纪', got '%s'", resp.Query) + } + + // Verify results contain data + if len(resp.Results) < 2 { + t.Errorf("Expected at least 2 results, got %d", len(resp.Results)) + } + + // Verify result nodes have expected fields + for _, node := range resp.Results { + if node.Path == "" { + t.Error("Result node has empty Path") + } + if node.Name == "" { + t.Error("Result node has empty Name") + } + } +} + +// TestServiceSearchPagination verifies that pagination works correctly +func TestServiceSearchPagination(t *testing.T) { + tempDir, err := os.MkdirTemp("", "alist115-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + t.Cleanup(func() { os.RemoveAll(tempDir) }) + + service, err := NewService(tempDir) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer service.Close() + + // Index 5 files with "test" + nodes := make([]IndexNode, 5) + for i := 0; i < 5; i++ { + nodes[i] = IndexNode{ + Path: fmt.Sprintf("/test/file%d.txt", i), + Name: fmt.Sprintf("file%d.txt", i), + Size: int64(i * 100), + IsDir: false, + Modified: time.Now(), + } + } + + indexed, err := service.BatchIndex(nodes) + if err != nil { + t.Fatalf("BatchIndex failed: %v", err) + } + if indexed != 5 { + t.Fatalf("Expected 5 indexed nodes, got %d", indexed) + } + + // Search with PerPage=2 + req := SearchRequest{ + Query: "test", + Page: 1, + PerPage: 2, + } + + resp, err := service.Search(req) + if err != nil { + t.Fatalf("Search failed: %v", err) + } + + // Verify Total is 5 but Results only has 2 + if resp.Total != 5 { + t.Errorf("Expected Total=5, got %d", resp.Total) + } + if len(resp.Results) != 2 { + t.Errorf("Expected 2 results (pagination), got %d", len(resp.Results)) + } + + // Search with Page=2 + req.Page = 2 + resp, err = service.Search(req) + if err != nil { + t.Fatalf("Search with page 2 failed: %v", err) + } + + if resp.Total != 5 { + t.Errorf("Expected Total=5, got %d", resp.Total) + } + if len(resp.Results) != 2 { + t.Errorf("Expected 2 results (pagination with page 2), got %d", len(resp.Results)) + } +} + +// TestServiceClear verifies that Clear empties the index +func TestServiceClear(t *testing.T) { + tempDir, err := os.MkdirTemp("", "alist115-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + t.Cleanup(func() { os.RemoveAll(tempDir) }) + + service, err := NewService(tempDir) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer service.Close() + + // Index some nodes + nodes := []IndexNode{ + {Path: "/test/file1.txt", Name: "file1.txt", Size: 100, IsDir: false, Modified: time.Now()}, + {Path: "/test/file2.txt", Name: "file2.txt", Size: 200, IsDir: false, Modified: time.Now()}, + } + + indexed, err := service.BatchIndex(nodes) + if err != nil { + t.Fatalf("BatchIndex failed: %v", err) + } + if indexed != 2 { + t.Fatalf("Expected 2 indexed nodes, got %d", indexed) + } + + // Verify documents are in index + docCount, err := service.index.DocCount() + if err != nil { + t.Fatalf("Failed to get document count: %v", err) + } + if docCount != 2 { + t.Errorf("Expected 2 documents before clear, got %d", docCount) + } + + // Clear the index + err = service.Clear(tempDir) + if err != nil { + t.Fatalf("Clear failed: %v", err) + } + + // Verify index is empty + docCount, err = service.index.DocCount() + if err != nil { + t.Fatalf("Failed to get document count after clear: %v", err) + } + if docCount != 0 { + t.Errorf("Expected 0 documents after clear, got %d", docCount) + } + + // Verify we can still index after clear + indexed, err = service.BatchIndex(nodes[:1]) + if err != nil { + t.Fatalf("BatchIndex after clear failed: %v", err) + } + if indexed != 1 { + t.Fatalf("Expected 1 indexed node after clear, got %d", indexed) + } +} + +// TestServiceSearchScope verifies that scope filtering works correctly +func TestServiceSearchScope(t *testing.T) { + tempDir, err := os.MkdirTemp("", "alist115-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + t.Cleanup(func() { os.RemoveAll(tempDir) }) + + service, err := NewService(tempDir) + if err != nil { + t.Fatalf("Failed to create service: %v", err) + } + defer service.Close() + + // Index mixed nodes: 2 folders and 3 files, all with "test" in path + nodes := []IndexNode{ + {Path: "/test/folder1", Name: "folder1", Size: 0, IsDir: true, Modified: time.Now()}, + {Path: "/test/folder2", Name: "folder2", Size: 0, IsDir: true, Modified: time.Now()}, + {Path: "/test/file1.txt", Name: "file1.txt", Size: 100, IsDir: false, Modified: time.Now()}, + {Path: "/test/file2.txt", Name: "file2.txt", Size: 200, IsDir: false, Modified: time.Now()}, + {Path: "/test/file3.txt", Name: "file3.txt", Size: 300, IsDir: false, Modified: time.Now()}, + } + + indexed, err := service.BatchIndex(nodes) + if err != nil { + t.Fatalf("BatchIndex failed: %v", err) + } + if indexed != 5 { + t.Fatalf("Expected 5 indexed nodes, got %d", indexed) + } + + // Test Scope 0: all results (default) + req := SearchRequest{ + Query: "test", + Page: 1, + PerPage: 20, + Scope: 0, + } + + resp, err := service.Search(req) + if err != nil { + t.Fatalf("Search with Scope=0 failed: %v", err) + } + + if resp.Total != 5 { + t.Errorf("Scope=0: Expected Total=5 (all), got %d", resp.Total) + } + + // Test Scope 1: folders only + req.Scope = 1 + resp, err = service.Search(req) + if err != nil { + t.Fatalf("Search with Scope=1 failed: %v", err) + } + + if resp.Total != 2 { + t.Errorf("Scope=1: Expected Total=2 (folders only), got %d", resp.Total) + } + + // Verify all results are folders + for _, node := range resp.Results { + if !node.IsDir { + t.Errorf("Scope=1: Expected only folders, but got file: %s", node.Path) + } + } + + // Test Scope 2: files only + req.Scope = 2 + resp, err = service.Search(req) + if err != nil { + t.Fatalf("Search with Scope=2 failed: %v", err) + } + + if resp.Total != 3 { + t.Errorf("Scope=2: Expected Total=3 (files only), got %d", resp.Total) + } + + // Verify all results are files + for _, node := range resp.Results { + if node.IsDir { + t.Errorf("Scope=2: Expected only files, but got folder: %s", node.Path) + } + } +} diff --git a/server/handles/alist115.go b/server/handles/alist115.go new file mode 100644 index 00000000..47f274e5 --- /dev/null +++ b/server/handles/alist115.go @@ -0,0 +1,124 @@ +package handles + +import ( + "fmt" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/search/alist115" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +var alist115Service *alist115.Service + +// InitAlist115 initializes the alist115 indexing service +func InitAlist115(dataDir string) error { + var err error + alist115Service, err = alist115.NewService(dataDir) + if err != nil { + return fmt.Errorf("failed to initialize alist115 service: %w", err) + } + return nil +} + +// Alist115ImportBatch handles batch import of 115 cloud storage nodes +func Alist115ImportBatch(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if !user.IsAdmin() { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + + var req alist115.ImportBatchRequest + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + + if len(req.Nodes) == 0 { + common.ErrorStrResp(c, "no nodes to import", 400) + return + } + + if alist115Service == nil { + common.ErrorStrResp(c, "alist115 service not initialized", 500) + return + } + + importedCount, err := alist115Service.BatchIndex(req.Nodes) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + + resp := alist115.ImportBatchResponse{ + Success: true, + ImportedCount: importedCount, + FailedCount: len(req.Nodes) - importedCount, + Message: fmt.Sprintf("Successfully imported %d nodes", importedCount), + } + + common.SuccessResp(c, resp) +} + +// Alist115Search handles search requests for 115 cloud storage +func Alist115Search(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if user == nil { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + + var req alist115.SearchRequest + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + + if req.Query == "" { + common.ErrorStrResp(c, "query cannot be empty", 400) + return + } + + if alist115Service == nil { + common.ErrorStrResp(c, "alist115 service not initialized", 500) + return + } + + results, err := alist115Service.Search(req) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + + common.SuccessResp(c, results) +} + +// Alist115Clear handles clearing the 115 index +func Alist115Clear(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if !user.IsAdmin() { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + + if alist115Service == nil { + common.ErrorStrResp(c, "alist115 service not initialized", 500) + return + } + + // Get data directory from config + dataDir := conf.Conf.Database.DBFile + if dataDir == "" { + dataDir = "data" + } + + if err := alist115Service.Clear(dataDir); err != nil { + common.ErrorResp(c, err, 500) + return + } + + common.SuccessWithMsgResp(c, "Successfully cleared 115 index") +} diff --git a/server/router.go b/server/router.go index f46b0b1b..fa969bdb 100644 --- a/server/router.go +++ b/server/router.go @@ -106,6 +106,7 @@ func Init(e *gin.Engine) { _task(auth.Group("/task", middlewares.AuthNotGuest)) _sharing(auth.Group("/share", middlewares.AuthNotGuest)) admin(auth.Group("/admin", middlewares.AuthAdmin)) + _alist115(auth.Group("/fs/115")) if flags.Debug || flags.Dev { debug(g.Group("/debug")) } @@ -248,6 +249,12 @@ func _sharing(g *gin.RouterGroup) { g.POST("/disable", handles.SetEnableSharing(true)) } +func _alist115(g *gin.RouterGroup) { + g.POST("/import-batch", handles.Alist115ImportBatch) + g.Any("/search", handles.Alist115Search) + g.DELETE("/clear", handles.Alist115Clear) +} + func Cors(r *gin.Engine) { config := cors.DefaultConfig() // config.AllowAllOrigins = true