feat: 提升 WebFetch 网页正文提取与模型可消费性

go.mod

@@ -6,6 +6,8 @@ require (
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
+	github.com/joho/godotenv v1.5.1
+	golang.org/x/net v0.50.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -20,7 +22,6 @@ require (
 	github.com/clipperhouse/stringish v0.1.1 // indirect
 	github.com/clipperhouse/uax29/v2 v2.5.0 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
-	github.com/joho/godotenv v1.5.1 // indirect
 	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
@@ -31,6 +32,6 @@ require (
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sahilm/fuzzy v0.1.1 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
-	golang.org/x/sys v0.38.0 // indirect
-	golang.org/x/text v0.3.8 // indirect
+	golang.org/x/sys v0.41.0 // indirect
+	golang.org/x/text v0.34.0 // indirect
 )

go.sum

@@ -56,12 +56,18 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
+golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

internal/app/bootstrap.go

@@ -23,14 +23,7 @@ func NewProgram(ctx context.Context) (*tea.Program, error) {
 		return nil, err
 	}
 
-	toolRegistry := tools.NewRegistry()
-	toolRegistry.Register(filesystem.New(cfg.Workdir))
-	toolRegistry.Register(filesystem.NewWrite(cfg.Workdir))
-	toolRegistry.Register(filesystem.NewGrep(cfg.Workdir))
-	toolRegistry.Register(filesystem.NewGlob(cfg.Workdir))
-	toolRegistry.Register(filesystem.NewEdit(cfg.Workdir))
-	toolRegistry.Register(bash.New(cfg.Workdir, cfg.Shell, time.Duration(cfg.ToolTimeoutSec)*time.Second))
-	toolRegistry.Register(webfetch.New(time.Duration(cfg.ToolTimeoutSec) * time.Second))
+	toolRegistry := buildToolRegistry(cfg)
 
 	sessionStore := agentruntime.NewSessionStore(loader.BaseDir())
 	runtimeSvc := agentruntime.New(manager, toolRegistry, sessionStore)
@@ -45,3 +38,19 @@ func NewProgram(ctx context.Context) (*tea.Program, error) {
 		tea.WithMouseCellMotion(),
 	), nil
 }
+
+func buildToolRegistry(cfg config.Config) *tools.Registry {
+	toolRegistry := tools.NewRegistry()
+	toolRegistry.Register(filesystem.New(cfg.Workdir))
+	toolRegistry.Register(filesystem.NewWrite(cfg.Workdir))
+	toolRegistry.Register(filesystem.NewGrep(cfg.Workdir))
+	toolRegistry.Register(filesystem.NewGlob(cfg.Workdir))
+	toolRegistry.Register(filesystem.NewEdit(cfg.Workdir))
+	toolRegistry.Register(bash.New(cfg.Workdir, cfg.Shell, time.Duration(cfg.ToolTimeoutSec)*time.Second))
+	toolRegistry.Register(webfetch.New(webfetch.Config{
+		Timeout:               time.Duration(cfg.ToolTimeoutSec) * time.Second,
+		MaxResponseBytes:      cfg.Tools.WebFetch.MaxResponseBytes,
+		SupportedContentTypes: cfg.Tools.WebFetch.SupportedContentTypes,
+	}))
+	return toolRegistry
+}

internal/app/bootstrap_test.go

@@ -2,9 +2,15 @@ package app
 
 import (
 	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/dust/neo-code/internal/config"
+	"github.com/dust/neo-code/internal/tools"
 )
 
 func TestNewProgram(t *testing.T) {
@@ -25,3 +31,42 @@ func TestNewProgram(t *testing.T) {
 		t.Fatalf("expected config file to be created at %q: %v", configPath, err)
 	}
 }
+
+func TestBuildToolRegistryUsesWebFetchConfig(t *testing.T) {
+	t.Parallel()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		_, _ = w.Write([]byte("1234567890"))
+	}))
+	defer server.Close()
+
+	cfg := config.Default().Clone()
+	cfg.Workdir = t.TempDir()
+	cfg.Tools.WebFetch.MaxResponseBytes = 4
+
+	registry := buildToolRegistry(cfg)
+	tool, err := registry.Get("webfetch")
+	if err != nil {
+		t.Fatalf("registry.Get(webfetch) error = %v", err)
+	}
+
+	args, err := json.Marshal(map[string]string{"url": server.URL})
+	if err != nil {
+		t.Fatalf("marshal args: %v", err)
+	}
+
+	result, execErr := tool.Execute(context.Background(), tools.ToolCallInput{
+		Name:      "webfetch",
+		Arguments: args,
+	})
+	if execErr != nil {
+		t.Fatalf("webfetch execute error = %v", execErr)
+	}
+	if truncated, ok := result.Metadata["truncated"].(bool); !ok || !truncated {
+		t.Fatalf("expected truncated metadata, got %+v", result.Metadata)
+	}
+	if result.Content == "" {
+		t.Fatalf("expected formatted webfetch content")
+	}
+}

internal/config/config_test.go

@@ -24,6 +24,12 @@ selected_provider: openai
 current_model: gpt-5.4
 workdir: .
 shell: powershell
+tools:
+  webfetch:
+    max_response_bytes: 4096
+    supported_content_types:
+      - text/html
+      - text/plain
 providers:
   - name: openai
     type: openai
@@ -43,6 +49,12 @@ providers:
 				if provider.BaseURL != "https://example.com/v1" {
 					t.Fatalf("expected custom base url, got %q", provider.BaseURL)
 				}
+				if cfg.Tools.WebFetch.MaxResponseBytes != 4096 {
+					t.Fatalf("expected custom max_response_bytes 4096, got %d", cfg.Tools.WebFetch.MaxResponseBytes)
+				}
+				if len(cfg.Tools.WebFetch.SupportedContentTypes) != 2 {
+					t.Fatalf("expected 2 supported content types, got %+v", cfg.Tools.WebFetch.SupportedContentTypes)
+				}
 			},
 		},
 		{
@@ -73,6 +85,12 @@ providers:
 				if provider.Model != "gpt-4o" {
 					t.Fatalf("expected provider model gpt-4o, got %q", provider.Model)
 				}
+				if cfg.Tools.WebFetch.MaxResponseBytes != DefaultWebFetchMaxResponseBytes {
+					t.Fatalf("expected default max_response_bytes %d, got %d", DefaultWebFetchMaxResponseBytes, cfg.Tools.WebFetch.MaxResponseBytes)
+				}
+				if len(cfg.Tools.WebFetch.SupportedContentTypes) != len(DefaultWebFetchSupportedContentTypes()) {
+					t.Fatalf("expected default supported content types, got %+v", cfg.Tools.WebFetch.SupportedContentTypes)
+				}
 			},
 		},
 	}
@@ -362,6 +380,12 @@ func TestConfigApplyDefaultsFillsMissingFields(t *testing.T) {
 	if !filepath.IsAbs(cfg.Workdir) {
 		t.Fatalf("expected absolute workdir, got %q", cfg.Workdir)
 	}
+	if cfg.Tools.WebFetch.MaxResponseBytes != DefaultWebFetchMaxResponseBytes {
+		t.Fatalf("expected default webfetch max_response_bytes %d, got %d", DefaultWebFetchMaxResponseBytes, cfg.Tools.WebFetch.MaxResponseBytes)
+	}
+	if len(cfg.Tools.WebFetch.SupportedContentTypes) != len(DefaultWebFetchSupportedContentTypes()) {
+		t.Fatalf("expected default supported content types, got %+v", cfg.Tools.WebFetch.SupportedContentTypes)
+	}
 }
 
 func TestConfigValidateFailures(t *testing.T) {
@@ -423,6 +447,24 @@ func TestConfigValidateFailures(t *testing.T) {
 			}(),
 			expectErr: "model is empty",
 		},
+		{
+			name: "invalid webfetch max response bytes",
+			config: func() *Config {
+				cfg := validConfig.Clone()
+				cfg.Tools.WebFetch.MaxResponseBytes = 0
+				return &cfg
+			}(),
+			expectErr: "max_response_bytes must be greater than 0",
+		},
+		{
+			name: "invalid webfetch supported content types",
+			config: func() *Config {
+				cfg := validConfig.Clone()
+				cfg.Tools.WebFetch.SupportedContentTypes = []string{""}
+				return &cfg
+			}(),
+			expectErr: "supported_content_types[0] is empty",
+		},
 	}
 
 	for _, tt := range tests {
@@ -542,6 +584,8 @@ func TestLoaderLoadAndSaveRoundTrip(t *testing.T) {
 			cfg.Providers[i].Model = "gpt-5.4"
 		}
 	}
+	cfg.Tools.WebFetch.MaxResponseBytes = 1024
+	cfg.Tools.WebFetch.SupportedContentTypes = []string{"text/html", "application/json"}
 	if err := loader.Save(context.Background(), cfg); err != nil {
 		t.Fatalf("Save() error = %v", err)
 	}
@@ -553,6 +597,12 @@ func TestLoaderLoadAndSaveRoundTrip(t *testing.T) {
 	if reloaded.CurrentModel != "gpt-5.4" {
 		t.Fatalf("expected current model %q, got %q", "gpt-5.4", reloaded.CurrentModel)
 	}
+	if reloaded.Tools.WebFetch.MaxResponseBytes != 1024 {
+		t.Fatalf("expected max_response_bytes %d, got %d", 1024, reloaded.Tools.WebFetch.MaxResponseBytes)
+	}
+	if len(reloaded.Tools.WebFetch.SupportedContentTypes) != 2 {
+		t.Fatalf("expected persisted supported content types, got %+v", reloaded.Tools.WebFetch.SupportedContentTypes)
+	}
 }
 
 func TestNormalizeWorkdirAndClone(t *testing.T) {
@@ -621,9 +671,13 @@ func TestNormalizeWorkdirAndClone(t *testing.T) {
 	cfg := Default()
 	cloned := cfg.Clone()
 	cloned.CurrentModel = "modified"
+	cloned.Tools.WebFetch.SupportedContentTypes[0] = "application/json"
 	if cfg.CurrentModel == cloned.CurrentModel {
 		t.Fatalf("expected clone to be independent from source")
 	}
+	if cfg.Tools.WebFetch.SupportedContentTypes[0] == cloned.Tools.WebFetch.SupportedContentTypes[0] {
+		t.Fatalf("expected webfetch supported content types to be cloned")
+	}
 }
 
 func TestManagerHelperMethodsAndReloads(t *testing.T) {