diff --git a/README.md b/README.md index 615a545..59d415a 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ | Conda 镜像 | 代理 `https://repo.anaconda.com` | | RubyGems 镜像 | 代理 `https://rubygems.org` | | Cargo 镜像 | 代理 `https://static.crates.io/crates` | -| NuGet 镜像 | 代理 `https://api.nuget.org/v3/index.json` | +| NuGet 镜像 | 代理 `https://api.nuget.org/v3` | | APT 镜像 | 代理 `https://deb.debian.org/debian` | | Alpine 镜像 | 代理 `https://dl-cdn.alpinelinux.org/alpine` | | Homebrew 镜像 | 代理 `https://ghcr.io/v2/homebrew/core` | @@ -30,6 +30,7 @@ | Docker v2 Auth | Token 认证代理，让 `docker pull` 不依赖直接访问上游 | | 镜像搜索 | Dashboard 搜索 npm、Docker Hub、PyPI、Conda、RubyGems、Cargo、NuGet 包 | | IP 限流 | 滚动时间窗口限流防滥用，白名单免限速，黑名单直接拒绝 | +| 缓存 TTL 可调 | 面板可逐镜像配置缓存 TTL（0=永久缓存），Git 代理缓存 TTL 独立可调 | | 健康检查端点 | `/health` 端点供 Kubernetes/Docker probe 使用 | | Prometheus 指标 | `/metrics` 端点暴露缓存命中率等指标 | | 优雅关闭 | 收到 SIGTERM 后等待请求完成再退出 | diff --git a/configs/devbox.yaml b/configs/devbox.yaml index a54bd8e..9aca4ae 100644 --- a/configs/devbox.yaml +++ b/configs/devbox.yaml @@ -43,7 +43,7 @@ mirrors: hf: enabled: true upstream: "https://huggingface.co" - cache_ttl: "7d" + cache_ttl: "0" conda: enabled: true upstream: "https://repo.anaconda.com" @@ -58,7 +58,7 @@ mirrors: cache_ttl: "7d" nuget: enabled: true - upstream: "https://api.nuget.org/v3/index.json" + upstream: "https://api.nuget.org/v3" cache_ttl: "7d" apt: enabled: true diff --git a/internal/alert/alert.go b/internal/alert/alert.go index 06224fe..4c8e8b3 100644 --- a/internal/alert/alert.go +++ b/internal/alert/alert.go @@ -46,7 +46,11 @@ func (w *WebhookProvider) Send(subject, body string) error { if err != nil { return fmt.Errorf("alert webhook: %w", err) } - resp.Body.Close() + defer resp.Body.Close() + // Treat non-2xx responses as failures so misconfigured webhooks surface. + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("alert webhook returned %d", resp.StatusCode) + } return nil } diff --git a/internal/config/config.go b/internal/config/config.go index 9ac36a7..3892d83 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -19,6 +19,10 @@ type Config struct { Logging LoggingConfig `yaml:"logging"` RateLimit RateLimitConfig `yaml:"rate_limit"` Alerts AlertConfig `yaml:"alerts"` + + // AuthTokenOrig holds the auth_token value from the yaml file so that + // Save() can avoid persisting an env-injected DEVBOX_AUTH_TOKEN. + AuthTokenOrig string `yaml:"-"` } type AlertConfig struct { @@ -74,11 +78,21 @@ type RateLimitConfig struct { } func (cfg *Config) Save(path string) error { + // Never persist an env-injected auth token: DEVBOX_AUTH_TOKEN must stay + // out of the config file. Save the file value (or empty) instead. + orig := cfg.Server.AuthToken + cfg.Server.AuthToken = cfg.AuthTokenOrig data, err := yaml.Marshal(cfg) + cfg.Server.AuthToken = orig if err != nil { return fmt.Errorf("marshal config: %w", err) } - return os.WriteFile(path, data, 0600) + + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0600); err != nil { + return fmt.Errorf("write config: %w", err) + } + return os.Rename(tmp, path) } func Load(path string) (*Config, error) { @@ -92,6 +106,9 @@ func Load(path string) (*Config, error) { return nil, fmt.Errorf("parse config: %w", err) } + // Remember the file value before env overrides replace it. + cfg.AuthTokenOrig = cfg.Server.AuthToken + applyDefaults(cfg) applyEnvOverrides(cfg) @@ -150,11 +167,11 @@ func applyDefaults(cfg *Config) { "quay": {Enabled: true, Upstream: "https://quay.io", CacheTTL: "0"}, "mcr": {Enabled: true, Upstream: "https://mcr.microsoft.com", CacheTTL: "0"}, "ghapi": {Enabled: true, Upstream: "https://api.github.com", CacheTTL: "0"}, - "hf": {Enabled: true, Upstream: "https://huggingface.co", CacheTTL: "7d"}, + "hf": {Enabled: true, Upstream: "https://huggingface.co", CacheTTL: "0"}, "conda": {Enabled: true, Upstream: "https://repo.anaconda.com", CacheTTL: "30d"}, "rubygems": {Enabled: true, Upstream: "https://rubygems.org", CacheTTL: "7d"}, "cargo": {Enabled: true, Upstream: "https://static.crates.io/crates", CacheTTL: "7d"}, - "nuget": {Enabled: true, Upstream: "https://api.nuget.org/v3/index.json", CacheTTL: "7d"}, + "nuget": {Enabled: true, Upstream: "https://api.nuget.org/v3", CacheTTL: "7d"}, "apt": {Enabled: true, Upstream: "https://deb.debian.org/debian", CacheTTL: "0"}, "alpine": {Enabled: true, Upstream: "https://dl-cdn.alpinelinux.org/alpine", CacheTTL: "0"}, "homebrew": {Enabled: true, Upstream: "https://ghcr.io/v2/homebrew/core", CacheTTL: "0"}, @@ -185,7 +202,11 @@ func applyDefaults(cfg *Config) { func applyEnvOverrides(cfg *Config) { if v := os.Getenv("DEVBOX_SERVER_PORT"); v != "" { - cfg.Server.Port = mustInt(v) + if n, err := mustInt(v); err == nil { + cfg.Server.Port = n + } else { + slog.Error("invalid env DEVBOX_SERVER_PORT, keeping file value", "value", v) + } } if v := os.Getenv("DEVBOX_AUTH_TOKEN"); v != "" { cfg.Server.AuthToken = v @@ -200,13 +221,25 @@ func applyEnvOverrides(cfg *Config) { cfg.Cache.MaxSize = v } if v := os.Getenv("DEVBOX_LOGGING_RETENTION_DAYS"); v != "" { - cfg.Logging.RetentionDays = mustInt(v) + if n, err := mustInt(v); err == nil { + cfg.Logging.RetentionDays = n + } else { + slog.Error("invalid env DEVBOX_LOGGING_RETENTION_DAYS, keeping file value", "value", v) + } } if v := os.Getenv("DEVBOX_RATE_LIMIT_ENABLED"); v != "" { - cfg.RateLimit.Enabled = mustBool(v) + if b, err := mustBool(v); err == nil { + cfg.RateLimit.Enabled = b + } else { + slog.Error("invalid env DEVBOX_RATE_LIMIT_ENABLED, keeping file value", "value", v) + } } if v := os.Getenv("DEVBOX_RATE_LIMIT_RATE"); v != "" { - cfg.RateLimit.Rate = mustInt(v) + if n, err := mustInt(v); err == nil { + cfg.RateLimit.Rate = n + } else { + slog.Error("invalid env DEVBOX_RATE_LIMIT_RATE, keeping file value", "value", v) + } } if v := os.Getenv("DEVBOX_RATE_LIMIT_INTERVAL"); v != "" { cfg.RateLimit.Interval = v @@ -224,9 +257,13 @@ func applyEnvOverrides(cfg *Config) { } enabled := os.Getenv(fmt.Sprintf("DEVBOX_MIRROR_%s_ENABLED", strings.ToUpper(name))) if enabled != "" { - m := cfg.Mirrors[name] - m.Enabled = mustBool(enabled) - cfg.Mirrors[name] = m + if b, err := mustBool(enabled); err == nil { + m := cfg.Mirrors[name] + m.Enabled = b + cfg.Mirrors[name] = m + } else { + slog.Error("invalid env DEVBOX_MIRROR_"+strings.ToUpper(name)+"_ENABLED, keeping file value", "value", enabled) + } } } } @@ -284,6 +321,10 @@ func ParseDuration(s string) (time.Duration, error) { if err != nil { return 0, fmt.Errorf("invalid days: %s", s) } + // Guard against int overflow when converting days to nanoseconds. + if days > int((1<<63-1)/(24*int(time.Hour))) { + return 0, fmt.Errorf("duration too large: %s", s) + } return time.Duration(days) * 24 * time.Hour, nil } return time.ParseDuration(s) @@ -304,6 +345,10 @@ func parseCacheMaxSize(cfg *Config) error { if err != nil { return fmt.Errorf("invalid cache max_size: %s", s) } + // Guard against multiplication overflow (e.g. huge values in GB). + if sf.mul != 1 && val > (1<<63-1)/sf.mul { + return fmt.Errorf("cache max_size too large: %s", s) + } cfg.Cache.MaxSizeBytes = val * sf.mul return nil } @@ -311,12 +356,10 @@ func parseCacheMaxSize(cfg *Config) error { return fmt.Errorf("invalid cache max_size suffix: %s", s) } -func mustInt(s string) int { - v, _ := strconv.Atoi(s) - return v +func mustInt(s string) (int, error) { + return strconv.Atoi(s) } -func mustBool(s string) bool { - v, _ := strconv.ParseBool(s) - return v +func mustBool(s string) (bool, error) { + return strconv.ParseBool(s) } diff --git a/internal/dashboard/dashboard.go b/internal/dashboard/dashboard.go index 9496036..678635b 100644 --- a/internal/dashboard/dashboard.go +++ b/internal/dashboard/dashboard.go @@ -11,6 +11,7 @@ import ( "time" "devbox/internal/alert" + "devbox/internal/config" "devbox/internal/mirror" "devbox/internal/store" ) @@ -20,6 +21,7 @@ type Dashboard struct { authToken string publicURL string rlConfig RateLimitConfigAccessor + gitProxyConfig GitProxyConfigAccessor saveConfig func() error alertEngine *alert.Engine releaseHTTP *http.Client @@ -51,6 +53,11 @@ type RateLimitConfigAccessor interface { SetRateLimitBlacklist(list []string) } +type GitProxyConfigAccessor interface { + GetCacheTTL() string + SetCacheTTL(ttl string) error +} + type RateLimitConfigView struct { Enabled bool `json:"enabled"` Rate int `json:"rate"` @@ -83,6 +90,10 @@ func (d *Dashboard) SetRateLimitConfigAccessor(rl RateLimitConfigAccessor) { d.rlConfig = rl } +func (d *Dashboard) SetGitProxyConfigAccessor(gp GitProxyConfigAccessor) { + d.gitProxyConfig = gp +} + func (d *Dashboard) GetRateLimitConfig() RateLimitConfigView { if d.rlConfig == nil { return RateLimitConfigView{} @@ -364,7 +375,7 @@ func (d *Dashboard) MirrorConfigHandler(w http.ResponseWriter, r *http.Request) if d.saveConfig != nil { if err := d.saveConfig(); err != nil { slog.Error("failed to persist config", "error", err) - writeJSON(w, map[string]string{"status": "persist_failed", "error": err.Error()}) + http.Error(w, "persist_failed", http.StatusInternalServerError) return } } @@ -442,6 +453,16 @@ func (d *Dashboard) RateLimitConfigHandler(w http.ResponseWriter, r *http.Reques http.Error(w, "invalid request", http.StatusBadRequest) return } + // Validate before mutating anything: a bad interval or rate must + // not silently keep the old values while reporting success. + if _, err := config.ParseDuration(req.Interval); err != nil { + http.Error(w, "invalid interval", http.StatusBadRequest) + return + } + if req.Rate <= 0 { + http.Error(w, "rate must be positive", http.StatusBadRequest) + return + } d.rlConfig.SetRateLimitEnabled(req.Enabled) d.rlConfig.SetRateLimitRate(req.Rate) d.rlConfig.SetRateLimitInterval(req.Interval) @@ -461,11 +482,55 @@ func (d *Dashboard) RateLimitConfigHandler(w http.ResponseWriter, r *http.Reques http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } +func (d *Dashboard) GitProxyConfigHandler(w http.ResponseWriter, r *http.Request) { + if !d.checkAuth(r) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + if d.gitProxyConfig == nil { + http.Error(w, "git proxy not available", http.StatusNotFound) + return + } + + if r.Method == http.MethodGet { + writeJSON(w, map[string]string{"cacheTTL": d.gitProxyConfig.GetCacheTTL()}) + return + } + + if r.Method == http.MethodPut { + var req struct { + CacheTTL string `json:"cacheTTL"` + } + if !readJSON(r, &req) { + http.Error(w, "invalid request", http.StatusBadRequest) + return + } + if err := d.gitProxyConfig.SetCacheTTL(req.CacheTTL); err != nil { + http.Error(w, "invalid cacheTTL: "+err.Error(), http.StatusBadRequest) + return + } + if d.saveConfig != nil { + if err := d.saveConfig(); err != nil { + slog.Error("failed to persist git proxy config", "error", err) + http.Error(w, "persist_failed", http.StatusInternalServerError) + return + } + } + writeJSON(w, map[string]string{"cacheTTL": d.gitProxyConfig.GetCacheTTL()}) + return + } + + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) +} + func (d *Dashboard) checkAuth(r *http.Request) bool { if d.authToken == "" { return true // no auth required } token := r.Header.Get("Authorization") - token = strings.TrimPrefix(token, "Bearer ") + if len(token) > 7 && strings.EqualFold(token[:7], "Bearer ") { + token = token[7:] + } return subtle.ConstantTimeCompare([]byte(token), []byte(d.authToken)) == 1 } diff --git a/internal/dashboard/release.go b/internal/dashboard/release.go index aac2865..66728b6 100644 --- a/internal/dashboard/release.go +++ b/internal/dashboard/release.go @@ -312,15 +312,18 @@ func (d *Dashboard) ReleaseDownloadHandler(w http.ResponseWriter, r *http.Reques contentType = "application/octet-stream" } w.Header().Set("Content-Type", contentType) - if ticket.asset.Size > 0 { - w.Header().Set("Content-Length", strconv.FormatInt(ticket.asset.Size, 10)) - } w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": ticket.asset.Name})) w.Header().Set("Cache-Control", "no-store") w.Header().Set("X-Content-Type-Options", "nosniff") w.Header().Set("Accept-Ranges", "bytes") if r.Method == http.MethodHead { + // Only HEAD preflight can rely on the ticket metadata: the actual + // GET streams the upstream body and must not pre-announce a length + // that could mismatch the real content. + if ticket.asset.Size > 0 { + w.Header().Set("Content-Length", strconv.FormatInt(ticket.asset.Size, 10)) + } w.WriteHeader(http.StatusOK) return } @@ -354,6 +357,12 @@ func (d *Dashboard) ReleaseDownloadHandler(w http.ResponseWriter, r *http.Reques defer resp.Body.Close() if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { slog.Warn("release download upstream status", "source_id", ticket.sourceID, "status", resp.StatusCode) + if resp.StatusCode >= 400 && resp.StatusCode < 500 { + // Client-side errors (404 deleted asset, 403 expired URL, ...) + // are pass-through so the caller sees the real reason. + http.Error(w, resp.Status, resp.StatusCode) + return + } http.Error(w, "release download upstream returned "+resp.Status, http.StatusBadGateway) return } diff --git a/internal/gitproxy/gitproxy.go b/internal/gitproxy/gitproxy.go index af3dbc3..71adc69 100644 --- a/internal/gitproxy/gitproxy.go +++ b/internal/gitproxy/gitproxy.go @@ -1,12 +1,15 @@ package gitproxy import ( + "fmt" "io" "log/slog" "net/http" "strings" + "sync" "time" + "devbox/internal/config" "devbox/internal/mirror" ) @@ -25,6 +28,7 @@ type GitProxy struct { rawUpstream string cacheTTL time.Duration cache *mirror.Cache + mu sync.RWMutex } func New(githubUpstream, gitlabUpstream, rawUpstream string, cacheTTL time.Duration, cache *mirror.Cache) *GitProxy { @@ -37,6 +41,37 @@ func New(githubUpstream, gitlabUpstream, rawUpstream string, cacheTTL time.Durat } } +func (gp *GitProxy) CacheTTL() string { + gp.mu.RLock() + defer gp.mu.RUnlock() + d := gp.cacheTTL + if d == 0 { + return "0" + } + secs := d / time.Second + if secs%86400 == 0 { + return fmt.Sprintf("%dd", secs/86400) + } + if secs%3600 == 0 { + return fmt.Sprintf("%dh", secs/3600) + } + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) +} + +func (gp *GitProxy) SetCacheTTL(ttl string) error { + d, err := config.ParseDuration(ttl) + if err != nil { + return err + } + gp.mu.Lock() + defer gp.mu.Unlock() + gp.cacheTTL = d + return nil +} + func (gp *GitProxy) Handler(w http.ResponseWriter, r *http.Request) { path := r.URL.Path @@ -86,9 +121,17 @@ func isRawRequest(path string) bool { } func (gp *GitProxy) proxyArchive(w http.ResponseWriter, r *http.Request, upstream, path string) { - if gp.cache != nil && gp.cacheTTL > 0 { + gp.mu.RLock() + cacheTTL := gp.cacheTTL + gp.mu.RUnlock() + if gp.cache != nil && cacheTTL > 0 { target := upstream + path if resp, err := client.Head(target); err == nil { + if resp.StatusCode == http.StatusNotFound { + resp.Body.Close() + http.Error(w, "not found", http.StatusNotFound) + return + } if isHTMLResponse(resp) { resp.Body.Close() slog.Warn("gitproxy blocking HTML response (cache)", "path", path) @@ -99,7 +142,7 @@ func (gp *GitProxy) proxyArchive(w http.ResponseWriter, r *http.Request, upstrea } orig := r.URL.Path r.URL.Path = path - gp.cache.ProxyHTTP(w, r, upstream, gp.cacheTTL) + gp.cache.ProxyHTTP(w, r, upstream, cacheTTL) r.URL.Path = orig return } @@ -121,9 +164,17 @@ func (gp *GitProxy) proxyArchive(w http.ResponseWriter, r *http.Request, upstrea } func (gp *GitProxy) proxyRaw(w http.ResponseWriter, r *http.Request, path string) { - if gp.cache != nil && gp.cacheTTL > 0 { + gp.mu.RLock() + cacheTTL := gp.cacheTTL + gp.mu.RUnlock() + if gp.cache != nil && cacheTTL > 0 { target := gp.rawUpstream + path if resp, err := client.Head(target); err == nil { + if resp.StatusCode == http.StatusNotFound { + resp.Body.Close() + http.Error(w, "not found", http.StatusNotFound) + return + } if isHTMLResponse(resp) { resp.Body.Close() slog.Warn("gitproxy blocking HTML response (cache)", "path", path) @@ -134,7 +185,7 @@ func (gp *GitProxy) proxyRaw(w http.ResponseWriter, r *http.Request, path string } orig := r.URL.Path r.URL.Path = path - gp.cache.ProxyHTTP(w, r, gp.rawUpstream, gp.cacheTTL) + gp.cache.ProxyHTTP(w, r, gp.rawUpstream, cacheTTL) r.URL.Path = orig return } @@ -156,10 +207,13 @@ func (gp *GitProxy) proxyRaw(w http.ResponseWriter, r *http.Request, path string } func (gp *GitProxy) proxySmartHTTP(w http.ResponseWriter, r *http.Request, upstream, path string) { - if gp.cache != nil && gp.cacheTTL > 0 && r.Method == "GET" { + gp.mu.RLock() + cacheTTL := gp.cacheTTL + gp.mu.RUnlock() + if gp.cache != nil && cacheTTL > 0 && r.Method == "GET" { orig := r.URL.Path r.URL.Path = path - gp.cache.ProxyHTTP(w, r, upstream, gp.cacheTTL) + gp.cache.ProxyHTTP(w, r, upstream, cacheTTL) r.URL.Path = orig return } @@ -173,6 +227,7 @@ func (gp *GitProxy) proxySmartHTTP(w http.ResponseWriter, r *http.Request, upstr http.Error(w, "request error", http.StatusInternalServerError) return } + newReq.ContentLength = r.ContentLength copyRequestHeaders(newReq, r) resp, err := client.Do(newReq) diff --git a/internal/mirror/alpine.go b/internal/mirror/alpine.go index bdb336c..cb19468 100644 --- a/internal/mirror/alpine.go +++ b/internal/mirror/alpine.go @@ -57,7 +57,10 @@ func (a *AlpineMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (a *AlpineMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -73,6 +76,7 @@ func (a *AlpineMirror) ProxyHandler(cache *Cache) http.HandlerFunc { a.mu.RLock() upstream := a.upstream a.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/alpine") cache.ProxyStream(w, r, upstream) } diff --git a/internal/mirror/apt.go b/internal/mirror/apt.go index e9ecccd..a8f19e1 100644 --- a/internal/mirror/apt.go +++ b/internal/mirror/apt.go @@ -57,7 +57,10 @@ func (a *AptMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (a *AptMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -73,6 +76,7 @@ func (a *AptMirror) ProxyHandler(cache *Cache) http.HandlerFunc { a.mu.RLock() upstream := a.upstream a.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/apt") cache.ProxyStream(w, r, upstream) } diff --git a/internal/mirror/cache.go b/internal/mirror/cache.go index daa3ddc..0bc00af 100644 --- a/internal/mirror/cache.go +++ b/internal/mirror/cache.go @@ -89,9 +89,9 @@ func (c *Cache) Set(key string, data []byte, hdr http.Header, ttl time.Duration) path := c.keyPath(key) - // Subtract old file size if overwriting + oldSize := int64(0) if info, err := os.Stat(path); err == nil { - c.usedBytes -= info.Size() + oldSize = info.Size() } if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { @@ -115,6 +115,10 @@ func (c *Cache) Set(key string, data []byte, hdr http.Header, ttl time.Duration) hf, err := os.Create(hdrPath) if err != nil { slog.Warn("cache header write error", "path", hdrPath, "error", err) + os.Remove(path) + // os.Create already truncated/replaced any previous file at path, so + // drop its size contribution to keep usedBytes in sync with disk. + c.usedBytes -= oldSize return } for k, vv := range hdr { @@ -124,18 +128,25 @@ func (c *Cache) Set(key string, data []byte, hdr http.Header, ttl time.Duration) } hf.Close() + expPath := path + ".exp" if ttl > 0 { - expPath := path + ".exp" ef, err := os.Create(expPath) if err != nil { slog.Warn("cache expiry write error", "path", expPath, "error", err) + os.Remove(path) + os.Remove(hdrPath) + c.usedBytes -= oldSize return } fmt.Fprintf(ef, "%d", time.Now().Add(ttl).Unix()) ef.Close() + } else { + // ttl<=0 means never expire: drop any stale expiry marker from an + // earlier TTL so shorter/zero TTLs take effect immediately. + os.Remove(expPath) } - c.usedBytes += int64(len(data)) + c.usedBytes += int64(len(data)) - oldSize if c.maxBytes > 0 && c.usedBytes > c.maxBytes { c.evictLRU() @@ -211,14 +222,14 @@ func (c *Cache) IsExpired(key string) bool { } func (c *Cache) ProxyHTTP(w http.ResponseWriter, r *http.Request, upstream string, ttl time.Duration) { - key := upstream + r.URL.Path + "?" + r.URL.RawQuery + key := r.Method + "|" + upstream + r.URL.Path + "?" + r.URL.RawQuery // Authenticated requests must not read or write the shared cache: // the key has no identity component, so cached responses fetched with // one client's credentials would leak to everyone else. authenticated := r.Header.Get("Authorization") != "" - if !authenticated && !c.IsExpired(key) { + if r.Method == http.MethodGet && !authenticated && !c.IsExpired(key) { data, hdr, ok := c.Get(key) if ok { for k, vv := range hdr { @@ -242,6 +253,7 @@ func (c *Cache) ProxyHTTP(w http.ResponseWriter, r *http.Request, upstream strin http.Error(w, "request error", http.StatusInternalServerError) return } + newReq.ContentLength = r.ContentLength copyRequestHeaders(newReq, r) resp, err := proxyClient.Do(newReq) @@ -255,7 +267,7 @@ func (c *Cache) ProxyHTTP(w http.ResponseWriter, r *http.Request, upstream strin // Stream to client while spooling cacheable (200 only, so partial // 206 responses never poison the cache) responses to a temp file. - cacheable := resp.StatusCode == http.StatusOK && ttl >= 0 && !authenticated + cacheable := resp.StatusCode == http.StatusOK && ttl >= 0 && !authenticated && r.Method == http.MethodGet path := c.keyPath(key) var tmp *os.File if cacheable { @@ -288,15 +300,24 @@ func (c *Cache) ProxyHTTP(w http.ResponseWriter, r *http.Request, upstream strin c.mu.Lock() defer c.mu.Unlock() + // Stat before the atomic rename: after it, path already points at the new + // file whose size equals written, which would cancel out the accounting + // below and keep usedBytes from ever growing past maxBytes. + oldSize := int64(0) if info, err := os.Stat(path); err == nil { - c.usedBytes -= info.Size() + oldSize = info.Size() } if err := os.Rename(tmp.Name(), path); err != nil { slog.Warn("cache rename error", "path", path, "error", err) return } - c.writeCacheMeta(path, respHdr, ttl) - c.usedBytes += written + if !c.writeCacheMeta(path, respHdr, ttl) { + // writeCacheMeta already removed the cache file on failure; the + // replaced old entry is gone too, so drop its size contribution. + c.usedBytes -= oldSize + return + } + c.usedBytes += written - oldSize if c.maxBytes > 0 && c.usedBytes > c.maxBytes { c.evictLRU() } @@ -313,6 +334,7 @@ func (c *Cache) ProxyStream(w http.ResponseWriter, r *http.Request, upstream str http.Error(w, "request error", http.StatusInternalServerError) return } + newReq.ContentLength = r.ContentLength copyRequestHeaders(newReq, r) resp, err := proxyClient.Do(newReq) @@ -355,13 +377,15 @@ func isHopByHopHeader(k string) bool { } // writeCacheMeta persists response headers and (optionally) an expiry file -// for a cached entry. Callers must hold c.mu. -func (c *Cache) writeCacheMeta(path string, hdr http.Header, ttl time.Duration) { +// for a cached entry. It returns false when metadata could not be written; +// in that case the cache file itself is removed. Callers must hold c.mu. +func (c *Cache) writeCacheMeta(path string, hdr http.Header, ttl time.Duration) bool { hdrPath := path + ".hdr" hf, err := os.Create(hdrPath) if err != nil { slog.Warn("cache header write error", "path", hdrPath, "error", err) - return + os.Remove(path) + return false } for k, vv := range hdr { for _, v := range vv { @@ -370,16 +394,23 @@ func (c *Cache) writeCacheMeta(path string, hdr http.Header, ttl time.Duration) } hf.Close() + expPath := path + ".exp" if ttl > 0 { - expPath := path + ".exp" ef, err := os.Create(expPath) if err != nil { slog.Warn("cache expiry write error", "path", expPath, "error", err) - return + os.Remove(path) + os.Remove(hdrPath) + return false } fmt.Fprintf(ef, "%d", time.Now().Add(ttl).Unix()) ef.Close() + } else { + // Never-expire entries must not keep a stale expiry marker from a + // previous TTL, otherwise shortening the TTL to 0 would not stick. + os.Remove(expPath) } + return true } func (c *Cache) Hits() int64 { return c.hits.Load() } diff --git a/internal/mirror/cache_test.go b/internal/mirror/cache_test.go index a901ccf..78720dd 100644 --- a/internal/mirror/cache_test.go +++ b/internal/mirror/cache_test.go @@ -258,6 +258,59 @@ func TestCacheUsedBytesTracking(t *testing.T) { } } +func TestCacheProxyHTTPUsedBytesTracking(t *testing.T) { + body := []byte("proxy-tracked-body") + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(body) + })) + defer ts.Close() + + dir := t.TempDir() + c := NewCache(dir, 1<<20) + + req1 := httptest.NewRequest("GET", "/track-1", nil) + w1 := httptest.NewRecorder() + c.ProxyHTTP(w1, req1, ts.URL, time.Minute) + if c.usedBytes != int64(len(body)) { + t.Fatalf("expected usedBytes=%d after proxy write, got %d", len(body), c.usedBytes) + } + + req2 := httptest.NewRequest("GET", "/track-2", nil) + w2 := httptest.NewRecorder() + c.ProxyHTTP(w2, req2, ts.URL, time.Minute) + if c.usedBytes != int64(2*len(body)) { + t.Fatalf("expected usedBytes=%d after second write, got %d", 2*len(body), c.usedBytes) + } +} + +func TestCacheProxyHTTPEvictsBeyondMaxBytes(t *testing.T) { + body := []byte("01234567890123") // 14 bytes + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(body) + })) + defer ts.Close() + + dir := t.TempDir() + c := NewCache(dir, 20) + + req1 := httptest.NewRequest("GET", "/evict-1", nil) + w1 := httptest.NewRecorder() + c.ProxyHTTP(w1, req1, ts.URL, time.Minute) + + req2 := httptest.NewRequest("GET", "/evict-2", nil) + w2 := httptest.NewRecorder() + c.ProxyHTTP(w2, req2, ts.URL, time.Minute) + + if c.usedBytes > 20 { + t.Fatalf("expected usedBytes <= maxBytes(20) after eviction, got %d", c.usedBytes) + } + if c.usedBytes != 14 { + t.Fatalf("expected usedBytes=14 (one entry evicted), got %d", c.usedBytes) + } +} + func TestCacheDir(t *testing.T) { dir := t.TempDir() c := NewCache(dir, 1<<20) diff --git a/internal/mirror/cargo.go b/internal/mirror/cargo.go index fb120a4..7361be3 100644 --- a/internal/mirror/cargo.go +++ b/internal/mirror/cargo.go @@ -57,7 +57,10 @@ func (c *CargoMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (c *CargoMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -74,6 +77,7 @@ func (c *CargoMirror) ProxyHandler(cache *Cache) http.HandlerFunc { upstream := c.upstream cacheTTL := c.cacheTTL c.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/cargo") cache.ProxyHTTP(w, r, upstream, cacheTTL) } diff --git a/internal/mirror/conda.go b/internal/mirror/conda.go index 64d3851..2d15339 100644 --- a/internal/mirror/conda.go +++ b/internal/mirror/conda.go @@ -57,7 +57,10 @@ func (c *CondaMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (c *CondaMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -74,6 +77,7 @@ func (c *CondaMirror) ProxyHandler(cache *Cache) http.HandlerFunc { upstream := c.upstream cacheTTL := c.cacheTTL c.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/conda") cache.ProxyHTTP(w, r, upstream, cacheTTL) } diff --git a/internal/mirror/cran.go b/internal/mirror/cran.go index 6f115e3..7faf7d9 100644 --- a/internal/mirror/cran.go +++ b/internal/mirror/cran.go @@ -57,7 +57,10 @@ func (c *CranMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (c *CranMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -74,6 +77,7 @@ func (c *CranMirror) ProxyHandler(cache *Cache) http.HandlerFunc { upstream := c.upstream cacheTTL := c.cacheTTL c.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/cran") cache.ProxyHTTP(w, r, upstream, cacheTTL) } diff --git a/internal/mirror/docker.go b/internal/mirror/docker.go index 4f44986..a1c1596 100644 --- a/internal/mirror/docker.go +++ b/internal/mirror/docker.go @@ -57,7 +57,10 @@ func (d *DockerMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (d *DockerMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -73,6 +76,7 @@ func (d *DockerMirror) ProxyHandler(cache *Cache) http.HandlerFunc { d.mu.RLock() upstream := d.upstream d.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/docker") cache.ProxyStream(w, r, upstream) } diff --git a/internal/mirror/ghcr.go b/internal/mirror/ghcr.go index d4af93f..a914641 100644 --- a/internal/mirror/ghcr.go +++ b/internal/mirror/ghcr.go @@ -57,7 +57,10 @@ func (g *GhcrMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (g *GhcrMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -73,6 +76,7 @@ func (g *GhcrMirror) ProxyHandler(cache *Cache) http.HandlerFunc { g.mu.RLock() upstream := g.upstream g.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/ghcr") cache.ProxyStream(w, r, upstream) } diff --git a/internal/mirror/github_api.go b/internal/mirror/github_api.go index 2d3eb2a..6a83a0a 100644 --- a/internal/mirror/github_api.go +++ b/internal/mirror/github_api.go @@ -57,7 +57,10 @@ func (g *GithubAPIMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (g *GithubAPIMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -74,6 +77,7 @@ func (g *GithubAPIMirror) ProxyHandler(cache *Cache) http.HandlerFunc { upstream := g.upstream cacheTTL := g.cacheTTL g.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/ghapi") cache.ProxyHTTP(w, r, upstream, cacheTTL) } diff --git a/internal/mirror/golang.go b/internal/mirror/golang.go index c7d3fdb..d3ca43a 100644 --- a/internal/mirror/golang.go +++ b/internal/mirror/golang.go @@ -57,7 +57,10 @@ func (g *GolangMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (g *GolangMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -74,6 +77,7 @@ func (g *GolangMirror) ProxyHandler(cache *Cache) http.HandlerFunc { upstream := g.upstream cacheTTL := g.cacheTTL g.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/golang") cache.ProxyHTTP(w, r, upstream, cacheTTL) } diff --git a/internal/mirror/homebrew.go b/internal/mirror/homebrew.go index bafa235..632f2f4 100644 --- a/internal/mirror/homebrew.go +++ b/internal/mirror/homebrew.go @@ -57,7 +57,10 @@ func (h *HomebrewMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (h *HomebrewMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -73,6 +76,7 @@ func (h *HomebrewMirror) ProxyHandler(cache *Cache) http.HandlerFunc { h.mu.RLock() upstream := h.upstream h.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/homebrew") cache.ProxyStream(w, r, upstream) } diff --git a/internal/mirror/huggingface.go b/internal/mirror/huggingface.go index 1e03b97..e9cc625 100644 --- a/internal/mirror/huggingface.go +++ b/internal/mirror/huggingface.go @@ -57,7 +57,10 @@ func (h *HfMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (h *HfMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -73,6 +76,7 @@ func (h *HfMirror) ProxyHandler(cache *Cache) http.HandlerFunc { h.mu.RLock() upstream := h.upstream h.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/hf") // HuggingFace files can be large (model weights), use streaming proxy cache.ProxyStream(w, r, upstream) diff --git a/internal/mirror/mcr.go b/internal/mirror/mcr.go index d07463f..fd3badc 100644 --- a/internal/mirror/mcr.go +++ b/internal/mirror/mcr.go @@ -57,7 +57,10 @@ func (m *McrMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (m *McrMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -73,6 +76,7 @@ func (m *McrMirror) ProxyHandler(cache *Cache) http.HandlerFunc { m.mu.RLock() upstream := m.upstream m.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/mcr") cache.ProxyStream(w, r, upstream) } diff --git a/internal/mirror/npm.go b/internal/mirror/npm.go index b69c341..debba89 100644 --- a/internal/mirror/npm.go +++ b/internal/mirror/npm.go @@ -57,7 +57,10 @@ func (n *NpmMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (n *NpmMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -74,6 +77,7 @@ func (n *NpmMirror) ProxyHandler(cache *Cache) http.HandlerFunc { upstream := n.upstream cacheTTL := n.cacheTTL n.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/npm") if r.URL.Path == "" || r.URL.Path == "/" { r.URL.Path = "/" diff --git a/internal/mirror/nuget.go b/internal/mirror/nuget.go index 2a0b3ae..bff464e 100644 --- a/internal/mirror/nuget.go +++ b/internal/mirror/nuget.go @@ -57,7 +57,10 @@ func (n *NuGetMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (n *NuGetMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -74,6 +77,7 @@ func (n *NuGetMirror) ProxyHandler(cache *Cache) http.HandlerFunc { upstream := n.upstream cacheTTL := n.cacheTTL n.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/nuget") cache.ProxyHTTP(w, r, upstream, cacheTTL) } @@ -91,7 +95,12 @@ func (n *NuGetMirror) SetCacheTTL(ttl string) error { } func (n *NuGetMirror) HealthCheck() error { - resp, err := HealthGet(strings.TrimRight(n.Upstream(), "/")) + target := strings.TrimRight(n.Upstream(), "/") + if strings.HasSuffix(target, "/v3") { + // The v3 API root itself is not a valid endpoint; probe its index. + target += "/index.json" + } + resp, err := HealthGet(target) if err != nil { return fmt.Errorf("nuget upstream unreachable: %w", err) } diff --git a/internal/mirror/pypi.go b/internal/mirror/pypi.go index 6e9fd30..0671157 100644 --- a/internal/mirror/pypi.go +++ b/internal/mirror/pypi.go @@ -57,7 +57,10 @@ func (p *PypiMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (p *PypiMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -74,6 +77,7 @@ func (p *PypiMirror) ProxyHandler(cache *Cache) http.HandlerFunc { upstream := p.upstream cacheTTL := p.cacheTTL p.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/pypi") cache.ProxyHTTP(w, r, upstream, cacheTTL) } diff --git a/internal/mirror/quay.go b/internal/mirror/quay.go index 1d9dd6d..3f62833 100644 --- a/internal/mirror/quay.go +++ b/internal/mirror/quay.go @@ -57,7 +57,10 @@ func (q *QuayMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (q *QuayMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -73,6 +76,7 @@ func (q *QuayMirror) ProxyHandler(cache *Cache) http.HandlerFunc { q.mu.RLock() upstream := q.upstream q.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/quay") cache.ProxyStream(w, r, upstream) } diff --git a/internal/mirror/rubygems.go b/internal/mirror/rubygems.go index 4ffb595..73220cd 100644 --- a/internal/mirror/rubygems.go +++ b/internal/mirror/rubygems.go @@ -57,7 +57,10 @@ func (rg *RubyGemsMirror) CacheTTL() string { if secs%3600 == 0 { return fmt.Sprintf("%dh", secs/3600) } - return fmt.Sprintf("%dm", secs/60) + if secs%60 == 0 { + return fmt.Sprintf("%dm", secs/60) + } + return fmt.Sprintf("%ds", secs) } func (rg *RubyGemsMirror) ApplyConfig(cfg config.MirrorConfig) { @@ -74,6 +77,7 @@ func (rg *RubyGemsMirror) ProxyHandler(cache *Cache) http.HandlerFunc { upstream := rg.upstream cacheTTL := rg.cacheTTL rg.mu.RUnlock() + upstream = strings.TrimRight(upstream, "/") r.URL.Path = strings.TrimPrefix(r.URL.Path, "/rubygems") cache.ProxyHTTP(w, r, upstream, cacheTTL) } diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index 1e7e472..e535f9e 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -1,6 +1,7 @@ package ratelimit import ( + "log/slog" "net" "net/http" "strings" @@ -46,12 +47,20 @@ func (l *Limiter) Allow(r *http.Request) bool { ip := l.extractIP(r) ipNet := parseIP(ip) + // Blacklist stays authoritative even for degenerate configs: a + // blacklisted IP must be rejected regardless of quota settings. for _, cidr := range l.blacklist { if cidr.Contains(ipNet) { return false } } + // Guard against degenerate configs (rate <= 0 or window <= 0): refusing + // everything or nothing would take the whole service down with it. + if l.limit <= 0 || l.window <= 0 { + return true + } + for _, cidr := range l.whitelist { if cidr.Contains(ipNet) { return true @@ -109,9 +118,11 @@ func parseCIDRList(list []string) []*net.IPNet { } } _, ipNet, err := net.ParseCIDR(entry) - if err == nil { - nets = append(nets, ipNet) + if err != nil { + slog.Warn("ratelimit: invalid CIDR entry ignored", "entry", entry, "error", err) + continue } + nets = append(nets, ipNet) } return nets } diff --git a/internal/server/server.go b/internal/server/server.go index 034e5a1..a979682 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -138,6 +138,7 @@ func New(cfg *config.Config, configPath string, frontDir string) (*Server, error } dash.SetRateLimitConfigAccessor(s) + dash.SetGitProxyConfigAccessor(s) dash.SetSaveConfig(s.saveConfig) return s, nil @@ -165,6 +166,7 @@ func (s *Server) Start() error { mux.HandleFunc("/api/stats/logs", s.dash.LogHandler) mux.HandleFunc("/api/config/mirrors", s.dash.MirrorConfigHandler) mux.HandleFunc("/api/config/ratelimit", s.dash.RateLimitConfigHandler) + mux.HandleFunc("/api/config/gitproxy", s.dash.GitProxyConfigHandler) mux.HandleFunc("/api/config/public", s.dash.PublicConfigHandler) mux.HandleFunc("/api/auth/login", s.dash.LoginHandler) mux.HandleFunc("/api/auth/check", s.dash.AuthCheckHandler) @@ -186,13 +188,31 @@ func (s *Server) Start() error { if s.frontDir != "" { if _, err := os.Stat(s.frontDir); err == nil { fileServer := http.FileServer(http.Dir(s.frontDir)) + frontDir := filepath.Clean(s.frontDir) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { path := r.URL.Path if path == "/" { path = "/index.html" } - if _, err := os.Stat(s.frontDir + path); err != nil { - r.URL.Path = "/index.html" + full := filepath.Join(frontDir, filepath.FromSlash(path)) + if full != frontDir && !strings.HasPrefix(full, frontDir+string(filepath.Separator)) { + // Path escapes the frontend dir (".." traversal): fall back. + full = filepath.Join(frontDir, "index.html") + } + if _, err := os.Stat(full); err != nil { + // SPA fallback. Serve index.html directly instead of + // rewriting r.URL.Path: http.FileServer 301-redirects a + // rewritten "/index.html" path to "./" (breaking deep + // links like /mirrors). + data, rerr := os.ReadFile(filepath.Join(frontDir, "index.html")) + if rerr != nil { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(data) + return } fileServer.ServeHTTP(w, r) }) @@ -208,16 +228,15 @@ func (s *Server) Start() error { go s.tokenCacheCleanup() port, accessLog := s.serverConfigSnapshot() - rl := s.rateLimitConfigSnapshot() addr := fmt.Sprintf(":%d", port) slog.Info("DevBox starting", "addr", addr) handler := logMiddleware(mux, accessLog) handler = s.bodyLimitMiddleware(handler) - if s.getLimiter() != nil { - handler = s.rateLimitMiddleware(handler) - slog.Info("rate limiting enabled", "rate", rl.Rate, "interval", rl.Interval) - } + // Always install the rate limit middleware; it internally no-ops when + // rate limiting is disabled, so enabling it at runtime works without a + // server restart. + handler = s.rateLimitMiddleware(handler) srv := &http.Server{ Addr: addr, @@ -340,6 +359,7 @@ func (s *Server) proxyRegistryRequest(w http.ResponseWriter, r *http.Request, ta http.Error(w, "request error", http.StatusInternalServerError) return } + upstreamReq.ContentLength = r.ContentLength // Copy client headers (except Host) for k, vv := range r.Header { @@ -381,10 +401,14 @@ func (s *Server) proxyRegistryRequest(w http.ResponseWriter, r *http.Request, ta } resp.Body.Close() - // Retry without token — let upstream give fresh 401 - slog.Warn("registry token rejected, retrying without token") - s.proxyRegistryRequest(w, r, target, "", depth+1) - return + // Retry without token — let upstream give fresh 401. + // Only replay body-less requests: http.Client.Do closed r.Body + // on the first attempt, so uploads cannot be safely retried. + if r.Method == http.MethodGet || r.ContentLength == 0 { + slog.Warn("registry token rejected, retrying without token") + s.proxyRegistryRequest(w, r, target, "", depth+1) + return + } } // If 401 without token, get a new token with scope and retry @@ -403,8 +427,10 @@ func (s *Server) proxyRegistryRequest(w http.ResponseWriter, r *http.Request, ta if err == nil && newToken != "" { resp.Body.Close() slog.Info("registry got new token, retrying", "scope", scope) - s.proxyRegistryRequest(w, r, target, newToken, depth+1) - return + if r.Method == http.MethodGet || r.ContentLength == 0 { + s.proxyRegistryRequest(w, r, target, newToken, depth+1) + return + } } } @@ -484,14 +510,16 @@ func (s *Server) getRegistryToken(regInfo registryInfo, scope string) (string, e return "", fmt.Errorf("empty token") } - // Cache token (use expires_in minus 5 min safety margin, min 2 min) + // Cache token with a 5 min safety margin; never cache a token for + // longer than it is actually valid (clamping to a minimum would serve + // expired tokens and cause 401 retry storms). ttl := time.Duration(tokenResp.ExpiresIn) * time.Second if ttl == 0 { ttl = 5 * time.Minute } ttl = ttl - 5*time.Minute - if ttl < 2*time.Minute { - ttl = 2 * time.Minute + if ttl <= 0 { + return tokenResp.Token, nil } s.tokenCache.mu.Lock() @@ -648,13 +676,17 @@ func (s *Server) configWatcher() { if fi.ModTime().Equal(lastMtime) { continue } - lastMtime = fi.ModTime() + newMtime := fi.ModTime() cfg, err := config.Load(s.configPath) if err != nil { + // Do not advance lastMtime on failure: keep polling this file + // state so a transient write (e.g. atomic rename in progress) + // retries until the config becomes valid again. slog.Error("config hot-reload failed to load, keeping old config", "error", err) continue } + lastMtime = newMtime s.applyRuntimeConfig(cfg) slog.Info("config hot-reloaded", "path", s.configPath) @@ -722,6 +754,8 @@ func (s *Server) saveConfig() error { s.cfg.Mirrors[m.Name()] = existing } + s.cfg.GitProxy.CacheTTL = s.gitProxy.CacheTTL() + if err := s.cfg.Save(s.configPath); err != nil { slog.Error("config failed to persist", "error", err) return err @@ -779,6 +813,20 @@ func (s *Server) SetRateLimitBlacklist(list []string) { s.rebuildLimiter() } +func (s *Server) GetCacheTTL() string { + s.cfgMu.RLock() + gp := s.gitProxy + s.cfgMu.RUnlock() + return gp.CacheTTL() +} + +func (s *Server) SetCacheTTL(ttl string) error { + s.cfgMu.RLock() + gp := s.gitProxy + s.cfgMu.RUnlock() + return gp.SetCacheTTL(ttl) +} + func (s *Server) rebuildLimiter() { rl := s.rateLimitConfigSnapshot() s.limiterMu.Lock() @@ -834,7 +882,11 @@ func (s *Server) wrapWithDynamicStats(nameFor func(*http.Request) string, handle start := time.Now() handler(sw, r) name := nameFor(r) - if err := s.store.RecordTraffic(name, r.Method, r.URL.Path, 0, sw.bytesWritten, sw.status); err != nil { + bytesIn := int64(0) + if r.ContentLength > 0 { + bytesIn = r.ContentLength + } + if err := s.store.RecordTraffic(name, r.Method, r.URL.Path, int(bytesIn), sw.bytesWritten, sw.status); err != nil { slog.Warn("record traffic failed", "name", name, "error", err) } slog.Info("request stats", @@ -884,7 +936,9 @@ type statusWriter struct { } func (sw *statusWriter) WriteHeader(code int) { - sw.status = code + if sw.status == 0 { + sw.status = code + } sw.ResponseWriter.WriteHeader(code) } @@ -919,8 +973,12 @@ func (s *Server) rateLimitMiddleware(next http.Handler) http.Handler { // force attempts are throttled like everything else. if path == "/api/auth/login" { // no-op: fall through to rate limiting - } else if strings.HasPrefix(path, "/api/") || strings.HasPrefix(path, "/v2/") || - strings.HasPrefix(path, "/token") || path == "/" || path == "/health" || + } else if (strings.HasPrefix(path, "/api/") && + path != "/api/search" && + !strings.HasPrefix(path, "/api/release-sources") && + !strings.HasPrefix(path, "/api/release-download")) || + strings.HasPrefix(path, "/v2/") || strings.HasPrefix(path, "/token") || + path == "/" || path == "/health" || strings.HasSuffix(path, ".html") || strings.HasSuffix(path, ".js") || strings.HasSuffix(path, ".css") || strings.HasSuffix(path, ".ico") { next.ServeHTTP(w, r) diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go index a9f7cc7..531ac8b 100644 --- a/internal/store/sqlite.go +++ b/internal/store/sqlite.go @@ -151,7 +151,7 @@ type TrafficSummary struct { func (s *Store) GetTrafficSummary(from, to time.Time) ([]TrafficSummary, error) { rows, err := s.db.Query( - "SELECT mirror, COUNT(*), SUM(bytes_in), SUM(bytes_out) FROM traffic WHERE created_at BETWEEN ? AND ? GROUP BY mirror", + "SELECT mirror, COUNT(*), SUM(bytes_in), SUM(bytes_out) FROM traffic WHERE created_at BETWEEN ? AND ? GROUP BY mirror ORDER BY mirror", from.Format(time.RFC3339), to.Format(time.RFC3339), ) if err != nil { @@ -199,7 +199,7 @@ func (s *Store) GetTrafficDaily(from, to time.Time) ([]TrafficHourly, error) { func (s *Store) GetTrafficWeekly(from, to time.Time) ([]TrafficHourly, error) { rows, err := s.db.Query( - "SELECT strftime('%Y-%m-%dT00:00:00Z', created_at, 'weekday 1', '-7 days'), mirror, COUNT(*), SUM(bytes_out) FROM traffic WHERE created_at BETWEEN ? AND ? GROUP BY strftime('%Y-%W', created_at), mirror ORDER BY 1", + "SELECT strftime('%Y-%m-%dT00:00:00Z', created_at, 'weekday 1', '-7 days'), mirror, COUNT(*), SUM(bytes_out) FROM traffic WHERE created_at BETWEEN ? AND ? GROUP BY strftime('%Y-%W', created_at), mirror ORDER BY strftime('%Y-%W', created_at), mirror", from.Format(time.RFC3339), to.Format(time.RFC3339), ) if err != nil { @@ -318,6 +318,9 @@ func (s *Store) Close() error { } func (s *Store) PurgeOldTraffic(retentionDays int) (int64, error) { + if retentionDays <= 0 { + return 0, nil + } cutoff := time.Now().AddDate(0, 0, -retentionDays).Format(time.RFC3339) result, err := s.db.Exec("DELETE FROM traffic WHERE created_at < ?", cutoff) if err != nil { diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 413a742..d5b2853 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -14,7 +14,9 @@ api.interceptors.response.use( localStorage.removeItem("devbox_token"); setToken(""); if (window.location.pathname !== "/login") { - window.location.href = "/login"; + window.location.href = + "/login?redirect=" + + encodeURIComponent(window.location.pathname + window.location.search); } } return Promise.reject(error); @@ -97,6 +99,14 @@ export async function getPublicConfig() { return api.get("/config/public").then((r) => r.data); } +export async function getGitProxyConfig() { + return api.get("/config/gitproxy").then((r) => r.data); +} + +export async function updateGitProxyConfig(cacheTTL: string) { + return api.put("/config/gitproxy", { cacheTTL }).then((r) => r.data); +} + export async function searchMirrors( q: string, registry?: string, diff --git a/web/src/router/index.ts b/web/src/router/index.ts index 7cf58a6..ef85aa7 100644 --- a/web/src/router/index.ts +++ b/web/src/router/index.ts @@ -39,7 +39,8 @@ router.beforeEach(async (to) => { const required = await checkAuthRequired() if (!required) return true - return '/login' + // Remember where the user was headed so the login page can bounce back. + return { path: '/login', query: { redirect: to.fullPath } } }) export default router \ No newline at end of file diff --git a/web/src/views/Dashboard.vue b/web/src/views/Dashboard.vue index d0b8f16..f3ed7a0 100644 --- a/web/src/views/Dashboard.vue +++ b/web/src/views/Dashboard.vue @@ -193,6 +193,7 @@ async function switchChartMode() { } let granularityRequestId = 0 +let logsRequestId = 0 async function switchGranularity(level: 'hourly' | 'daily' | 'weekly') { const requestId = ++granularityRequestId @@ -209,7 +210,17 @@ async function switchGranularity(level: 'hourly' | 'daily' | 'weekly') { } async function refreshLogs() { - logs.value = await getRecentLogs(50) + const requestId = ++logsRequestId + try { + const data = await getRecentLogs(50) + if (requestId !== logsRequestId) return + logs.value = Array.isArray(data) ? data : [] + errorMsg.value = '' + } catch (e: any) { + if (requestId === logsRequestId) { + errorMsg.value = e.response?.statusText || '日志加载失败' + } + } } function formatBytes(b: number) { diff --git a/web/src/views/Login.vue b/web/src/views/Login.vue index 0a10258..96eb7e9 100644 --- a/web/src/views/Login.vue +++ b/web/src/views/Login.vue @@ -15,7 +15,16 @@ async function submit() { error.value = '' try { await login(password.value) - window.location.href = '/' + // Only allow relative-path redirects to avoid open-redirect abuse. + // Reject protocol-relative URLs too: `//evil.com` and `/\evil.com` + // start with "/" but navigate to an external host. + const redirect = new URLSearchParams(window.location.search).get('redirect') + const safe = + redirect && + redirect.startsWith('/') && + !redirect.startsWith('//') && + !redirect.startsWith('/\\') + window.location.href = safe ? redirect : '/' } catch { error.value = '认证被拒绝，请检查 AUTH_TOKEN' } diff --git a/web/src/views/Mirrors.vue b/web/src/views/Mirrors.vue index 1f3b10e..cbfc5d1 100644 --- a/web/src/views/Mirrors.vue +++ b/web/src/views/Mirrors.vue @@ -1,6 +1,7 @@