From 16a6a925cc31459b51c297e531a5ab5555776ac2 Mon Sep 17 00:00:00 2001 From: Cory LaNou Date: Sun, 15 Mar 2026 22:18:17 -0500 Subject: [PATCH 1/4] feat(link): add URL reachability checking for link validation Add opt-in link validation that checks whether URLs in documents are reachable via HTTP. Implements HEAD-first-then-GET checking with per-host rate limiting, URL caching, configurable timeouts, exclude patterns, redirect following, and 429 retry-after support. Closes #101 --- cmd/hype/cli/export.go | 29 +++++ go.mod | 2 +- link.go | 27 +++++ link_check_config.go | 27 +++++ link_check_config_test.go | 25 +++++ link_check_error.go | 38 +++++++ link_test.go | 121 +++++++++++++++++++++ link_validator.go | 215 ++++++++++++++++++++++++++++++++++++++ link_validator_test.go | 209 ++++++++++++++++++++++++++++++++++++ parser.go | 32 +++--- 10 files changed, 710 insertions(+), 15 deletions(-) create mode 100644 link_check_config.go create mode 100644 link_check_config_test.go create mode 100644 link_check_error.go create mode 100644 link_validator.go create mode 100644 link_validator_test.go diff --git a/cmd/hype/cli/export.go b/cmd/hype/cli/export.go index 0b82caf..384566a 100644 --- a/cmd/hype/cli/export.go +++ b/cmd/hype/cli/export.go @@ -40,6 +40,11 @@ type Export struct { NoCSS bool // output raw HTML without styling ListThemes bool // list available themes and exit + CheckLinks bool // enable link reachability checking + LinkTimeout time.Duration // per-link check timeout + LinkExclude string // comma-separated URL patterns to exclude + LinkRate float64 // requests per second per host + flags *flag.FlagSet mu sync.RWMutex @@ -108,6 +113,9 @@ Examples: hype export -themes hype export -f README.md -format markdown -timeout=10s hype export -f input.md -format markdown -o README.md + hype export -f hype.md -check-links + hype export -f hype.md -check-links -link-timeout=20s -link-rate=1 + hype export -f hype.md -check-links -link-exclude="https://localhost:*,https://internal.example.com/*" ` if err := cmd.validate(); err != nil { @@ -132,6 +140,10 @@ Examples: cmd.flags.StringVar(&cmd.CustomCSS, "css", "", "path to custom CSS file for HTML export") cmd.flags.BoolVar(&cmd.NoCSS, "no-css", false, "output raw HTML without styling") cmd.flags.BoolVar(&cmd.ListThemes, "themes", false, "list available themes and exit") + cmd.flags.BoolVar(&cmd.CheckLinks, "check-links", false, "enable URL reachability checking for links") + cmd.flags.DurationVar(&cmd.LinkTimeout, "link-timeout", 10*time.Second, "per-link check timeout") + cmd.flags.StringVar(&cmd.LinkExclude, "link-exclude", "", "comma-separated URL patterns to exclude from link checking") + cmd.flags.Float64Var(&cmd.LinkRate, "link-rate", 2, "max requests per second per host for link checking") cmd.flags.Usage = func() { fmt.Fprintf(stderr, "Usage of %s:\n", os.Args[0]) @@ -272,6 +284,23 @@ func (cmd *Export) execute(ctx context.Context, pwd string) error { p.FS = parserFS } + if cmd.CheckLinks { + cfg := hype.DefaultLinkCheckConfig() + cfg.Enabled = true + cfg.Timeout = cmd.LinkTimeout + cfg.RatePerHost = cmd.LinkRate + if cmd.LinkExclude != "" { + for _, pattern := range strings.Split(cmd.LinkExclude, ",") { + pattern = strings.TrimSpace(pattern) + if pattern != "" { + cfg.ExcludePatterns = append(cfg.ExcludePatterns, pattern) + } + } + } + p.LinkCheck = cfg + p.LinkValidator = hype.NewLinkValidator(cfg) + } + p.Root = filepath.Join(filepath.Dir(mp), fileDir) doc, err := p.ParseFile(fileName) diff --git a/go.mod b/go.mod index ebe79a9..96e3615 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/stretchr/testify v1.11.1 golang.org/x/net v0.51.0 golang.org/x/sync v0.19.0 + golang.org/x/time v0.14.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -168,7 +169,6 @@ require ( golang.org/x/sys v0.41.0 // indirect golang.org/x/term v0.40.0 // indirect golang.org/x/text v0.34.0 // indirect - golang.org/x/time v0.14.0 // indirect golang.org/x/tools v0.42.0 // indirect google.golang.org/api v0.266.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect diff --git a/link.go b/link.go index 8ddd620..f5c8631 100644 --- a/link.go +++ b/link.go @@ -1,6 +1,7 @@ package hype import ( + "context" "encoding/json" "fmt" "strings" @@ -56,6 +57,32 @@ func (l *Link) MD() string { return fmt.Sprintf("[%s](%s)", l.Children().MD(), h) } +func (l *Link) Execute(ctx context.Context, doc *Document) error { + if l == nil { + return ErrIsNil("link") + } + + if doc == nil || doc.Parser == nil { + return nil + } + + if !doc.Parser.LinkCheck.Enabled { + return nil + } + + href, err := l.Href() + if err != nil { + return nil + } + + v := doc.Parser.LinkValidator + if v == nil { + return nil + } + + return v.Check(ctx, href) +} + func NewLink(el *Element) (*Link, error) { if el == nil { return nil, ErrIsNil("element") diff --git a/link_check_config.go b/link_check_config.go new file mode 100644 index 0000000..0f5433c --- /dev/null +++ b/link_check_config.go @@ -0,0 +1,27 @@ +package hype + +import "time" + +type LinkCheckConfig struct { + Enabled bool + Timeout time.Duration + AcceptedCodes []int + ExcludePatterns []string + RatePerHost float64 + RateBurst int + MaxRedirects int +} + +func DefaultLinkCheckConfig() LinkCheckConfig { + return LinkCheckConfig{ + Enabled: false, + Timeout: 10 * time.Second, + AcceptedCodes: []int{ + 200, 201, 202, 203, 204, + 301, 302, 307, 308, + }, + RatePerHost: 2, + RateBurst: 1, + MaxRedirects: 10, + } +} diff --git a/link_check_config_test.go b/link_check_config_test.go new file mode 100644 index 0000000..5ef35ec --- /dev/null +++ b/link_check_config_test.go @@ -0,0 +1,25 @@ +package hype + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func Test_DefaultLinkCheckConfig(t *testing.T) { + t.Parallel() + r := require.New(t) + + cfg := DefaultLinkCheckConfig() + + r.False(cfg.Enabled) + r.Equal(10*time.Second, cfg.Timeout) + r.Contains(cfg.AcceptedCodes, 200) + r.Contains(cfg.AcceptedCodes, 301) + r.Contains(cfg.AcceptedCodes, 302) + r.Equal(float64(2), cfg.RatePerHost) + r.Equal(1, cfg.RateBurst) + r.Equal(10, cfg.MaxRedirects) + r.Empty(cfg.ExcludePatterns) +} diff --git a/link_check_error.go b/link_check_error.go new file mode 100644 index 0000000..d62629f --- /dev/null +++ b/link_check_error.go @@ -0,0 +1,38 @@ +package hype + +import ( + "encoding/json" + "fmt" +) + +type LinkCheckError struct { + URL string + StatusCode int + Err error +} + +func (e LinkCheckError) Error() string { + if e.StatusCode > 0 { + return fmt.Sprintf("link check failed for %q: status %d", e.URL, e.StatusCode) + } + if e.Err != nil { + return fmt.Sprintf("link check failed for %q: %s", e.URL, e.Err) + } + return fmt.Sprintf("link check failed for %q", e.URL) +} + +func (e LinkCheckError) Unwrap() error { + return e.Err +} + +func (e LinkCheckError) MarshalJSON() ([]byte, error) { + m := map[string]any{ + "type": "link_check_error", + "url": e.URL, + "status_code": e.StatusCode, + } + if e.Err != nil { + m["error"] = e.Err.Error() + } + return json.MarshalIndent(m, "", " ") +} diff --git a/link_test.go b/link_test.go index 367328c..f46102d 100644 --- a/link_test.go +++ b/link_test.go @@ -1,7 +1,11 @@ package hype import ( + "context" + "net/http" + "net/http/httptest" "testing" + "testing/fstest" "github.com/stretchr/testify/require" ) @@ -21,3 +25,120 @@ func Test_Link_MarshalJSON(t *testing.T) { testJSON(t, "link", link) } + +func Test_Link_Execute_Disabled(t *testing.T) { + t.Parallel() + r := require.New(t) + + link := &Link{Element: NewEl("a", nil)} + r.NoError(link.Set("href", "https://example.com")) + + doc := &Document{ + FS: fstest.MapFS{}, + Parser: &Parser{}, + } + + err := link.Execute(context.Background(), doc) + r.NoError(err) +} + +func Test_Link_Execute_HTTP_OK(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + + link := &Link{Element: NewEl("a", nil)} + r.NoError(link.Set("href", srv.URL+"/page")) + + doc := &Document{ + FS: fstest.MapFS{}, + Parser: &Parser{ + LinkCheck: cfg, + LinkValidator: NewLinkValidator(cfg), + }, + } + + err := link.Execute(context.Background(), doc) + r.NoError(err) +} + +func Test_Link_Execute_HTTP_NotFound(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + + link := &Link{Element: NewEl("a", nil)} + r.NoError(link.Set("href", srv.URL+"/missing")) + + doc := &Document{ + FS: fstest.MapFS{}, + Parser: &Parser{ + LinkCheck: cfg, + LinkValidator: NewLinkValidator(cfg), + }, + } + + err := link.Execute(context.Background(), doc) + r.Error(err) + + var lce LinkCheckError + r.ErrorAs(err, &lce) + r.Equal(404, lce.StatusCode) +} + +func Test_Link_Execute_NonHTTP(t *testing.T) { + t.Parallel() + r := require.New(t) + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + + link := &Link{Element: NewEl("a", nil)} + r.NoError(link.Set("href", "mailto:test@example.com")) + + doc := &Document{ + FS: fstest.MapFS{}, + Parser: &Parser{ + LinkCheck: cfg, + LinkValidator: NewLinkValidator(cfg), + }, + } + + err := link.Execute(context.Background(), doc) + r.NoError(err) +} + +func Test_Link_Execute_NoHref(t *testing.T) { + t.Parallel() + r := require.New(t) + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + + link := &Link{Element: NewEl("a", nil)} + + doc := &Document{ + FS: fstest.MapFS{}, + Parser: &Parser{ + LinkCheck: cfg, + LinkValidator: NewLinkValidator(cfg), + }, + } + + err := link.Execute(context.Background(), doc) + r.NoError(err) +} diff --git a/link_validator.go b/link_validator.go new file mode 100644 index 0000000..51d6850 --- /dev/null +++ b/link_validator.go @@ -0,0 +1,215 @@ +package hype + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "path" + "slices" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/time/rate" +) + +type LinkValidator struct { + Config LinkCheckConfig + client *http.Client + + mu sync.RWMutex + cache map[string]error + limiters map[string]*rate.Limiter +} + +func NewLinkValidator(cfg LinkCheckConfig) *LinkValidator { + v := &LinkValidator{ + Config: cfg, + cache: make(map[string]error), + limiters: make(map[string]*rate.Limiter), + } + + v.client = &http.Client{ + Timeout: cfg.Timeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= cfg.MaxRedirects { + return fmt.Errorf("too many redirects (%d)", len(via)) + } + return nil + }, + } + + return v +} + +func (v *LinkValidator) Check(ctx context.Context, rawURL string) error { + if v.shouldSkip(rawURL) { + return nil + } + + v.mu.RLock() + if err, ok := v.cache[rawURL]; ok { + v.mu.RUnlock() + return err + } + v.mu.RUnlock() + + u, err := url.Parse(rawURL) + if err != nil { + lce := LinkCheckError{URL: rawURL, Err: err} + v.cacheResult(rawURL, lce) + return lce + } + + limiter := v.limiterFor(u.Host) + if err := limiter.Wait(ctx); err != nil { + return LinkCheckError{URL: rawURL, Err: err} + } + + checkErr := v.doCheck(ctx, rawURL) + v.cacheResult(rawURL, checkErr) + return checkErr +} + +func (v *LinkValidator) shouldSkip(rawURL string) bool { + u, err := url.Parse(rawURL) + if err != nil { + return false + } + + switch u.Scheme { + case "http", "https": + case "": + return true + default: + return true + } + + for _, pattern := range v.Config.ExcludePatterns { + matched, err := path.Match(pattern, rawURL) + if err == nil && matched { + return true + } + if strings.Contains(rawURL, strings.ReplaceAll(pattern, "*", "")) { + continue + } + } + + return false +} + +func (v *LinkValidator) limiterFor(host string) *rate.Limiter { + v.mu.Lock() + defer v.mu.Unlock() + + if l, ok := v.limiters[host]; ok { + return l + } + + l := rate.NewLimiter(rate.Limit(v.Config.RatePerHost), v.Config.RateBurst) + v.limiters[host] = l + return l +} + +func (v *LinkValidator) doCheck(ctx context.Context, rawURL string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, rawURL, nil) + if err != nil { + return LinkCheckError{URL: rawURL, Err: err} + } + req.Header.Set("User-Agent", "hype-link-checker/1.0") + + resp, err := v.client.Do(req) + if err != nil { + return LinkCheckError{URL: rawURL, Err: err} + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + + if resp.StatusCode == http.StatusMethodNotAllowed { + return v.doGet(ctx, rawURL) + } + + if resp.StatusCode == http.StatusTooManyRequests { + return v.retryAfter(ctx, rawURL, resp) + } + + if !v.isAccepted(resp.StatusCode) { + return LinkCheckError{URL: rawURL, StatusCode: resp.StatusCode} + } + + return nil +} + +func (v *LinkValidator) doGet(ctx context.Context, rawURL string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return LinkCheckError{URL: rawURL, Err: err} + } + req.Header.Set("User-Agent", "hype-link-checker/1.0") + + resp, err := v.client.Do(req) + if err != nil { + return LinkCheckError{URL: rawURL, Err: err} + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + + if resp.StatusCode == http.StatusTooManyRequests { + return v.retryAfter(ctx, rawURL, resp) + } + + if !v.isAccepted(resp.StatusCode) { + return LinkCheckError{URL: rawURL, StatusCode: resp.StatusCode} + } + + return nil +} + +func (v *LinkValidator) retryAfter(ctx context.Context, rawURL string, resp *http.Response) error { + wait := parseRetryAfter(resp.Header.Get("Retry-After")) + wait = min(wait, 60*time.Second) + if wait <= 0 { + wait = 5 * time.Second + } + + select { + case <-ctx.Done(): + return LinkCheckError{URL: rawURL, Err: ctx.Err()} + case <-time.After(wait): + } + + return v.doGet(ctx, rawURL) +} + +func parseRetryAfter(val string) time.Duration { + if val == "" { + return 0 + } + + if secs, err := strconv.Atoi(val); err == nil { + return time.Duration(secs) * time.Second + } + + if t, err := http.ParseTime(val); err == nil { + d := time.Until(t) + if d < 0 { + return 0 + } + return d + } + + return 0 +} + +func (v *LinkValidator) isAccepted(code int) bool { + return slices.Contains(v.Config.AcceptedCodes, code) +} + +func (v *LinkValidator) cacheResult(rawURL string, err error) { + v.mu.Lock() + defer v.mu.Unlock() + v.cache[rawURL] = err +} diff --git a/link_validator_test.go b/link_validator_test.go new file mode 100644 index 0000000..59c4905 --- /dev/null +++ b/link_validator_test.go @@ -0,0 +1,209 @@ +package hype + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func Test_LinkValidator_Check_Success(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/page") + r.NoError(err) +} + +func Test_LinkValidator_Check_HeadFallbackToGet(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Method == http.MethodHead { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/page") + r.NoError(err) +} + +func Test_LinkValidator_Check_NotFound(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/missing") + r.Error(err) + + var lce LinkCheckError + r.ErrorAs(err, &lce) + r.Equal(404, lce.StatusCode) +} + +func Test_LinkValidator_Check_SkipNonHTTP(t *testing.T) { + t.Parallel() + r := require.New(t) + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + r.NoError(v.Check(context.Background(), "mailto:test@example.com")) + r.NoError(v.Check(context.Background(), "tel:+1234567890")) + r.NoError(v.Check(context.Background(), "ftp://example.com/file")) + r.NoError(v.Check(context.Background(), "#section-anchor")) +} + +func Test_LinkValidator_Check_SkipExcluded(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + cfg.ExcludePatterns = []string{srv.URL + "/*"} + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/should-skip") + r.NoError(err) +} + +func Test_LinkValidator_Check_Cache(t *testing.T) { + t.Parallel() + r := require.New(t) + + var hitCount atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + hitCount.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + u := srv.URL + "/cached" + r.NoError(v.Check(context.Background(), u)) + r.NoError(v.Check(context.Background(), u)) + + r.Equal(int32(1), hitCount.Load()) +} + +func Test_LinkValidator_Check_RetryAfter(t *testing.T) { + t.Parallel() + r := require.New(t) + + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + n := calls.Add(1) + if n <= 2 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/rate-limited") + r.NoError(err) +} + +func Test_LinkValidator_Check_TooManyRedirects(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + http.Redirect(w, req, req.URL.Path, http.StatusMovedPermanently) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + cfg.MaxRedirects = 3 + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/loop") + r.Error(err) +} + +func Test_LinkValidator_Check_Timeout(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + time.Sleep(5 * time.Second) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + cfg.Timeout = 100 * time.Millisecond + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/slow") + r.Error(err) +} + +func Test_LinkValidator_Check_FollowRedirects(t *testing.T) { + t.Parallel() + r := require.New(t) + + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + n := calls.Add(1) + if n == 1 { + http.Redirect(w, req, "/final", http.StatusMovedPermanently) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/redirect") + r.NoError(err) +} diff --git a/parser.go b/parser.go index b2da315..3c1d315 100644 --- a/parser.go +++ b/parser.go @@ -25,16 +25,18 @@ type ParseElementFn func(p *Parser, el *Element) (Nodes, error) type Parser struct { fs.FS - DisablePages bool - DocIDGen func() (string, error) // default: uuid.NewV4().String() - Filename string // only set when Parser.ParseFile() is used - NodeParsers map[Atom]ParseElementFn - NowFn func() time.Time // default: time.Now() - PreParsers PreParsers - Root string - Section int - Vars syncx.Map[string, any] - Contents []byte // a copy of the contents being parsed - set just before parsing + DisablePages bool + DocIDGen func() (string, error) // default: uuid.NewV4().String() + Filename string // only set when Parser.ParseFile() is used + LinkCheck LinkCheckConfig + LinkValidator *LinkValidator + NodeParsers map[Atom]ParseElementFn + NowFn func() time.Time // default: time.Now() + PreParsers PreParsers + Root string + Section int + Vars syncx.Map[string, any] + Contents []byte // a copy of the contents being parsed - set just before parsing mu sync.RWMutex } @@ -402,10 +404,12 @@ func (p *Parser) Sub(dir string) (*Parser, error) { } p2 := &Parser{ - FS: p.FS, - Root: filepath.Join(p.Root, dir), - PreParsers: p.PreParsers, - NodeParsers: p.NodeParsers, + FS: p.FS, + Root: filepath.Join(p.Root, dir), + PreParsers: p.PreParsers, + NodeParsers: p.NodeParsers, + LinkCheck: p.LinkCheck, + LinkValidator: p.LinkValidator, } if len(dir) == 0 || dir == "." { From 194cd6de5a383b0299ac0d9e3dd500bfeea8296d Mon Sep 17 00:00:00 2001 From: Cory LaNou Date: Mon, 16 Mar 2026 09:24:39 -0500 Subject: [PATCH 2/4] fix(link): address codex review findings (pass 1) - Fix exclude patterns to match nested URL paths using prefix matching - Deduplicate concurrent checks for the same URL via inflight map - Reject non-positive link-rate values with safe defaults --- link_validator.go | 62 ++++++++++++++++++++++++++++++------------ link_validator_test.go | 47 ++++++++++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 19 deletions(-) diff --git a/link_validator.go b/link_validator.go index 51d6850..4b78536 100644 --- a/link_validator.go +++ b/link_validator.go @@ -6,7 +6,6 @@ import ( "io" "net/http" "net/url" - "path" "slices" "strconv" "strings" @@ -22,13 +21,22 @@ type LinkValidator struct { mu sync.RWMutex cache map[string]error + inflight map[string]chan struct{} limiters map[string]*rate.Limiter } func NewLinkValidator(cfg LinkCheckConfig) *LinkValidator { + if cfg.RatePerHost <= 0 { + cfg.RatePerHost = 2 + } + if cfg.RateBurst <= 0 { + cfg.RateBurst = 1 + } + v := &LinkValidator{ Config: cfg, cache: make(map[string]error), + inflight: make(map[string]chan struct{}), limiters: make(map[string]*rate.Limiter), } @@ -50,30 +58,56 @@ func (v *LinkValidator) Check(ctx context.Context, rawURL string) error { return nil } - v.mu.RLock() + v.mu.Lock() if err, ok := v.cache[rawURL]; ok { + v.mu.Unlock() + return err + } + + if ch, ok := v.inflight[rawURL]; ok { + v.mu.Unlock() + select { + case <-ch: + case <-ctx.Done(): + return LinkCheckError{URL: rawURL, Err: ctx.Err()} + } + v.mu.RLock() + err := v.cache[rawURL] v.mu.RUnlock() return err } - v.mu.RUnlock() + + ch := make(chan struct{}) + v.inflight[rawURL] = ch + v.mu.Unlock() u, err := url.Parse(rawURL) if err != nil { lce := LinkCheckError{URL: rawURL, Err: err} - v.cacheResult(rawURL, lce) + v.finishCheck(rawURL, lce, ch) return lce } limiter := v.limiterFor(u.Host) if err := limiter.Wait(ctx); err != nil { - return LinkCheckError{URL: rawURL, Err: err} + lce := LinkCheckError{URL: rawURL, Err: err} + v.finishCheck(rawURL, lce, ch) + return lce } checkErr := v.doCheck(ctx, rawURL) - v.cacheResult(rawURL, checkErr) + v.finishCheck(rawURL, checkErr, ch) return checkErr } +func (v *LinkValidator) finishCheck(rawURL string, err error, ch chan struct{}) { + v.mu.Lock() + v.cache[rawURL] = err + delete(v.inflight, rawURL) + v.mu.Unlock() + close(ch) +} + func (v *LinkValidator) shouldSkip(rawURL string) bool { u, err := url.Parse(rawURL) if err != nil { @@ -89,13 +123,13 @@ func (v *LinkValidator) shouldSkip(rawURL string) bool { } for _, pattern := range v.Config.ExcludePatterns { - matched, err := path.Match(pattern, rawURL) - if err == nil && matched { + if prefix, ok := strings.CutSuffix(pattern, "*"); ok { + if strings.HasPrefix(rawURL, prefix) { + return true + } + } else if pattern == rawURL { return true } - if strings.Contains(rawURL, strings.ReplaceAll(pattern, "*", "")) { - continue - } } return false @@ -207,9 +241,3 @@ func parseRetryAfter(val string) time.Duration { func (v *LinkValidator) isAccepted(code int) bool { return slices.Contains(v.Config.AcceptedCodes, code) } - -func (v *LinkValidator) cacheResult(rawURL string, err error) { - v.mu.Lock() - defer v.mu.Unlock() - v.cache[rawURL] = err -} diff --git a/link_validator_test.go b/link_validator_test.go index 59c4905..482ec15 100644 --- a/link_validator_test.go +++ b/link_validator_test.go @@ -98,8 +98,51 @@ func Test_LinkValidator_Check_SkipExcluded(t *testing.T) { cfg.ExcludePatterns = []string{srv.URL + "/*"} v := NewLinkValidator(cfg) - err := v.Check(context.Background(), srv.URL+"/should-skip") - r.NoError(err) + r.NoError(v.Check(context.Background(), srv.URL+"/should-skip")) + r.NoError(v.Check(context.Background(), srv.URL+"/docs/v1/nested/page")) +} + +func Test_LinkValidator_Check_ConcurrentDedup(t *testing.T) { + t.Parallel() + r := require.New(t) + + var hitCount atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + hitCount.Add(1) + time.Sleep(50 * time.Millisecond) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + u := srv.URL + "/dedup" + errs := make(chan error, 5) + for range 5 { + go func() { + errs <- v.Check(context.Background(), u) + }() + } + + for range 5 { + r.NoError(<-errs) + } + + r.Equal(int32(1), hitCount.Load()) +} + +func Test_LinkValidator_ZeroRate(t *testing.T) { + t.Parallel() + r := require.New(t) + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + cfg.RatePerHost = 0 + v := NewLinkValidator(cfg) + + r.Equal(float64(2), v.Config.RatePerHost) } func Test_LinkValidator_Check_Cache(t *testing.T) { From d87c6e04eb3073fec1cef031ea42827de7907a32 Mon Sep 17 00:00:00 2001 From: Cory LaNou Date: Mon, 16 Mar 2026 09:32:49 -0500 Subject: [PATCH 3/4] fix(link): address codex review findings (pass 2) - Fall back to GET when HEAD returns non-accepted status (not just 405) - Normalize protocol-relative URLs (//example.com) to https - Fix off-by-one in redirect limit (allow exactly MaxRedirects hops) --- link_validator.go | 19 +++++++------ link_validator_test.go | 64 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/link_validator.go b/link_validator.go index 4b78536..b2bba58 100644 --- a/link_validator.go +++ b/link_validator.go @@ -43,7 +43,7 @@ func NewLinkValidator(cfg LinkCheckConfig) *LinkValidator { v.client = &http.Client{ Timeout: cfg.Timeout, CheckRedirect: func(req *http.Request, via []*http.Request) error { - if len(via) >= cfg.MaxRedirects { + if len(via) > cfg.MaxRedirects { return fmt.Errorf("too many redirects (%d)", len(via)) } return nil @@ -54,6 +54,10 @@ func NewLinkValidator(cfg LinkCheckConfig) *LinkValidator { } func (v *LinkValidator) Check(ctx context.Context, rawURL string) error { + if strings.HasPrefix(rawURL, "//") { + rawURL = "https:" + rawURL + } + if v.shouldSkip(rawURL) { return nil } @@ -117,6 +121,9 @@ func (v *LinkValidator) shouldSkip(rawURL string) bool { switch u.Scheme { case "http", "https": case "": + if strings.HasPrefix(rawURL, "//") { + return false + } return true default: return true @@ -162,19 +169,15 @@ func (v *LinkValidator) doCheck(ctx context.Context, rawURL string) error { io.Copy(io.Discard, resp.Body) resp.Body.Close() - if resp.StatusCode == http.StatusMethodNotAllowed { - return v.doGet(ctx, rawURL) - } - if resp.StatusCode == http.StatusTooManyRequests { return v.retryAfter(ctx, rawURL, resp) } - if !v.isAccepted(resp.StatusCode) { - return LinkCheckError{URL: rawURL, StatusCode: resp.StatusCode} + if v.isAccepted(resp.StatusCode) { + return nil } - return nil + return v.doGet(ctx, rawURL) } func (v *LinkValidator) doGet(ctx context.Context, rawURL string) error { diff --git a/link_validator_test.go b/link_validator_test.go index 482ec15..1f53a48 100644 --- a/link_validator_test.go +++ b/link_validator_test.go @@ -133,6 +133,70 @@ func Test_LinkValidator_Check_ConcurrentDedup(t *testing.T) { r.Equal(int32(1), hitCount.Load()) } +func Test_LinkValidator_Check_HeadForbiddenFallbackToGet(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Method == http.MethodHead { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/blocked-head") + r.NoError(err) +} + +func Test_LinkValidator_Check_ProtocolRelative(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + r.False(v.shouldSkip("//" + srv.URL[len("http://"):] + "/page")) + + err := v.Check(context.Background(), srv.URL[len("http:"):]+"/page") + r.Error(err) +} + +func Test_LinkValidator_Check_SingleRedirectAllowed(t *testing.T) { + t.Parallel() + r := require.New(t) + + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + n := calls.Add(1) + if n == 1 { + http.Redirect(w, req, "/final", http.StatusMovedPermanently) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + cfg.MaxRedirects = 1 + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/one-redirect") + r.NoError(err) +} + func Test_LinkValidator_ZeroRate(t *testing.T) { t.Parallel() r := require.New(t) From 86c929c80d30f7ffaf11178e7565dd549eda0c4d Mon Sep 17 00:00:00 2001 From: Cory LaNou Date: Mon, 16 Mar 2026 09:42:15 -0500 Subject: [PATCH 4/4] fix(link): address codex review findings (pass 3) - Merge partial configs with defaults in NewLinkValidator - Apply link timeout to rate-limit waits and retry-after sleeps - Clear parser link-check state when flag is off --- cmd/hype/cli/export.go | 3 +++ link_validator.go | 22 +++++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/cmd/hype/cli/export.go b/cmd/hype/cli/export.go index 384566a..0294882 100644 --- a/cmd/hype/cli/export.go +++ b/cmd/hype/cli/export.go @@ -299,6 +299,9 @@ func (cmd *Export) execute(ctx context.Context, pwd string) error { } p.LinkCheck = cfg p.LinkValidator = hype.NewLinkValidator(cfg) + } else { + p.LinkCheck = hype.LinkCheckConfig{} + p.LinkValidator = nil } p.Root = filepath.Join(filepath.Dir(mp), fileDir) diff --git a/link_validator.go b/link_validator.go index b2bba58..af2a346 100644 --- a/link_validator.go +++ b/link_validator.go @@ -26,11 +26,21 @@ type LinkValidator struct { } func NewLinkValidator(cfg LinkCheckConfig) *LinkValidator { + defaults := DefaultLinkCheckConfig() + if cfg.Timeout <= 0 { + cfg.Timeout = defaults.Timeout + } + if len(cfg.AcceptedCodes) == 0 { + cfg.AcceptedCodes = defaults.AcceptedCodes + } if cfg.RatePerHost <= 0 { - cfg.RatePerHost = 2 + cfg.RatePerHost = defaults.RatePerHost } if cfg.RateBurst <= 0 { - cfg.RateBurst = 1 + cfg.RateBurst = defaults.RateBurst + } + if cfg.MaxRedirects <= 0 { + cfg.MaxRedirects = defaults.MaxRedirects } v := &LinkValidator{ @@ -41,7 +51,6 @@ func NewLinkValidator(cfg LinkCheckConfig) *LinkValidator { } v.client = &http.Client{ - Timeout: cfg.Timeout, CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) > cfg.MaxRedirects { return fmt.Errorf("too many redirects (%d)", len(via)) @@ -85,6 +94,9 @@ func (v *LinkValidator) Check(ctx context.Context, rawURL string) error { v.inflight[rawURL] = ch v.mu.Unlock() + linkCtx, cancel := context.WithTimeout(ctx, v.Config.Timeout) + defer cancel() + u, err := url.Parse(rawURL) if err != nil { lce := LinkCheckError{URL: rawURL, Err: err} @@ -93,13 +105,13 @@ func (v *LinkValidator) Check(ctx context.Context, rawURL string) error { } limiter := v.limiterFor(u.Host) - if err := limiter.Wait(ctx); err != nil { + if err := limiter.Wait(linkCtx); err != nil { lce := LinkCheckError{URL: rawURL, Err: err} v.finishCheck(rawURL, lce, ch) return lce } - checkErr := v.doCheck(ctx, rawURL) + checkErr := v.doCheck(linkCtx, rawURL) v.finishCheck(rawURL, checkErr, ch) return checkErr }