diff --git a/cmd/hype/cli/export.go b/cmd/hype/cli/export.go index 0b82caf..0294882 100644 --- a/cmd/hype/cli/export.go +++ b/cmd/hype/cli/export.go @@ -40,6 +40,11 @@ type Export struct { NoCSS bool // output raw HTML without styling ListThemes bool // list available themes and exit + CheckLinks bool // enable link reachability checking + LinkTimeout time.Duration // per-link check timeout + LinkExclude string // comma-separated URL patterns to exclude + LinkRate float64 // requests per second per host + flags *flag.FlagSet mu sync.RWMutex @@ -108,6 +113,9 @@ Examples: hype export -themes hype export -f README.md -format markdown -timeout=10s hype export -f input.md -format markdown -o README.md + hype export -f hype.md -check-links + hype export -f hype.md -check-links -link-timeout=20s -link-rate=1 + hype export -f hype.md -check-links -link-exclude="https://localhost:*,https://internal.example.com/*" ` if err := cmd.validate(); err != nil { @@ -132,6 +140,10 @@ Examples: cmd.flags.StringVar(&cmd.CustomCSS, "css", "", "path to custom CSS file for HTML export") cmd.flags.BoolVar(&cmd.NoCSS, "no-css", false, "output raw HTML without styling") cmd.flags.BoolVar(&cmd.ListThemes, "themes", false, "list available themes and exit") + cmd.flags.BoolVar(&cmd.CheckLinks, "check-links", false, "enable URL reachability checking for links") + cmd.flags.DurationVar(&cmd.LinkTimeout, "link-timeout", 10*time.Second, "per-link check timeout") + cmd.flags.StringVar(&cmd.LinkExclude, "link-exclude", "", "comma-separated URL patterns to exclude from link checking") + cmd.flags.Float64Var(&cmd.LinkRate, "link-rate", 2, "max requests per second per host for link checking") cmd.flags.Usage = func() { fmt.Fprintf(stderr, "Usage of %s:\n", os.Args[0]) @@ -272,6 +284,26 @@ func (cmd *Export) execute(ctx context.Context, pwd string) error { p.FS = parserFS } + if cmd.CheckLinks { + cfg := hype.DefaultLinkCheckConfig() + cfg.Enabled = true + cfg.Timeout = cmd.LinkTimeout + cfg.RatePerHost = cmd.LinkRate + if cmd.LinkExclude != "" { + for _, pattern := range strings.Split(cmd.LinkExclude, ",") { + pattern = strings.TrimSpace(pattern) + if pattern != "" { + cfg.ExcludePatterns = append(cfg.ExcludePatterns, pattern) + } + } + } + p.LinkCheck = cfg + p.LinkValidator = hype.NewLinkValidator(cfg) + } else { + p.LinkCheck = hype.LinkCheckConfig{} + p.LinkValidator = nil + } + p.Root = filepath.Join(filepath.Dir(mp), fileDir) doc, err := p.ParseFile(fileName) diff --git a/go.mod b/go.mod index ebe79a9..96e3615 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/stretchr/testify v1.11.1 golang.org/x/net v0.51.0 golang.org/x/sync v0.19.0 + golang.org/x/time v0.14.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -168,7 +169,6 @@ require ( golang.org/x/sys v0.41.0 // indirect golang.org/x/term v0.40.0 // indirect golang.org/x/text v0.34.0 // indirect - golang.org/x/time v0.14.0 // indirect golang.org/x/tools v0.42.0 // indirect google.golang.org/api v0.266.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect diff --git a/link.go b/link.go index 8ddd620..f5c8631 100644 --- a/link.go +++ b/link.go @@ -1,6 +1,7 @@ package hype import ( + "context" "encoding/json" "fmt" "strings" @@ -56,6 +57,32 @@ func (l *Link) MD() string { return fmt.Sprintf("[%s](%s)", l.Children().MD(), h) } +func (l *Link) Execute(ctx context.Context, doc *Document) error { + if l == nil { + return ErrIsNil("link") + } + + if doc == nil || doc.Parser == nil { + return nil + } + + if !doc.Parser.LinkCheck.Enabled { + return nil + } + + href, err := l.Href() + if err != nil { + return nil + } + + v := doc.Parser.LinkValidator + if v == nil { + return nil + } + + return v.Check(ctx, href) +} + func NewLink(el *Element) (*Link, error) { if el == nil { return nil, ErrIsNil("element") diff --git a/link_check_config.go b/link_check_config.go new file mode 100644 index 0000000..0f5433c --- /dev/null +++ b/link_check_config.go @@ -0,0 +1,27 @@ +package hype + +import "time" + +type LinkCheckConfig struct { + Enabled bool + Timeout time.Duration + AcceptedCodes []int + ExcludePatterns []string + RatePerHost float64 + RateBurst int + MaxRedirects int +} + +func DefaultLinkCheckConfig() LinkCheckConfig { + return LinkCheckConfig{ + Enabled: false, + Timeout: 10 * time.Second, + AcceptedCodes: []int{ + 200, 201, 202, 203, 204, + 301, 302, 307, 308, + }, + RatePerHost: 2, + RateBurst: 1, + MaxRedirects: 10, + } +} diff --git a/link_check_config_test.go b/link_check_config_test.go new file mode 100644 index 0000000..5ef35ec --- /dev/null +++ b/link_check_config_test.go @@ -0,0 +1,25 @@ +package hype + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func Test_DefaultLinkCheckConfig(t *testing.T) { + t.Parallel() + r := require.New(t) + + cfg := DefaultLinkCheckConfig() + + r.False(cfg.Enabled) + r.Equal(10*time.Second, cfg.Timeout) + r.Contains(cfg.AcceptedCodes, 200) + r.Contains(cfg.AcceptedCodes, 301) + r.Contains(cfg.AcceptedCodes, 302) + r.Equal(float64(2), cfg.RatePerHost) + r.Equal(1, cfg.RateBurst) + r.Equal(10, cfg.MaxRedirects) + r.Empty(cfg.ExcludePatterns) +} diff --git a/link_check_error.go b/link_check_error.go new file mode 100644 index 0000000..d62629f --- /dev/null +++ b/link_check_error.go @@ -0,0 +1,38 @@ +package hype + +import ( + "encoding/json" + "fmt" +) + +type LinkCheckError struct { + URL string + StatusCode int + Err error +} + +func (e LinkCheckError) Error() string { + if e.StatusCode > 0 { + return fmt.Sprintf("link check failed for %q: status %d", e.URL, e.StatusCode) + } + if e.Err != nil { + return fmt.Sprintf("link check failed for %q: %s", e.URL, e.Err) + } + return fmt.Sprintf("link check failed for %q", e.URL) +} + +func (e LinkCheckError) Unwrap() error { + return e.Err +} + +func (e LinkCheckError) MarshalJSON() ([]byte, error) { + m := map[string]any{ + "type": "link_check_error", + "url": e.URL, + "status_code": e.StatusCode, + } + if e.Err != nil { + m["error"] = e.Err.Error() + } + return json.MarshalIndent(m, "", " ") +} diff --git a/link_test.go b/link_test.go index 367328c..f46102d 100644 --- a/link_test.go +++ b/link_test.go @@ -1,7 +1,11 @@ package hype import ( + "context" + "net/http" + "net/http/httptest" "testing" + "testing/fstest" "github.com/stretchr/testify/require" ) @@ -21,3 +25,120 @@ func Test_Link_MarshalJSON(t *testing.T) { testJSON(t, "link", link) } + +func Test_Link_Execute_Disabled(t *testing.T) { + t.Parallel() + r := require.New(t) + + link := &Link{Element: NewEl("a", nil)} + r.NoError(link.Set("href", "https://example.com")) + + doc := &Document{ + FS: fstest.MapFS{}, + Parser: &Parser{}, + } + + err := link.Execute(context.Background(), doc) + r.NoError(err) +} + +func Test_Link_Execute_HTTP_OK(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + + link := &Link{Element: NewEl("a", nil)} + r.NoError(link.Set("href", srv.URL+"/page")) + + doc := &Document{ + FS: fstest.MapFS{}, + Parser: &Parser{ + LinkCheck: cfg, + LinkValidator: NewLinkValidator(cfg), + }, + } + + err := link.Execute(context.Background(), doc) + r.NoError(err) +} + +func Test_Link_Execute_HTTP_NotFound(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + + link := &Link{Element: NewEl("a", nil)} + r.NoError(link.Set("href", srv.URL+"/missing")) + + doc := &Document{ + FS: fstest.MapFS{}, + Parser: &Parser{ + LinkCheck: cfg, + LinkValidator: NewLinkValidator(cfg), + }, + } + + err := link.Execute(context.Background(), doc) + r.Error(err) + + var lce LinkCheckError + r.ErrorAs(err, &lce) + r.Equal(404, lce.StatusCode) +} + +func Test_Link_Execute_NonHTTP(t *testing.T) { + t.Parallel() + r := require.New(t) + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + + link := &Link{Element: NewEl("a", nil)} + r.NoError(link.Set("href", "mailto:test@example.com")) + + doc := &Document{ + FS: fstest.MapFS{}, + Parser: &Parser{ + LinkCheck: cfg, + LinkValidator: NewLinkValidator(cfg), + }, + } + + err := link.Execute(context.Background(), doc) + r.NoError(err) +} + +func Test_Link_Execute_NoHref(t *testing.T) { + t.Parallel() + r := require.New(t) + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + + link := &Link{Element: NewEl("a", nil)} + + doc := &Document{ + FS: fstest.MapFS{}, + Parser: &Parser{ + LinkCheck: cfg, + LinkValidator: NewLinkValidator(cfg), + }, + } + + err := link.Execute(context.Background(), doc) + r.NoError(err) +} diff --git a/link_validator.go b/link_validator.go new file mode 100644 index 0000000..af2a346 --- /dev/null +++ b/link_validator.go @@ -0,0 +1,258 @@ +package hype + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "slices" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/time/rate" +) + +type LinkValidator struct { + Config LinkCheckConfig + client *http.Client + + mu sync.RWMutex + cache map[string]error + inflight map[string]chan struct{} + limiters map[string]*rate.Limiter +} + +func NewLinkValidator(cfg LinkCheckConfig) *LinkValidator { + defaults := DefaultLinkCheckConfig() + if cfg.Timeout <= 0 { + cfg.Timeout = defaults.Timeout + } + if len(cfg.AcceptedCodes) == 0 { + cfg.AcceptedCodes = defaults.AcceptedCodes + } + if cfg.RatePerHost <= 0 { + cfg.RatePerHost = defaults.RatePerHost + } + if cfg.RateBurst <= 0 { + cfg.RateBurst = defaults.RateBurst + } + if cfg.MaxRedirects <= 0 { + cfg.MaxRedirects = defaults.MaxRedirects + } + + v := &LinkValidator{ + Config: cfg, + cache: make(map[string]error), + inflight: make(map[string]chan struct{}), + limiters: make(map[string]*rate.Limiter), + } + + v.client = &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > cfg.MaxRedirects { + return fmt.Errorf("too many redirects (%d)", len(via)) + } + return nil + }, + } + + return v +} + +func (v *LinkValidator) Check(ctx context.Context, rawURL string) error { + if strings.HasPrefix(rawURL, "//") { + rawURL = "https:" + rawURL + } + + if v.shouldSkip(rawURL) { + return nil + } + + v.mu.Lock() + if err, ok := v.cache[rawURL]; ok { + v.mu.Unlock() + return err + } + + if ch, ok := v.inflight[rawURL]; ok { + v.mu.Unlock() + select { + case <-ch: + case <-ctx.Done(): + return LinkCheckError{URL: rawURL, Err: ctx.Err()} + } + v.mu.RLock() + err := v.cache[rawURL] + v.mu.RUnlock() + return err + } + + ch := make(chan struct{}) + v.inflight[rawURL] = ch + v.mu.Unlock() + + linkCtx, cancel := context.WithTimeout(ctx, v.Config.Timeout) + defer cancel() + + u, err := url.Parse(rawURL) + if err != nil { + lce := LinkCheckError{URL: rawURL, Err: err} + v.finishCheck(rawURL, lce, ch) + return lce + } + + limiter := v.limiterFor(u.Host) + if err := limiter.Wait(linkCtx); err != nil { + lce := LinkCheckError{URL: rawURL, Err: err} + v.finishCheck(rawURL, lce, ch) + return lce + } + + checkErr := v.doCheck(linkCtx, rawURL) + v.finishCheck(rawURL, checkErr, ch) + return checkErr +} + +func (v *LinkValidator) finishCheck(rawURL string, err error, ch chan struct{}) { + v.mu.Lock() + v.cache[rawURL] = err + delete(v.inflight, rawURL) + v.mu.Unlock() + close(ch) +} + +func (v *LinkValidator) shouldSkip(rawURL string) bool { + u, err := url.Parse(rawURL) + if err != nil { + return false + } + + switch u.Scheme { + case "http", "https": + case "": + if strings.HasPrefix(rawURL, "//") { + return false + } + return true + default: + return true + } + + for _, pattern := range v.Config.ExcludePatterns { + if prefix, ok := strings.CutSuffix(pattern, "*"); ok { + if strings.HasPrefix(rawURL, prefix) { + return true + } + } else if pattern == rawURL { + return true + } + } + + return false +} + +func (v *LinkValidator) limiterFor(host string) *rate.Limiter { + v.mu.Lock() + defer v.mu.Unlock() + + if l, ok := v.limiters[host]; ok { + return l + } + + l := rate.NewLimiter(rate.Limit(v.Config.RatePerHost), v.Config.RateBurst) + v.limiters[host] = l + return l +} + +func (v *LinkValidator) doCheck(ctx context.Context, rawURL string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, rawURL, nil) + if err != nil { + return LinkCheckError{URL: rawURL, Err: err} + } + req.Header.Set("User-Agent", "hype-link-checker/1.0") + + resp, err := v.client.Do(req) + if err != nil { + return LinkCheckError{URL: rawURL, Err: err} + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + + if resp.StatusCode == http.StatusTooManyRequests { + return v.retryAfter(ctx, rawURL, resp) + } + + if v.isAccepted(resp.StatusCode) { + return nil + } + + return v.doGet(ctx, rawURL) +} + +func (v *LinkValidator) doGet(ctx context.Context, rawURL string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return LinkCheckError{URL: rawURL, Err: err} + } + req.Header.Set("User-Agent", "hype-link-checker/1.0") + + resp, err := v.client.Do(req) + if err != nil { + return LinkCheckError{URL: rawURL, Err: err} + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + + if resp.StatusCode == http.StatusTooManyRequests { + return v.retryAfter(ctx, rawURL, resp) + } + + if !v.isAccepted(resp.StatusCode) { + return LinkCheckError{URL: rawURL, StatusCode: resp.StatusCode} + } + + return nil +} + +func (v *LinkValidator) retryAfter(ctx context.Context, rawURL string, resp *http.Response) error { + wait := parseRetryAfter(resp.Header.Get("Retry-After")) + wait = min(wait, 60*time.Second) + if wait <= 0 { + wait = 5 * time.Second + } + + select { + case <-ctx.Done(): + return LinkCheckError{URL: rawURL, Err: ctx.Err()} + case <-time.After(wait): + } + + return v.doGet(ctx, rawURL) +} + +func parseRetryAfter(val string) time.Duration { + if val == "" { + return 0 + } + + if secs, err := strconv.Atoi(val); err == nil { + return time.Duration(secs) * time.Second + } + + if t, err := http.ParseTime(val); err == nil { + d := time.Until(t) + if d < 0 { + return 0 + } + return d + } + + return 0 +} + +func (v *LinkValidator) isAccepted(code int) bool { + return slices.Contains(v.Config.AcceptedCodes, code) +} diff --git a/link_validator_test.go b/link_validator_test.go new file mode 100644 index 0000000..1f53a48 --- /dev/null +++ b/link_validator_test.go @@ -0,0 +1,316 @@ +package hype + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func Test_LinkValidator_Check_Success(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/page") + r.NoError(err) +} + +func Test_LinkValidator_Check_HeadFallbackToGet(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Method == http.MethodHead { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/page") + r.NoError(err) +} + +func Test_LinkValidator_Check_NotFound(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/missing") + r.Error(err) + + var lce LinkCheckError + r.ErrorAs(err, &lce) + r.Equal(404, lce.StatusCode) +} + +func Test_LinkValidator_Check_SkipNonHTTP(t *testing.T) { + t.Parallel() + r := require.New(t) + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + r.NoError(v.Check(context.Background(), "mailto:test@example.com")) + r.NoError(v.Check(context.Background(), "tel:+1234567890")) + r.NoError(v.Check(context.Background(), "ftp://example.com/file")) + r.NoError(v.Check(context.Background(), "#section-anchor")) +} + +func Test_LinkValidator_Check_SkipExcluded(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + cfg.ExcludePatterns = []string{srv.URL + "/*"} + v := NewLinkValidator(cfg) + + r.NoError(v.Check(context.Background(), srv.URL+"/should-skip")) + r.NoError(v.Check(context.Background(), srv.URL+"/docs/v1/nested/page")) +} + +func Test_LinkValidator_Check_ConcurrentDedup(t *testing.T) { + t.Parallel() + r := require.New(t) + + var hitCount atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + hitCount.Add(1) + time.Sleep(50 * time.Millisecond) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + u := srv.URL + "/dedup" + errs := make(chan error, 5) + for range 5 { + go func() { + errs <- v.Check(context.Background(), u) + }() + } + + for range 5 { + r.NoError(<-errs) + } + + r.Equal(int32(1), hitCount.Load()) +} + +func Test_LinkValidator_Check_HeadForbiddenFallbackToGet(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Method == http.MethodHead { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/blocked-head") + r.NoError(err) +} + +func Test_LinkValidator_Check_ProtocolRelative(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + r.False(v.shouldSkip("//" + srv.URL[len("http://"):] + "/page")) + + err := v.Check(context.Background(), srv.URL[len("http:"):]+"/page") + r.Error(err) +} + +func Test_LinkValidator_Check_SingleRedirectAllowed(t *testing.T) { + t.Parallel() + r := require.New(t) + + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + n := calls.Add(1) + if n == 1 { + http.Redirect(w, req, "/final", http.StatusMovedPermanently) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + cfg.MaxRedirects = 1 + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/one-redirect") + r.NoError(err) +} + +func Test_LinkValidator_ZeroRate(t *testing.T) { + t.Parallel() + r := require.New(t) + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + cfg.RatePerHost = 0 + v := NewLinkValidator(cfg) + + r.Equal(float64(2), v.Config.RatePerHost) +} + +func Test_LinkValidator_Check_Cache(t *testing.T) { + t.Parallel() + r := require.New(t) + + var hitCount atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + hitCount.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + u := srv.URL + "/cached" + r.NoError(v.Check(context.Background(), u)) + r.NoError(v.Check(context.Background(), u)) + + r.Equal(int32(1), hitCount.Load()) +} + +func Test_LinkValidator_Check_RetryAfter(t *testing.T) { + t.Parallel() + r := require.New(t) + + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + n := calls.Add(1) + if n <= 2 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/rate-limited") + r.NoError(err) +} + +func Test_LinkValidator_Check_TooManyRedirects(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + http.Redirect(w, req, req.URL.Path, http.StatusMovedPermanently) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + cfg.MaxRedirects = 3 + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/loop") + r.Error(err) +} + +func Test_LinkValidator_Check_Timeout(t *testing.T) { + t.Parallel() + r := require.New(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + time.Sleep(5 * time.Second) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + cfg.Timeout = 100 * time.Millisecond + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/slow") + r.Error(err) +} + +func Test_LinkValidator_Check_FollowRedirects(t *testing.T) { + t.Parallel() + r := require.New(t) + + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + n := calls.Add(1) + if n == 1 { + http.Redirect(w, req, "/final", http.StatusMovedPermanently) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + cfg := DefaultLinkCheckConfig() + cfg.Enabled = true + v := NewLinkValidator(cfg) + + err := v.Check(context.Background(), srv.URL+"/redirect") + r.NoError(err) +} diff --git a/parser.go b/parser.go index b2da315..3c1d315 100644 --- a/parser.go +++ b/parser.go @@ -25,16 +25,18 @@ type ParseElementFn func(p *Parser, el *Element) (Nodes, error) type Parser struct { fs.FS - DisablePages bool - DocIDGen func() (string, error) // default: uuid.NewV4().String() - Filename string // only set when Parser.ParseFile() is used - NodeParsers map[Atom]ParseElementFn - NowFn func() time.Time // default: time.Now() - PreParsers PreParsers - Root string - Section int - Vars syncx.Map[string, any] - Contents []byte // a copy of the contents being parsed - set just before parsing + DisablePages bool + DocIDGen func() (string, error) // default: uuid.NewV4().String() + Filename string // only set when Parser.ParseFile() is used + LinkCheck LinkCheckConfig + LinkValidator *LinkValidator + NodeParsers map[Atom]ParseElementFn + NowFn func() time.Time // default: time.Now() + PreParsers PreParsers + Root string + Section int + Vars syncx.Map[string, any] + Contents []byte // a copy of the contents being parsed - set just before parsing mu sync.RWMutex } @@ -402,10 +404,12 @@ func (p *Parser) Sub(dir string) (*Parser, error) { } p2 := &Parser{ - FS: p.FS, - Root: filepath.Join(p.Root, dir), - PreParsers: p.PreParsers, - NodeParsers: p.NodeParsers, + FS: p.FS, + Root: filepath.Join(p.Root, dir), + PreParsers: p.PreParsers, + NodeParsers: p.NodeParsers, + LinkCheck: p.LinkCheck, + LinkValidator: p.LinkValidator, } if len(dir) == 0 || dir == "." {