[Feature] tree sitter markdown scan

.github/pull_request_template.md

@@ -6,4 +6,4 @@
 If any of these checks are missing, please provide an explanation.
 
 - [ ] I have updated README files, if applicable.
-- [ ] I have run `make pre-pr` and all checks have passed.
+- [ ] This PR is small, otherwise explain why.

.github/workflows/test.yml → .github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: test
+name: ci
 
 on:
   push:
@@ -7,7 +7,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  test:
+  ci:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -22,4 +22,6 @@ jobs:
           name: litdoc
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
 
+      - run: nix develop --ignore-environment --command make fmt-check
+      - run: nix develop --ignore-environment --command make vet
       - run: nix develop --ignore-environment --command make test

Makefile

@@ -1,5 +1,18 @@
 .PHONY: pre-pr
-pre-pr: test
+pre-pr: fmt-check vet test
+
+.PHONY: vet
+vet:
+	@go vet ./...
+
+.PHONY: fmt-check
+fmt-check:
+	@unformatted=$$(gofmt -l .); \
+	if [ -n "$$unformatted" ]; then \
+		echo "unformatted files:"; \
+		echo "$$unformatted"; \
+		exit 1; \
+	fi
 
 GO_FILES := $(shell find . -name '*.go' -not -path './vendor/*')
 

README.md

@@ -1 +1,9 @@
 # Literate Documentation (`litdoc`)
+
+### Development
+
+To test, run:
+
+```bash
+nix develop --ignore-environment --command make test
+```

go.mod

@@ -3,10 +3,18 @@ module litdoc
 go 1.23
 
 require (
+	github.com/rogpeppe/go-internal v1.14.1
+	github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82
+	github.com/spf13/cobra v1.10.2
+	github.com/stretchr/testify v1.11.1
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/rogpeppe/go-internal v1.14.1 // indirect
-	github.com/spf13/cobra v1.10.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
 	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/tools v0.26.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,16 +1,27 @@
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 h1:6C8qej6f1bStuePVkLSFxoU22XBS165D3klxlzRg8F4=
+github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82/go.mod h1:xe4pgH49k4SsmkQq5OT8abwhWmnzkhpgnXeekbx2efw=
 github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
 github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
 github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
 github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
 golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
 golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/block.go

@@ -0,0 +1,94 @@
+package internal
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	sitter "github.com/smacker/go-tree-sitter"
+	"github.com/smacker/go-tree-sitter/markdown"
+)
+
+type BlockKind string
+
+const (
+	BlockKindText        BlockKind = "Text"
+	BlockKindFencedCode  BlockKind = "fencedCode"
+	BlockKindHTMLComment BlockKind = "HTMLComment"
+)
+
+type Block struct {
+	kind    BlockKind
+	content []byte
+}
+
+func MakeBlockFromRaw(kind BlockKind, raw []byte) Block {
+	return Block{kind: kind, content: raw}
+}
+
+func (b Block) Kind() BlockKind { return b.kind }
+
+func (b Block) Content() []byte { return b.content }
+
+func MakeBlocksFromMarkdown(content []byte) ([]Block, error) {
+	tree, err := markdown.ParseCtx(context.Background(), nil, content)
+	if err != nil {
+		return nil, fmt.Errorf("parsing markdown: %w", err)
+	}
+
+	root := tree.BlockTree().RootNode()
+	var blocks []Block
+	pos := uint32(0)
+
+	collectBlockNodes(root, content, &pos, &blocks)
+
+	if pos < uint32(len(content)) {
+		blocks = append(blocks, Block{kind: BlockKindText, content: content[pos:]})
+	}
+
+	return blocks, nil
+}
+
+func collectBlockNodes(
+	node *sitter.Node,
+	content []byte,
+	pos *uint32,
+	blocks *[]Block,
+) {
+	switch node.Type() {
+	case "document", "section":
+		for i := 0; i < int(node.ChildCount()); i++ {
+			collectBlockNodes(node.Child(i), content, pos, blocks)
+		}
+	default:
+		start := node.StartByte()
+		end := node.EndByte()
+
+		if start > *pos {
+			*blocks = append(
+				*blocks,
+				MakeBlockFromRaw(BlockKindText, content[*pos:start]),
+			)
+		}
+
+		*blocks = append(
+			*blocks,
+			MakeBlockFromRaw(blockKind(node, content), content[start:end]),
+		)
+		*pos = end
+	}
+}
+
+func blockKind(node *sitter.Node, content []byte) BlockKind {
+	switch node.Type() {
+	case "fenced_code_block":
+		return BlockKindFencedCode
+	case "html_block":
+		if bytes.HasPrefix(content[node.StartByte():node.EndByte()], []byte("<!--")) {
+			return BlockKindHTMLComment
+		}
+		return BlockKindText
+	default:
+		return BlockKindText
+	}
+}

internal/block_test.go

@@ -0,0 +1,169 @@
+package internal_test
+
+import (
+	"testing"
+
+	"litdoc/internal"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMakeBlockFromRaw(t *testing.T) {
+	// given
+	kind := internal.BlockKindFencedCode
+	content := []byte("```bash\necho hello\n```\n")
+
+	// when
+	got := internal.MakeBlockFromRaw(kind, content)
+
+	// then
+	assert.Equal(t, kind, got.Kind())
+	assert.Equal(t, content, got.Content())
+}
+
+func TestMakeBlocksFromMarkdown(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  []struct {
+			kind    internal.BlockKind
+			content string
+		}
+	}{
+		{
+			name:  "heading",
+			input: "# Hello\n",
+			want: []struct {
+				kind    internal.BlockKind
+				content string
+			}{
+				{internal.BlockKindText, "# Hello\n"},
+			},
+		},
+		{
+			name:  "text",
+			input: "text\nand more text",
+			want: []struct {
+				kind    internal.BlockKind
+				content string
+			}{
+				{internal.BlockKindText, "text\nand more text"},
+			},
+		},
+		{
+			name:  "fenced code block",
+			input: "```bash\necho \"hello\"\n```\n",
+			want: []struct {
+				kind    internal.BlockKind
+				content string
+			}{
+				{internal.BlockKindFencedCode, "```bash\necho \"hello\"\n```\n"},
+			},
+		},
+		{
+			name:  "tilde fenced code block",
+			input: "~~~bash\necho \"hello\"\n~~~\n",
+			want: []struct {
+				kind    internal.BlockKind
+				content string
+			}{
+				{internal.BlockKindFencedCode, "~~~bash\necho \"hello\"\n~~~\n"},
+			},
+		},
+		{
+			name:  "longer backtick fence",
+			input: "````bash\necho \"hello\"\n````\n",
+			want: []struct {
+				kind    internal.BlockKind
+				content string
+			}{
+				{internal.BlockKindFencedCode, "````bash\necho \"hello\"\n````\n"},
+			},
+		},
+		{
+			name:  "quad fenced verbatim block containing litdoc source",
+			input: "````md\n```bash | litdoc\necho \"hello\"\n```\n````",
+			want: []struct {
+				kind    internal.BlockKind
+				content string
+			}{
+				{
+					internal.BlockKindFencedCode,
+					"````md\n```bash | litdoc\necho \"hello\"\n```\n````",
+				},
+			},
+		},
+		{
+			name:  "indented code block",
+			input: "    echo \"hello\"\n",
+			want: []struct {
+				kind    internal.BlockKind
+				content string
+			}{
+				{internal.BlockKindText, "    echo \"hello\"\n"},
+			},
+		},
+		{
+			name:  "html comment block",
+			input: "<!--\ncomment\n-->\n",
+			want: []struct {
+				kind    internal.BlockKind
+				content string
+			}{
+				{internal.BlockKindHTMLComment, "<!--\ncomment\n-->\n"},
+			},
+		},
+		{
+			name:  "html block is not a comment",
+			input: "<div>\nhello\n</div>\n",
+			want: []struct {
+				kind    internal.BlockKind
+				content string
+			}{
+				{internal.BlockKindText, "<div>\nhello\n</div>\n"},
+			},
+		},
+		{
+			name:  "mixed content",
+			input: "# Title\n\n```go\nfmt.Println()\n```\n\n<!--\nnote\n-->\n",
+			want: []struct {
+				kind    internal.BlockKind
+				content string
+			}{
+				{internal.BlockKindText, "# Title\n"},
+				{internal.BlockKindText, "\n"},
+				{internal.BlockKindFencedCode, "```go\nfmt.Println()\n```\n"},
+				{internal.BlockKindText, "\n"},
+				{internal.BlockKindHTMLComment, "<!--\nnote\n-->\n"},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// when
+			blocks, err := internal.MakeBlocksFromMarkdown([]byte(tt.input))
+
+			// then
+			require.NoError(t, err)
+			require.Len(t, blocks, len(tt.want))
+			for i, w := range tt.want {
+				assert.Equal(
+					t,
+					w.kind,
+					blocks[i].Kind(),
+					"block[%d] kind",
+					i,
+				)
+				assert.Equal(
+					t,
+					w.content,
+					string(blocks[i].Content()),
+					"block[%d] content",
+					i,
+				)
+			}
+		})
+	}
+}