From 623f64066ceb500739d400fe9e2158c41ecd5534 Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Fri, 24 Apr 2026 11:28:16 -0600 Subject: [PATCH 1/3] Scan a markdown file using tree-sitter into an array of content blocks --- go.mod | 12 ++- go.sum | 11 +++ internal/block.go | 94 ++++++++++++++++++++ internal/block_test.go | 169 ++++++++++++++++++++++++++++++++++++ internal/testdata/input.md | 43 +++++++++ internal/testdata/output.md | 51 +++++++++++ 6 files changed, 378 insertions(+), 2 deletions(-) create mode 100644 internal/block.go create mode 100644 internal/block_test.go create mode 100644 internal/testdata/input.md create mode 100644 internal/testdata/output.md diff --git a/go.mod b/go.mod index 52b40c3..5b09597 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,18 @@ module litdoc go 1.23 require ( + github.com/rogpeppe/go-internal v1.14.1 + github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 + github.com/spf13/cobra v1.10.2 + github.com/stretchr/testify v1.11.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/rogpeppe/go-internal v1.14.1 // indirect - github.com/spf13/cobra v1.10.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.9 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/tools v0.26.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f5c365b..73935a9 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,27 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 h1:6C8qej6f1bStuePVkLSFxoU22XBS165D3klxlzRg8F4= +github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82/go.mod h1:xe4pgH49k4SsmkQq5OT8abwhWmnzkhpgnXeekbx2efw= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/block.go b/internal/block.go new file mode 100644 index 0000000..1c09b53 --- /dev/null +++ b/internal/block.go @@ -0,0 +1,94 @@ +package internal + +import ( + "bytes" + "context" + "fmt" + + sitter "github.com/smacker/go-tree-sitter" + "github.com/smacker/go-tree-sitter/markdown" +) + +type BlockKind string + +const ( + BlockKindText BlockKind = "Text" + BlockKindFencedCode BlockKind = "fencedCode" + BlockKindHTMLComment BlockKind = "HTMLComment" +) + +type Block struct { + kind BlockKind + content []byte +} + +func MakeBlockFromRaw(kind BlockKind, raw []byte) Block { + return Block{kind: kind, content: raw} +} + +func (b Block) Kind() BlockKind { return b.kind } + +func (b Block) Content() []byte { return b.content } + +func MakeBlocksFromMarkdown(content []byte) ([]Block, error) { + tree, err := markdown.ParseCtx(context.Background(), nil, content) + if err != nil { + return nil, fmt.Errorf("parsing markdown: %w", err) + } + + root := tree.BlockTree().RootNode() + var blocks []Block + pos := uint32(0) + + collectBlockNodes(root, content, &pos, &blocks) + + if pos < uint32(len(content)) { + blocks = append(blocks, Block{kind: BlockKindText, content: content[pos:]}) + } + + return blocks, nil +} + +func collectBlockNodes( + node *sitter.Node, + content []byte, + pos *uint32, + blocks *[]Block, +) { + switch node.Type() { + case "document", "section": + for i := 0; i < int(node.ChildCount()); i++ { + collectBlockNodes(node.Child(i), content, pos, blocks) + } + default: + start := node.StartByte() + end := node.EndByte() + + if start > *pos { + *blocks = append( + *blocks, + MakeBlockFromRaw(BlockKindText, content[*pos:start]), + ) + } + + *blocks = append( + *blocks, + MakeBlockFromRaw(blockKind(node, content), content[start:end]), + ) + *pos = end + } +} + +func blockKind(node *sitter.Node, content []byte) BlockKind { + switch node.Type() { + case "fenced_code_block": + return BlockKindFencedCode + case "html_block": + if bytes.HasPrefix(content[node.StartByte():node.EndByte()], []byte("\n", + want: []struct { + kind internal.BlockKind + content string + }{ + {internal.BlockKindHTMLComment, "\n"}, + }, + }, + { + name: "html block is not a comment", + input: "
\nhello\n
\n", + want: []struct { + kind internal.BlockKind + content string + }{ + {internal.BlockKindText, "
\nhello\n
\n"}, + }, + }, + { + name: "mixed content", + input: "# Title\n\n```go\nfmt.Println()\n```\n\n\n", + want: []struct { + kind internal.BlockKind + content string + }{ + {internal.BlockKindText, "# Title\n"}, + {internal.BlockKindText, "\n"}, + {internal.BlockKindFencedCode, "```go\nfmt.Println()\n```\n"}, + {internal.BlockKindText, "\n"}, + {internal.BlockKindHTMLComment, "\n"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // when + blocks, err := internal.MakeBlocksFromMarkdown([]byte(tt.input)) + + // then + require.NoError(t, err) + require.Len(t, blocks, len(tt.want)) + for i, w := range tt.want { + assert.Equal( + t, + w.kind, + blocks[i].Kind(), + "block[%d] kind", + i, + ) + assert.Equal( + t, + w.content, + string(blocks[i].Content()), + "block[%d] content", + i, + ) + } + }) + } +} diff --git a/internal/testdata/input.md b/internal/testdata/input.md new file mode 100644 index 0000000..cb143f2 --- /dev/null +++ b/internal/testdata/input.md @@ -0,0 +1,43 @@ +# Hello + +The following fenced code blocks should be ignored: + +```bash +echo "just, bash" +``` + +This next one should be executed: + +```bash | litdoc +echo "hello, world" +``` + +Another one to be ignored: + + + +And one more to be executed: + + + +Here's a previously executed block: + +```bash | litdoc +echo "hello, world" +``` + + +output + + +And a verbatim block that should be ignored: + +````md +```bash | litdoc +echo "hello, world" +``` +```` \ No newline at end of file diff --git a/internal/testdata/output.md b/internal/testdata/output.md new file mode 100644 index 0000000..4b7c795 --- /dev/null +++ b/internal/testdata/output.md @@ -0,0 +1,51 @@ +# Hello + +The following fenced code blocks should be ignored: + +```bash +echo "just, bash" +``` + +This next one should be executed: + +```bash | litdoc +echo "hello, world" +``` + + +output + + +Another one to be ignored: + + + +And one more to be executed: + + + + +output + + +Here's a previously executed block: + +```bash | litdoc +echo "hello, world" +``` + + +output + + +And a verbatim block that should be ignored: + +````md +```bash | litdoc +echo "hello, world" +``` +```` \ No newline at end of file From 9b8f1bb24aa0bc9da5a02399b48ac4fedee602f2 Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Fri, 24 Apr 2026 11:32:12 -0600 Subject: [PATCH 2/3] Add development instructions to Readme --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 35fbf75..6686c58 100644 --- a/README.md +++ b/README.md @@ -1 +1,9 @@ # Literate Documentation (`litdoc`) + +### Development + +To test, run: + +```bash +nix develop --ignore-environment --command make test +``` \ No newline at end of file From 63dd34f06fe9d9a8970c605cc853466031da39aa Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Fri, 24 Apr 2026 11:40:22 -0600 Subject: [PATCH 3/3] Add linting actions to CI rules --- .github/pull_request_template.md | 2 +- .github/workflows/{test.yml => ci.yml} | 6 ++++-- Makefile | 15 ++++++++++++++- 3 files changed, 19 insertions(+), 4 deletions(-) rename .github/workflows/{test.yml => ci.yml} (76%) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 0fbc9ad..53965ea 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -6,4 +6,4 @@ If any of these checks are missing, please provide an explanation. - [ ] I have updated README files, if applicable. -- [ ] I have run `make pre-pr` and all checks have passed. \ No newline at end of file +- [ ] This PR is small, otherwise explain why. \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/ci.yml similarity index 76% rename from .github/workflows/test.yml rename to .github/workflows/ci.yml index e2274a4..9aa71ba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: test +name: ci on: push: @@ -7,7 +7,7 @@ on: workflow_dispatch: jobs: - test: + ci: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -22,4 +22,6 @@ jobs: name: litdoc authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - run: nix develop --ignore-environment --command make fmt-check + - run: nix develop --ignore-environment --command make vet - run: nix develop --ignore-environment --command make test \ No newline at end of file diff --git a/Makefile b/Makefile index b869515..4828d68 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,18 @@ .PHONY: pre-pr -pre-pr: test +pre-pr: fmt-check vet test + +.PHONY: vet +vet: + @go vet ./... + +.PHONY: fmt-check +fmt-check: + @unformatted=$$(gofmt -l .); \ + if [ -n "$$unformatted" ]; then \ + echo "unformatted files:"; \ + echo "$$unformatted"; \ + exit 1; \ + fi GO_FILES := $(shell find . -name '*.go' -not -path './vendor/*')