From 86b58f45adb2ebf0a4bfd2cae219bc96174abbf7 Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Tue, 28 Apr 2026 16:22:25 -0600 Subject: [PATCH 01/18] Handle inline HTML comments --- internal/block.go | 47 +++++++++--- internal/block_test.go | 166 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 184 insertions(+), 29 deletions(-) diff --git a/internal/block.go b/internal/block.go index 1c09b53..fe4d229 100644 --- a/internal/block.go +++ b/internal/block.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "regexp" sitter "github.com/smacker/go-tree-sitter" "github.com/smacker/go-tree-sitter/markdown" @@ -30,6 +31,12 @@ func (b Block) Kind() BlockKind { return b.kind } func (b Block) Content() []byte { return b.content } +func (b Block) String() string { + return fmt.Sprintf("{%s %q}", b.kind, b.content) +} + +var htmlCommentRe = regexp.MustCompile(`(?s)`) + func MakeBlocksFromMarkdown(content []byte) ([]Block, error) { tree, err := markdown.ParseCtx(context.Background(), nil, content) if err != nil { @@ -46,7 +53,35 @@ func MakeBlocksFromMarkdown(content []byte) ([]Block, error) { blocks = append(blocks, Block{kind: BlockKindText, content: content[pos:]}) } - return blocks, nil + return splitInlineHTMLComments(blocks), nil +} + +func splitInlineHTMLComments(blocks []Block) []Block { + result := make([]Block, 0, len(blocks)) + for _, b := range blocks { + if b.Kind() != BlockKindText { + result = append(result, b) + continue + } + content := b.Content() + locs := htmlCommentRe.FindAllIndex(content, -1) + if len(locs) == 0 { + result = append(result, b) + continue + } + pos := 0 + for _, loc := range locs { + if loc[0] > pos { + result = append(result, MakeBlockFromRaw(BlockKindText, content[pos:loc[0]])) + } + result = append(result, MakeBlockFromRaw(BlockKindHTMLComment, content[loc[0]:loc[1]])) + pos = loc[1] + } + if pos < len(content) { + result = append(result, MakeBlockFromRaw(BlockKindText, content[pos:])) + } + } + return result } func collectBlockNodes( @@ -65,16 +100,10 @@ func collectBlockNodes( end := node.EndByte() if start > *pos { - *blocks = append( - *blocks, - MakeBlockFromRaw(BlockKindText, content[*pos:start]), - ) + *blocks = append(*blocks, MakeBlockFromRaw(BlockKindText, content[*pos:start])) } - *blocks = append( - *blocks, - MakeBlockFromRaw(blockKind(node, content), content[start:end]), - ) + *blocks = append(*blocks, MakeBlockFromRaw(blockKind(node, content), content[start:end])) *pos = end } } diff --git a/internal/block_test.go b/internal/block_test.go index 6f0bab3..7bb7e8e 100644 --- a/internal/block_test.go +++ b/internal/block_test.go @@ -1,6 +1,7 @@ package internal_test import ( + "strings" "testing" "litdoc/internal" @@ -22,6 +23,10 @@ func TestMakeBlockFromRaw(t *testing.T) { assert.Equal(t, content, got.Content()) } +func joinLines(lines ...string) string { + return strings.Join(lines, "\n") +} + func TestMakeBlocksFromMarkdown(t *testing.T) { tests := []struct { name string @@ -42,18 +47,69 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { }, }, { - name: "text", - input: "text\nand more text", + name: "nested heading and text", + input: joinLines( + "# Level 1", + "", + "some text", + "", + "## Level 2", + "", + "more text", + "", + ), + want: []struct { + kind internal.BlockKind + content string + }{ + {internal.BlockKindText, "# Level 1\n"}, + {internal.BlockKindText, "\n"}, + {internal.BlockKindText, "some text\n"}, + {internal.BlockKindText, "\n"}, + {internal.BlockKindText, "## Level 2\n"}, + {internal.BlockKindText, "\n"}, + {internal.BlockKindText, "more text\n"}, + }, + }, + { + name: "paragraphs grouped together", + input: joinLines( + "text", + "and more text", + "", + "last line", + ), want: []struct { kind internal.BlockKind content string }{ - {internal.BlockKindText, "text\nand more text"}, + {internal.BlockKindText, "text\nand more text\n"}, + {internal.BlockKindText, "\n"}, + {internal.BlockKindText, "last line"}, }, }, { - name: "fenced code block", - input: "```bash\necho \"hello\"\n```\n", + name: "line break grouped with paragraph", + input: joinLines( + "text", + "and more text ", + "after line break", + ), + want: []struct { + kind internal.BlockKind + content string + }{ + {internal.BlockKindText, "text\nand more text \nafter line break"}, + }, + }, + { + name: "fenced code block as single block", + input: joinLines( + "```bash", + "echo \"hello\"", + "```", + "", + ), want: []struct { kind internal.BlockKind content string @@ -62,56 +118,119 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { }, }, { - name: "tilde fenced code block", - input: "~~~bash\necho \"hello\"\n~~~\n", + name: "fenced code block with no trailing newline", + input: joinLines( + "```bash", + "echo \"hello\"", + "```", + ), want: []struct { kind internal.BlockKind content string }{ - {internal.BlockKindFencedCode, "~~~bash\necho \"hello\"\n~~~\n"}, + {internal.BlockKindFencedCode, "```bash\necho \"hello\"\n```"}, }, }, { - name: "longer backtick fence", - input: "````bash\necho \"hello\"\n````\n", + name: "tilde fenced code block as single block", + input: joinLines( + "~~~bash", + "echo \"hello\"", + "~~~", + ), want: []struct { kind internal.BlockKind content string }{ - {internal.BlockKindFencedCode, "````bash\necho \"hello\"\n````\n"}, + {internal.BlockKindFencedCode, "~~~bash\necho \"hello\"\n~~~"}, }, }, { - name: "quad fenced verbatim block containing litdoc source", - input: "````md\n```bash | litdoc\necho \"hello\"\n```\n````", + name: "longer backtick fence as single block", + input: joinLines( + "````md", + "```bash", + "echo \"hello\"", + "```", + "````", + ), want: []struct { kind internal.BlockKind content string }{ { internal.BlockKindFencedCode, - "````md\n```bash | litdoc\necho \"hello\"\n```\n````", + "````md\n```bash\necho \"hello\"\n```\n````", }, }, }, { - name: "indented code block", - input: " echo \"hello\"\n", + name: "indented code block", + input: joinLines( + " echo \"hello, \"", + " echo \"world\"", + ), + want: []struct { + kind internal.BlockKind + content string + }{ + { + internal.BlockKindText, + " echo \"hello, \"\n echo \"world\"", + }, + }, + }, + { + name: "inline code block", + input: "text `echo \"hello, \" text`", + want: []struct { + kind internal.BlockKind + content string + }{ + {internal.BlockKindText, "text `echo \"hello, \" text`"}, + }, + }, + { + name: "html comment inline comment", + input: "text text", + want: []struct { + kind internal.BlockKind + content string + }{ + {internal.BlockKindText, "text "}, + {internal.BlockKindHTMLComment, ""}, + {internal.BlockKindText, " text"}, + }, + }, + { + name: "multiple inline html comments", + input: "text text", want: []struct { kind internal.BlockKind content string }{ - {internal.BlockKindText, " echo \"hello\"\n"}, + {internal.BlockKindText, "text "}, + {internal.BlockKindHTMLComment, ""}, + {internal.BlockKindHTMLComment, ""}, + {internal.BlockKindText, " text"}, }, }, { - name: "html comment block", - input: "\n", + name: "html comment block comment", + input: joinLines( + "text", + "", + "text", + ), want: []struct { kind internal.BlockKind content string }{ + {internal.BlockKindText, "text\n"}, {internal.BlockKindHTMLComment, "\n"}, + {internal.BlockKindText, "text"}, }, }, { @@ -147,7 +266,14 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { // then require.NoError(t, err) - require.Len(t, blocks, len(tt.want)) + require.Len( + t, + blocks, + len(tt.want), + "expected %d blocks, got %d", + len(tt.want), + len(blocks), + ) for i, w := range tt.want { assert.Equal( t, From cdcdde3170602553dacd339ae06eb5e46b06aea4 Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Tue, 28 Apr 2026 16:59:56 -0600 Subject: [PATCH 02/18] Process single blockquotes --- internal/block.go | 57 +++++++++++++++++--- internal/block_test.go | 115 +++++++++++++++++++++++++++++------------ 2 files changed, 130 insertions(+), 42 deletions(-) diff --git a/internal/block.go b/internal/block.go index fe4d229..dcffe87 100644 --- a/internal/block.go +++ b/internal/block.go @@ -20,6 +20,7 @@ const ( type Block struct { kind BlockKind + indent []byte content []byte } @@ -27,8 +28,14 @@ func MakeBlockFromRaw(kind BlockKind, raw []byte) Block { return Block{kind: kind, content: raw} } +func makeBlock(kind BlockKind, content, indent []byte) Block { + return Block{kind: kind, indent: indent, content: content} +} + func (b Block) Kind() BlockKind { return b.kind } +func (b Block) Indent() []byte { return b.indent } + func (b Block) Content() []byte { return b.content } func (b Block) String() string { @@ -47,10 +54,10 @@ func MakeBlocksFromMarkdown(content []byte) ([]Block, error) { var blocks []Block pos := uint32(0) - collectBlockNodes(root, content, &pos, &blocks) + collectBlockNodes(root, content, nil, &pos, &blocks) if pos < uint32(len(content)) { - blocks = append(blocks, Block{kind: BlockKindText, content: content[pos:]}) + blocks = append(blocks, makeBlock(BlockKindText, content[pos:], nil)) } return splitInlineHTMLComments(blocks), nil @@ -72,13 +79,13 @@ func splitInlineHTMLComments(blocks []Block) []Block { pos := 0 for _, loc := range locs { if loc[0] > pos { - result = append(result, MakeBlockFromRaw(BlockKindText, content[pos:loc[0]])) + result = append(result, makeBlock(BlockKindText, content[pos:loc[0]], b.indent)) } - result = append(result, MakeBlockFromRaw(BlockKindHTMLComment, content[loc[0]:loc[1]])) + result = append(result, makeBlock(BlockKindHTMLComment, content[loc[0]:loc[1]], b.indent)) pos = loc[1] } if pos < len(content) { - result = append(result, MakeBlockFromRaw(BlockKindText, content[pos:])) + result = append(result, makeBlock(BlockKindText, content[pos:], b.indent)) } } return result @@ -87,27 +94,61 @@ func splitInlineHTMLComments(blocks []Block) []Block { func collectBlockNodes( node *sitter.Node, content []byte, + indent []byte, pos *uint32, blocks *[]Block, ) { switch node.Type() { case "document", "section": for i := 0; i < int(node.ChildCount()); i++ { - collectBlockNodes(node.Child(i), content, pos, blocks) + collectBlockNodes(node.Child(i), content, indent, pos, blocks) + } + case "block_quote": + childIndent := append(append([]byte(nil), indent...), "> "...) + for i := 0; i < int(node.ChildCount()); i++ { + child := node.Child(i) + if child.Type() == "block_quote_marker" { + if child.StartByte() > *pos { + gap := stripIndent(content[*pos:child.StartByte()], childIndent) + if len(gap) > 0 { + *blocks = append(*blocks, makeBlock(BlockKindText, gap, childIndent)) + } + } + *pos = child.EndByte() + continue + } + collectBlockNodes(child, content, childIndent, pos, blocks) } default: start := node.StartByte() end := node.EndByte() if start > *pos { - *blocks = append(*blocks, MakeBlockFromRaw(BlockKindText, content[*pos:start])) + gap := stripIndent(content[*pos:start], indent) + if len(gap) > 0 { + *blocks = append(*blocks, makeBlock(BlockKindText, gap, indent)) + } } - *blocks = append(*blocks, MakeBlockFromRaw(blockKind(node, content), content[start:end])) + raw := stripIndent(content[start:end], indent) + if len(raw) > 0 { + *blocks = append(*blocks, makeBlock(blockKind(node, content), raw, indent)) + } *pos = end } } +func stripIndent(content, indent []byte) []byte { + if len(indent) == 0 { + return content + } + lines := bytes.Split(content, []byte("\n")) + for i, line := range lines { + lines[i] = bytes.TrimPrefix(line, indent) + } + return bytes.Join(lines, []byte("\n")) +} + func blockKind(node *sitter.Node, content []byte) BlockKind { switch node.Type() { case "fenced_code_block": diff --git a/internal/block_test.go b/internal/block_test.go index 7bb7e8e..27ecdb4 100644 --- a/internal/block_test.go +++ b/internal/block_test.go @@ -33,6 +33,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { input string want []struct { kind internal.BlockKind + indent string content string } }{ @@ -41,9 +42,10 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { input: "# Hello\n", want: []struct { kind internal.BlockKind + indent string content string }{ - {internal.BlockKindText, "# Hello\n"}, + {internal.BlockKindText, "", "# Hello\n"}, }, }, { @@ -60,15 +62,16 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { ), want: []struct { kind internal.BlockKind + indent string content string }{ - {internal.BlockKindText, "# Level 1\n"}, - {internal.BlockKindText, "\n"}, - {internal.BlockKindText, "some text\n"}, - {internal.BlockKindText, "\n"}, - {internal.BlockKindText, "## Level 2\n"}, - {internal.BlockKindText, "\n"}, - {internal.BlockKindText, "more text\n"}, + {internal.BlockKindText, "", "# Level 1\n"}, + {internal.BlockKindText, "", "\n"}, + {internal.BlockKindText, "", "some text\n"}, + {internal.BlockKindText, "", "\n"}, + {internal.BlockKindText, "", "## Level 2\n"}, + {internal.BlockKindText, "", "\n"}, + {internal.BlockKindText, "", "more text\n"}, }, }, { @@ -81,11 +84,12 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { ), want: []struct { kind internal.BlockKind + indent string content string }{ - {internal.BlockKindText, "text\nand more text\n"}, - {internal.BlockKindText, "\n"}, - {internal.BlockKindText, "last line"}, + {internal.BlockKindText, "", "text\nand more text\n"}, + {internal.BlockKindText, "", "\n"}, + {internal.BlockKindText, "", "last line"}, }, }, { @@ -97,9 +101,10 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { ), want: []struct { kind internal.BlockKind + indent string content string }{ - {internal.BlockKindText, "text\nand more text \nafter line break"}, + {internal.BlockKindText, "", "text\nand more text \nafter line break"}, }, }, { @@ -112,9 +117,10 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { ), want: []struct { kind internal.BlockKind + indent string content string }{ - {internal.BlockKindFencedCode, "```bash\necho \"hello\"\n```\n"}, + {internal.BlockKindFencedCode, "", "```bash\necho \"hello\"\n```\n"}, }, }, { @@ -126,9 +132,10 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { ), want: []struct { kind internal.BlockKind + indent string content string }{ - {internal.BlockKindFencedCode, "```bash\necho \"hello\"\n```"}, + {internal.BlockKindFencedCode, "", "```bash\necho \"hello\"\n```"}, }, }, { @@ -140,9 +147,10 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { ), want: []struct { kind internal.BlockKind + indent string content string }{ - {internal.BlockKindFencedCode, "~~~bash\necho \"hello\"\n~~~"}, + {internal.BlockKindFencedCode, "", "~~~bash\necho \"hello\"\n~~~"}, }, }, { @@ -156,10 +164,12 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { ), want: []struct { kind internal.BlockKind + indent string content string }{ { internal.BlockKindFencedCode, + "", "````md\n```bash\necho \"hello\"\n```\n````", }, }, @@ -172,10 +182,12 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { ), want: []struct { kind internal.BlockKind + indent string content string }{ { internal.BlockKindText, + "", " echo \"hello, \"\n echo \"world\"", }, }, @@ -185,9 +197,10 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { input: "text `echo \"hello, \" text`", want: []struct { kind internal.BlockKind + indent string content string }{ - {internal.BlockKindText, "text `echo \"hello, \" text`"}, + {internal.BlockKindText, "", "text `echo \"hello, \" text`"}, }, }, { @@ -195,11 +208,12 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { input: "text text", want: []struct { kind internal.BlockKind + indent string content string }{ - {internal.BlockKindText, "text "}, - {internal.BlockKindHTMLComment, ""}, - {internal.BlockKindText, " text"}, + {internal.BlockKindText, "", "text "}, + {internal.BlockKindHTMLComment, "", ""}, + {internal.BlockKindText, "", " text"}, }, }, { @@ -207,12 +221,13 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { input: "text text", want: []struct { kind internal.BlockKind + indent string content string }{ - {internal.BlockKindText, "text "}, - {internal.BlockKindHTMLComment, ""}, - {internal.BlockKindHTMLComment, ""}, - {internal.BlockKindText, " text"}, + {internal.BlockKindText, "", "text "}, + {internal.BlockKindHTMLComment, "", ""}, + {internal.BlockKindHTMLComment, "", ""}, + {internal.BlockKindText, "", " text"}, }, }, { @@ -226,11 +241,12 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { ), want: []struct { kind internal.BlockKind + indent string content string }{ - {internal.BlockKindText, "text\n"}, - {internal.BlockKindHTMLComment, "\n"}, - {internal.BlockKindText, "text"}, + {internal.BlockKindText, "", "text\n"}, + {internal.BlockKindHTMLComment, "", "\n"}, + {internal.BlockKindText, "", "text"}, }, }, { @@ -238,23 +254,47 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { input: "
\nhello\n
\n", want: []struct { kind internal.BlockKind + indent string content string }{ - {internal.BlockKindText, "
\nhello\n
\n"}, + {internal.BlockKindText, "", "
\nhello\n
\n"}, }, }, { - name: "mixed content", - input: "# Title\n\n```go\nfmt.Println()\n```\n\n\n", + name: "multiline blockquote", + input: joinLines( + "> hello, ", + "> world", + "> ", + "> again", + ), want: []struct { kind internal.BlockKind + indent string content string }{ - {internal.BlockKindText, "# Title\n"}, - {internal.BlockKindText, "\n"}, - {internal.BlockKindFencedCode, "```go\nfmt.Println()\n```\n"}, - {internal.BlockKindText, "\n"}, - {internal.BlockKindHTMLComment, "\n"}, + {internal.BlockKindText, "> ", "hello, \nworld\n"}, + {internal.BlockKindText, "> ", "\n"}, + {internal.BlockKindText, "> ", "again"}, + }, + }, + { + name: "blockquote with fenced code block", + input: joinLines( + "> text", + "> ```bash", + "> echo \"hello\"", + "> ```", + "> text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "> ", "text\n"}, + {internal.BlockKindFencedCode, "> ", "```bash\necho \"hello\"\n```\n"}, + {internal.BlockKindText, "> ", "text"}, }, }, } @@ -282,6 +322,13 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "block[%d] kind", i, ) + assert.Equal( + t, + w.indent, + string(blocks[i].Indent()), + "block[%d] indent", + i, + ) assert.Equal( t, w.content, From f07991528801102c40794fea5ebd0e08bcbe4232 Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Tue, 28 Apr 2026 18:11:12 -0600 Subject: [PATCH 03/18] Process nested blockquotes --- internal/block.go | 15 ++++++++++++++- internal/block_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/internal/block.go b/internal/block.go index dcffe87..59ac307 100644 --- a/internal/block.go +++ b/internal/block.go @@ -143,12 +143,25 @@ func stripIndent(content, indent []byte) []byte { return content } lines := bytes.Split(content, []byte("\n")) + parentIndent := quoteParentIndent(indent) for i, line := range lines { - lines[i] = bytes.TrimPrefix(line, indent) + switch { + case bytes.HasPrefix(line, indent): + lines[i] = bytes.TrimPrefix(line, indent) + case len(parentIndent) > 0 && bytes.Equal(line, parentIndent): + lines[i] = nil + } } return bytes.Join(lines, []byte("\n")) } +func quoteParentIndent(indent []byte) []byte { + if len(indent) < len("> ") || !bytes.HasSuffix(indent, []byte("> ")) { + return nil + } + return indent[:len(indent)-len("> ")] +} + func blockKind(node *sitter.Node, content []byte) BlockKind { switch node.Type() { case "fenced_code_block": diff --git a/internal/block_test.go b/internal/block_test.go index 27ecdb4..def20aa 100644 --- a/internal/block_test.go +++ b/internal/block_test.go @@ -297,6 +297,44 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { {internal.BlockKindText, "> ", "text"}, }, }, + { + name: "nested blockquote with fenced code block", + input: joinLines( + "> text", + "> > ```bash", + "> > echo \"hello\"", + "> > ```", + "> text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "> ", "text\n"}, + {internal.BlockKindFencedCode, "> > ", "```bash\necho \"hello\"\n```\n"}, + {internal.BlockKindText, "> ", "text"}, + }, + }, + { + name: "nested blockquote with html comment block", + input: joinLines( + "> text", + "> > ", + "> text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "> ", "text\n"}, + {internal.BlockKindHTMLComment, "> > ", "\n"}, + {internal.BlockKindText, "> ", "text"}, + }, + }, } for _, tt := range tests { From edda95a2c718a2e8d04c7ef8b41ae55127a6fe68 Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Tue, 28 Apr 2026 18:21:25 -0600 Subject: [PATCH 04/18] Process nested lists --- internal/block.go | 97 +++++++++++++++++++++++++++++++++++++++--- internal/block_test.go | 76 +++++++++++++++++++++++++++++++++ internal/cell.go | 39 ++++++++++++++++- 3 files changed, 204 insertions(+), 8 deletions(-) diff --git a/internal/block.go b/internal/block.go index 59ac307..0fa9b5b 100644 --- a/internal/block.go +++ b/internal/block.go @@ -54,7 +54,7 @@ func MakeBlocksFromMarkdown(content []byte) ([]Block, error) { var blocks []Block pos := uint32(0) - collectBlockNodes(root, content, nil, &pos, &blocks) + collectBlockNodes(root, content, nil, nil, &pos, &blocks) if pos < uint32(len(content)) { blocks = append(blocks, makeBlock(BlockKindText, content[pos:], nil)) @@ -95,21 +95,23 @@ func collectBlockNodes( node *sitter.Node, content []byte, indent []byte, + stripPrefix []byte, pos *uint32, blocks *[]Block, ) { switch node.Type() { case "document", "section": for i := 0; i < int(node.ChildCount()); i++ { - collectBlockNodes(node.Child(i), content, indent, pos, blocks) + collectBlockNodes(node.Child(i), content, indent, stripPrefix, pos, blocks) } case "block_quote": childIndent := append(append([]byte(nil), indent...), "> "...) + childStripPrefix := append(append([]byte(nil), stripPrefix...), "> "...) for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) if child.Type() == "block_quote_marker" { if child.StartByte() > *pos { - gap := stripIndent(content[*pos:child.StartByte()], childIndent) + gap := stripIndent(content[*pos:child.StartByte()], childStripPrefix) if len(gap) > 0 { *blocks = append(*blocks, makeBlock(BlockKindText, gap, childIndent)) } @@ -117,27 +119,94 @@ func collectBlockNodes( *pos = child.EndByte() continue } - collectBlockNodes(child, content, childIndent, pos, blocks) + collectBlockNodes(child, content, childIndent, childStripPrefix, pos, blocks) + } + case "list": + for i := 0; i < int(node.ChildCount()); i++ { + collectBlockNodes(node.Child(i), content, indent, stripPrefix, pos, blocks) + } + case "list_item": + childIndent, childStripPrefix := listItemIndent(node, content) + for i := 0; i < int(node.ChildCount()); i++ { + child := node.Child(i) + if isListMarker(child) { + if child.StartByte() > *pos { + gap := stripIndent(content[*pos:child.StartByte()], stripPrefix) + if len(gap) > 0 { + *blocks = append(*blocks, makeBlock(BlockKindText, gap, indent)) + } + } + *pos = child.EndByte() + continue + } + collectBlockNodes(child, content, childIndent, childStripPrefix, pos, blocks) } default: start := node.StartByte() end := node.EndByte() + blockIndent := indent + blockStripPrefix := stripPrefix + linePrefix := linePrefixBefore(content, start) + if len(linePrefix) > 0 && !bytes.Equal(linePrefix, indent) { + blockIndent = linePrefix + blockStripPrefix = linePrefix + if isSpaceIndent(linePrefix) { + rawLeading := leadingLineSpaces(content[start:end]) + if len(rawLeading) > 0 { + blockIndent = append(append([]byte(nil), linePrefix...), rawLeading...) + blockStripPrefix = blockIndent + } + } + } if start > *pos { - gap := stripIndent(content[*pos:start], indent) + gap := stripIndent(content[*pos:start], stripPrefix) if len(gap) > 0 { *blocks = append(*blocks, makeBlock(BlockKindText, gap, indent)) } } - raw := stripIndent(content[start:end], indent) + raw := stripIndent(content[start:end], blockStripPrefix) if len(raw) > 0 { - *blocks = append(*blocks, makeBlock(blockKind(node, content), raw, indent)) + *blocks = append(*blocks, makeBlock(blockKind(node, content), raw, blockIndent)) } *pos = end } } +func listItemIndent(node *sitter.Node, content []byte) ([]byte, []byte) { + for i := 0; i < int(node.ChildCount()); i++ { + child := node.Child(i) + if !isListMarker(child) { + continue + } + linePrefix := linePrefixBefore(content, child.StartByte()) + marker := content[child.StartByte():child.EndByte()] + indent := append(append([]byte(nil), linePrefix...), marker...) + stripPrefix := append(append([]byte(nil), linePrefix...), bytes.Repeat([]byte(" "), len(marker))...) + return indent, stripPrefix + } + return nil, nil +} + +func isListMarker(node *sitter.Node) bool { + return bytes.HasPrefix([]byte(node.Type()), []byte("list_marker_")) +} + +func linePrefixBefore(content []byte, pos uint32) []byte { + lineStart := bytes.LastIndexByte(content[:pos], '\n') + 1 + return content[lineStart:pos] +} + +func leadingLineSpaces(content []byte) []byte { + end := bytes.IndexByte(content, '\n') + if end < 0 { + end = len(content) + } + line := content[:end] + return line[:len(line)-len(bytes.TrimLeft(line, " "))] +} + func stripIndent(content, indent []byte) []byte { if len(indent) == 0 { return content @@ -148,6 +217,8 @@ func stripIndent(content, indent []byte) []byte { switch { case bytes.HasPrefix(line, indent): lines[i] = bytes.TrimPrefix(line, indent) + case isSpaceIndent(indent): + lines[i] = trimSpaceIndent(line, len(indent)) case len(parentIndent) > 0 && bytes.Equal(line, parentIndent): lines[i] = nil } @@ -155,6 +226,18 @@ func stripIndent(content, indent []byte) []byte { return bytes.Join(lines, []byte("\n")) } +func isSpaceIndent(indent []byte) bool { + return len(bytes.Trim(indent, " ")) == 0 +} + +func trimSpaceIndent(line []byte, width int) []byte { + i := 0 + for i < len(line) && i < width && line[i] == ' ' { + i++ + } + return line[i:] +} + func quoteParentIndent(indent []byte) []byte { if len(indent) < len("> ") || !bytes.HasSuffix(indent, []byte("> ")) { return nil diff --git a/internal/block_test.go b/internal/block_test.go index def20aa..26db1b2 100644 --- a/internal/block_test.go +++ b/internal/block_test.go @@ -335,6 +335,82 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { {internal.BlockKindText, "> ", "text"}, }, }, + { + name: "list with fenced code block", + input: joinLines( + "- text", + "- ```bash", + " echo \"hello\"", + " ```", + "- text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "- ", "text\n"}, + {internal.BlockKindFencedCode, "- ", "```bash\necho \"hello\"\n```\n"}, + {internal.BlockKindText, "- ", "text"}, + }, + }, + { + name: "list with html comment block", + input: joinLines( + "- text", + "- ", + "- text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "- ", "text\n"}, + {internal.BlockKindHTMLComment, "- ", "\n"}, + {internal.BlockKindText, "- ", "text"}, + }, + }, + { + name: "nested list with fenced code block", + input: joinLines( + "- text", + " - ```bash", + " echo \"hello\"", + " ```", + "- text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "- ", "text\n"}, + {internal.BlockKindFencedCode, " - ", "```bash\necho \"hello\"\n```\n"}, + {internal.BlockKindText, "- ", "text"}, + }, + }, + { + name: "nested list with html comment block", + input: joinLines( + "- text", + " - ", + "- text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "- ", "text\n"}, + {internal.BlockKindHTMLComment, " - ", "\n"}, + {internal.BlockKindText, "- ", "text"}, + }, + }, } for _, tt := range tests { diff --git a/internal/cell.go b/internal/cell.go index 8569dcb..55588cb 100644 --- a/internal/cell.go +++ b/internal/cell.go @@ -83,12 +83,49 @@ func Classify(blocks []Block) ([]Cell, error) { case info.IsLitdoc: return nil, fmt.Errorf("unsupported language: %q", info.Lang) default: - cells = append(cells, MakeStaticCellFromRaw(string(b.content))) + cells = append(cells, MakeStaticCellFromRaw(renderStaticBlock(b))) } } return cells, nil } +func renderStaticBlock(b Block) string { + if len(b.indent) == 0 { + return string(b.content) + } + + lines := bytes.Split(b.content, []byte("\n")) + var rendered bytes.Buffer + continuationIndent := blockContinuationIndent(b.indent) + for i, line := range lines { + if i == len(lines)-1 && len(line) == 0 { + break + } + if i > 0 { + rendered.WriteByte('\n') + if len(line) > 0 { + rendered.Write(continuationIndent) + } + } else { + if len(line) > 0 { + rendered.Write(b.indent) + } + } + rendered.Write(line) + } + if bytes.HasSuffix(b.content, []byte("\n")) { + rendered.WriteByte('\n') + } + return rendered.String() +} + +func blockContinuationIndent(indent []byte) []byte { + if bytes.HasSuffix(indent, []byte("> ")) { + return indent + } + return bytes.Repeat([]byte(" "), len(indent)) +} + func Execute(cells []Cell) ([]Cell, error) { var executedCells []Cell for _, c := range cells { From b07ad3d56d19007acc9b2173705103c8e51de76c Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Tue, 28 Apr 2026 18:25:13 -0600 Subject: [PATCH 05/18] Process nested lists and block quotes --- internal/block.go | 15 +++ internal/block_test.go | 211 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 226 insertions(+) diff --git a/internal/block.go b/internal/block.go index 0fa9b5b..dcefc59 100644 --- a/internal/block.go +++ b/internal/block.go @@ -219,6 +219,8 @@ func stripIndent(content, indent []byte) []byte { lines[i] = bytes.TrimPrefix(line, indent) case isSpaceIndent(indent): lines[i] = trimSpaceIndent(line, len(indent)) + case bytes.Contains(indent, []byte("> ")) && isQuoteOnlyLine(line): + lines[i] = nil case len(parentIndent) > 0 && bytes.Equal(line, parentIndent): lines[i] = nil } @@ -238,6 +240,19 @@ func trimSpaceIndent(line []byte, width int) []byte { return line[i:] } +func isQuoteOnlyLine(line []byte) bool { + trimmed := bytes.TrimSpace(line) + if len(trimmed) == 0 { + return false + } + for _, b := range trimmed { + if b != '>' && b != ' ' { + return false + } + } + return true +} + func quoteParentIndent(indent []byte) []byte { if len(indent) < len("> ") || !bytes.HasSuffix(indent, []byte("> ")) { return nil diff --git a/internal/block_test.go b/internal/block_test.go index 26db1b2..36bbd22 100644 --- a/internal/block_test.go +++ b/internal/block_test.go @@ -411,6 +411,217 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { {internal.BlockKindText, "- ", "text"}, }, }, + { + name: "blockquote with nested list fenced code block", + input: joinLines( + "> text", + "> - item", + "> - ```bash", + "> echo \"hello\"", + "> ```", + "> text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "> ", "text\n"}, + {internal.BlockKindText, "> - ", "item\n"}, + {internal.BlockKindFencedCode, "> - ", "```bash\necho \"hello\"\n```\n"}, + {internal.BlockKindText, "> ", "text"}, + }, + }, + { + name: "blockquote with list html comment block", + input: joinLines( + "> text", + "> - ", + "> text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "> ", "text\n"}, + {internal.BlockKindHTMLComment, "> - ", "\n"}, + {internal.BlockKindText, "> ", "text"}, + }, + }, + { + name: "list with blockquote fenced code block", + input: joinLines( + "- text", + " > ```bash", + " > echo \"hello\"", + " > ```", + "- text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "- ", "text\n"}, + {internal.BlockKindFencedCode, " > ", "```bash\necho \"hello\"\n```\n"}, + {internal.BlockKindText, "- ", "text"}, + }, + }, + { + name: "nested blockquote with list html comment block", + input: joinLines( + "> text", + "> > - ", + "> text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "> ", "text\n"}, + {internal.BlockKindHTMLComment, "> > - ", "\n"}, + {internal.BlockKindText, "> ", "text"}, + }, + }, + { + name: "ordered list with fenced code block", + input: joinLines( + "1. text", + "2. ```bash", + " echo \"hello\"", + " ```", + "3. text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "1. ", "text\n"}, + {internal.BlockKindFencedCode, "2. ", "```bash\necho \"hello\"\n```\n"}, + {internal.BlockKindText, "3. ", "text"}, + }, + }, + { + name: "ordered list with html comment block", + input: joinLines( + "1. text", + "2. ", + "3. text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "1. ", "text\n"}, + {internal.BlockKindHTMLComment, "2. ", "\n"}, + {internal.BlockKindText, "3. ", "text"}, + }, + }, + { + name: "nested ordered list with fenced code block", + input: joinLines( + "1. text", + " 1. ```bash", + " echo \"hello\"", + " ```", + "2. text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "1. ", "text\n"}, + {internal.BlockKindFencedCode, " 1. ", "```bash\necho \"hello\"\n```\n"}, + {internal.BlockKindText, "2. ", "text"}, + }, + }, + { + name: "plus list with tilde fenced code block", + input: joinLines( + "+ text", + "+ ~~~bash", + " echo \"hello\"", + " ~~~", + "+ text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "+ ", "text\n"}, + {internal.BlockKindFencedCode, "+ ", "~~~bash\necho \"hello\"\n~~~\n"}, + {internal.BlockKindText, "+ ", "text"}, + }, + }, + { + name: "star list with html comment block", + input: joinLines( + "* text", + "* ", + "* text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "* ", "text\n"}, + {internal.BlockKindHTMLComment, "* ", "\n"}, + {internal.BlockKindText, "* ", "text"}, + }, + }, + { + name: "blockquote with ordered list fenced code block", + input: joinLines( + "> text", + "> 1. ```bash", + "> echo \"hello\"", + "> ```", + "> text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "> ", "text\n"}, + {internal.BlockKindFencedCode, "> 1. ", "```bash\necho \"hello\"\n```\n"}, + {internal.BlockKindText, "> ", "text"}, + }, + }, + { + name: "ordered list with blockquote html comment block", + input: joinLines( + "1. text", + " > ", + "2. text", + ), + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "1. ", "text\n"}, + {internal.BlockKindHTMLComment, " > ", "\n"}, + {internal.BlockKindText, "2. ", "text"}, + }, + }, } for _, tt := range tests { From 386872d2262b4bb80123f74c72da877bd4430bb8 Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Wed, 29 Apr 2026 10:19:39 -0600 Subject: [PATCH 06/18] Don't extract indent for inline comments --- internal/block.go | 11 ++++++++- internal/block_test.go | 56 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/internal/block.go b/internal/block.go index dcefc59..4b85572 100644 --- a/internal/block.go +++ b/internal/block.go @@ -81,7 +81,11 @@ func splitInlineHTMLComments(blocks []Block) []Block { if loc[0] > pos { result = append(result, makeBlock(BlockKindText, content[pos:loc[0]], b.indent)) } - result = append(result, makeBlock(BlockKindHTMLComment, content[loc[0]:loc[1]], b.indent)) + commentIndent := []byte(nil) + if isWholeTextBlock(content, loc) { + commentIndent = b.indent + } + result = append(result, makeBlock(BlockKindHTMLComment, content[loc[0]:loc[1]], commentIndent)) pos = loc[1] } if pos < len(content) { @@ -91,6 +95,11 @@ func splitInlineHTMLComments(blocks []Block) []Block { return result } +func isWholeTextBlock(content []byte, loc []int) bool { + return len(bytes.TrimSpace(content[:loc[0]])) == 0 && + len(bytes.TrimSpace(content[loc[1]:])) == 0 +} + func collectBlockNodes( node *sitter.Node, content []byte, diff --git a/internal/block_test.go b/internal/block_test.go index 36bbd22..93ee710 100644 --- a/internal/block_test.go +++ b/internal/block_test.go @@ -230,6 +230,62 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { {internal.BlockKindText, "", " text"}, }, }, + { + name: "blockquote with multiple inline html comments", + input: "> text text", + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "> ", "text "}, + {internal.BlockKindHTMLComment, "", ""}, + {internal.BlockKindHTMLComment, "", ""}, + {internal.BlockKindText, "> ", " text"}, + }, + }, + { + name: "list with multiple inline html comments", + input: "- text text", + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "- ", "text "}, + {internal.BlockKindHTMLComment, "", ""}, + {internal.BlockKindHTMLComment, "", ""}, + {internal.BlockKindText, "- ", " text"}, + }, + }, + { + name: "nested list with multiple inline html comments", + input: " - text text", + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, " - ", "text "}, + {internal.BlockKindHTMLComment, "", ""}, + {internal.BlockKindHTMLComment, "", ""}, + {internal.BlockKindText, " - ", " text"}, + }, + }, + { + name: "blockquote list with multiple inline html comments", + input: "> - text text", + want: []struct { + kind internal.BlockKind + indent string + content string + }{ + {internal.BlockKindText, "> - ", "text "}, + {internal.BlockKindHTMLComment, "", ""}, + {internal.BlockKindHTMLComment, "", ""}, + {internal.BlockKindText, "> - ", " text"}, + }, + }, { name: "html comment block comment", input: joinLines( From 741a6e220e7ffc7021ff1a468ec860ab66fbc52a Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Wed, 29 Apr 2026 10:54:04 -0600 Subject: [PATCH 07/18] Restructure tests --- internal/block.go | 30 +- internal/block_test.go | 875 ++++++++++++++++++++--------------------- internal/cell_test.go | 18 +- 3 files changed, 445 insertions(+), 478 deletions(-) diff --git a/internal/block.go b/internal/block.go index 4b85572..20c44ee 100644 --- a/internal/block.go +++ b/internal/block.go @@ -24,11 +24,7 @@ type Block struct { content []byte } -func MakeBlockFromRaw(kind BlockKind, raw []byte) Block { - return Block{kind: kind, content: raw} -} - -func makeBlock(kind BlockKind, content, indent []byte) Block { +func MakeBlock(kind BlockKind, content, indent []byte) Block { return Block{kind: kind, indent: indent, content: content} } @@ -57,7 +53,7 @@ func MakeBlocksFromMarkdown(content []byte) ([]Block, error) { collectBlockNodes(root, content, nil, nil, &pos, &blocks) if pos < uint32(len(content)) { - blocks = append(blocks, makeBlock(BlockKindText, content[pos:], nil)) + blocks = append(blocks, MakeBlock(BlockKindText, content[pos:], nil)) } return splitInlineHTMLComments(blocks), nil @@ -79,17 +75,17 @@ func splitInlineHTMLComments(blocks []Block) []Block { pos := 0 for _, loc := range locs { if loc[0] > pos { - result = append(result, makeBlock(BlockKindText, content[pos:loc[0]], b.indent)) + result = append(result, MakeBlock(BlockKindText, content[pos:loc[0]], b.indent)) } commentIndent := []byte(nil) if isWholeTextBlock(content, loc) { commentIndent = b.indent } - result = append(result, makeBlock(BlockKindHTMLComment, content[loc[0]:loc[1]], commentIndent)) + result = append(result, MakeBlock(BlockKindHTMLComment, content[loc[0]:loc[1]], commentIndent)) pos = loc[1] } if pos < len(content) { - result = append(result, makeBlock(BlockKindText, content[pos:], b.indent)) + result = append(result, MakeBlock(BlockKindText, content[pos:], b.indent)) } } return result @@ -122,7 +118,7 @@ func collectBlockNodes( if child.StartByte() > *pos { gap := stripIndent(content[*pos:child.StartByte()], childStripPrefix) if len(gap) > 0 { - *blocks = append(*blocks, makeBlock(BlockKindText, gap, childIndent)) + *blocks = append(*blocks, MakeBlock(BlockKindText, gap, childIndent)) } } *pos = child.EndByte() @@ -142,7 +138,7 @@ func collectBlockNodes( if child.StartByte() > *pos { gap := stripIndent(content[*pos:child.StartByte()], stripPrefix) if len(gap) > 0 { - *blocks = append(*blocks, makeBlock(BlockKindText, gap, indent)) + *blocks = append(*blocks, MakeBlock(BlockKindText, gap, indent)) } } *pos = child.EndByte() @@ -166,18 +162,26 @@ func collectBlockNodes( blockStripPrefix = blockIndent } } + } else if node.Type() == "fenced_code_block" && len(indent) == 0 { + // CommonMark §4.5: a fence may be indented up to 3 spaces; that indent is + // part of the block's prefix, not the code content. + rawLeading := leadingLineSpaces(content[start:end]) + if n := len(rawLeading); n > 0 && n <= 3 { + blockIndent = rawLeading + blockStripPrefix = rawLeading + } } if start > *pos { gap := stripIndent(content[*pos:start], stripPrefix) if len(gap) > 0 { - *blocks = append(*blocks, makeBlock(BlockKindText, gap, indent)) + *blocks = append(*blocks, MakeBlock(BlockKindText, gap, indent)) } } raw := stripIndent(content[start:end], blockStripPrefix) if len(raw) > 0 { - *blocks = append(*blocks, makeBlock(blockKind(node, content), raw, blockIndent)) + *blocks = append(*blocks, MakeBlock(blockKind(node, content), raw, blockIndent)) } *pos = end } diff --git a/internal/block_test.go b/internal/block_test.go index 93ee710..ff51fc4 100644 --- a/internal/block_test.go +++ b/internal/block_test.go @@ -10,46 +10,59 @@ import ( "github.com/stretchr/testify/require" ) -func TestMakeBlockFromRaw(t *testing.T) { +type wantBlock struct { + kind internal.BlockKind + indent string + content string +} + +func text(indent, content string) wantBlock { + return wantBlock{internal.BlockKindText, indent, content} +} + +func code(indent, content string) wantBlock { + return wantBlock{internal.BlockKindFencedCode, indent, content} +} + +func comment(indent, content string) wantBlock { + return wantBlock{internal.BlockKindHTMLComment, indent, content} +} + +func joinLines(lines ...string) string { + return strings.Join(lines, "\n") +} + +func TestMakeBlock(t *testing.T) { // given kind := internal.BlockKindFencedCode content := []byte("```bash\necho hello\n```\n") + indent := []byte("> ") // when - got := internal.MakeBlockFromRaw(kind, content) + got := internal.MakeBlock(kind, content, indent) // then assert.Equal(t, kind, got.Kind()) assert.Equal(t, content, got.Content()) -} - -func joinLines(lines ...string) string { - return strings.Join(lines, "\n") + assert.Equal(t, indent, got.Indent()) } func TestMakeBlocksFromMarkdown(t *testing.T) { tests := []struct { name string input string - want []struct { - kind internal.BlockKind - indent string - content string - } + want []wantBlock }{ + // Text — baseline cases that should produce no FencedCode or HTMLComment blocks. { - name: "heading", + name: "Text/heading", input: "# Hello\n", - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "", "# Hello\n"}, + want: []wantBlock{ + text("", "# Hello\n"), }, }, { - name: "nested heading and text", + name: "Text/heading/nested", input: joinLines( "# Level 1", "", @@ -60,101 +73,117 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "more text", "", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "", "# Level 1\n"}, - {internal.BlockKindText, "", "\n"}, - {internal.BlockKindText, "", "some text\n"}, - {internal.BlockKindText, "", "\n"}, - {internal.BlockKindText, "", "## Level 2\n"}, - {internal.BlockKindText, "", "\n"}, - {internal.BlockKindText, "", "more text\n"}, + want: []wantBlock{ + text("", "# Level 1\n"), + text("", "\n"), + text("", "some text\n"), + text("", "\n"), + text("", "## Level 2\n"), + text("", "\n"), + text("", "more text\n"), }, }, { - name: "paragraphs grouped together", + name: "Text/paragraph/grouped", input: joinLines( "text", "and more text", "", "last line", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "", "text\nand more text\n"}, - {internal.BlockKindText, "", "\n"}, - {internal.BlockKindText, "", "last line"}, + want: []wantBlock{ + text("", "text\nand more text\n"), + text("", "\n"), + text("", "last line"), }, }, { - name: "line break grouped with paragraph", + name: "Text/paragraph/line-break", input: joinLines( "text", "and more text ", "after line break", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "", "text\nand more text \nafter line break"}, + want: []wantBlock{ + text("", "text\nand more text \nafter line break"), + }, + }, + { + name: "Text/indented-code-as-text", + input: joinLines( + " echo \"hello, \"", + " echo \"world\"", + ), + want: []wantBlock{ + text("", " echo \"hello, \"\n echo \"world\""), }, }, { - name: "fenced code block as single block", + name: "Text/inline-code-span", + input: "text `echo \"hello, \" text`", + want: []wantBlock{ + text("", "text `echo \"hello, \" text`"), + }, + }, + { + name: "Text/html-block-not-comment", + input: "
\nhello\n
\n", + want: []wantBlock{ + text("", "
\nhello\n
\n"), + }, + }, + { + name: "Text/blockquote/multiline", + input: joinLines( + "> hello, ", + "> world", + "> ", + "> again", + ), + want: []wantBlock{ + text("> ", "hello, \nworld\n"), + text("> ", "\n"), + text("> ", "again"), + }, + }, + + // FencedCode — top-level (CommonMark §4.5). + { + name: "FencedCode/top-level/backtick", input: joinLines( "```bash", "echo \"hello\"", "```", "", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindFencedCode, "", "```bash\necho \"hello\"\n```\n"}, + want: []wantBlock{ + code("", "```bash\necho \"hello\"\n```\n"), }, }, { - name: "fenced code block with no trailing newline", + name: "FencedCode/top-level/backtick-no-trailing-newline", input: joinLines( "```bash", "echo \"hello\"", "```", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindFencedCode, "", "```bash\necho \"hello\"\n```"}, + want: []wantBlock{ + code("", "```bash\necho \"hello\"\n```"), }, }, { - name: "tilde fenced code block as single block", + name: "FencedCode/top-level/tilde", input: joinLines( "~~~bash", "echo \"hello\"", "~~~", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindFencedCode, "", "~~~bash\necho \"hello\"\n~~~"}, + want: []wantBlock{ + code("", "~~~bash\necho \"hello\"\n~~~"), }, }, { - name: "longer backtick fence as single block", + name: "FencedCode/top-level/longer-fence-around-fence", input: joinLines( "````md", "```bash", @@ -162,180 +191,84 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "```", "````", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - { - internal.BlockKindFencedCode, - "", - "````md\n```bash\necho \"hello\"\n```\n````", - }, + want: []wantBlock{ + code("", "````md\n```bash\necho \"hello\"\n```\n````"), }, }, { - name: "indented code block", + name: "FencedCode/top-level/no-info-string", input: joinLines( - " echo \"hello, \"", - " echo \"world\"", + "```", + "echo \"hello\"", + "```", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - { - internal.BlockKindText, - "", - " echo \"hello, \"\n echo \"world\"", - }, - }, - }, - { - name: "inline code block", - input: "text `echo \"hello, \" text`", - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "", "text `echo \"hello, \" text`"}, + want: []wantBlock{ + code("", "```\necho \"hello\"\n```"), }, }, { - name: "html comment inline comment", - input: "text text", - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "", "text "}, - {internal.BlockKindHTMLComment, "", ""}, - {internal.BlockKindText, "", " text"}, - }, - }, - { - name: "multiple inline html comments", - input: "text text", - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "", "text "}, - {internal.BlockKindHTMLComment, "", ""}, - {internal.BlockKindHTMLComment, "", ""}, - {internal.BlockKindText, "", " text"}, - }, - }, - { - name: "blockquote with multiple inline html comments", - input: "> text text", - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "> ", "text "}, - {internal.BlockKindHTMLComment, "", ""}, - {internal.BlockKindHTMLComment, "", ""}, - {internal.BlockKindText, "> ", " text"}, - }, - }, - { - name: "list with multiple inline html comments", - input: "- text text", - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "- ", "text "}, - {internal.BlockKindHTMLComment, "", ""}, - {internal.BlockKindHTMLComment, "", ""}, - {internal.BlockKindText, "- ", " text"}, - }, - }, - { - name: "nested list with multiple inline html comments", - input: " - text text", - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, " - ", "text "}, - {internal.BlockKindHTMLComment, "", ""}, - {internal.BlockKindHTMLComment, "", ""}, - {internal.BlockKindText, " - ", " text"}, + name: "FencedCode/top-level/empty", + input: joinLines( + "```bash", + "```", + ), + want: []wantBlock{ + code("", "```bash\n```"), }, }, { - name: "blockquote list with multiple inline html comments", - input: "> - text text", - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "> - ", "text "}, - {internal.BlockKindHTMLComment, "", ""}, - {internal.BlockKindHTMLComment, "", ""}, - {internal.BlockKindText, "> - ", " text"}, + name: "FencedCode/top-level/consecutive", + input: joinLines( + "```bash", + "echo a", + "```", + "", + "```bash", + "echo b", + "```", + ), + want: []wantBlock{ + code("", "```bash\necho a\n```\n"), + text("", "\n"), + code("", "```bash\necho b\n```"), }, }, { - name: "html comment block comment", + name: "FencedCode/top-level/indented-1-space", input: joinLines( - "text", - "", - "text", + " ```bash", + " echo \"hello\"", + " ```", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "", "text\n"}, - {internal.BlockKindHTMLComment, "", "\n"}, - {internal.BlockKindText, "", "text"}, + want: []wantBlock{ + code(" ", "```bash\necho \"hello\"\n```"), }, }, { - name: "html block is not a comment", - input: "
\nhello\n
\n", - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "", "
\nhello\n
\n"}, + name: "FencedCode/top-level/indented-3-spaces", + input: joinLines( + " ```bash", + " echo \"hello\"", + " ```", + ), + want: []wantBlock{ + code(" ", "```bash\necho \"hello\"\n```"), }, }, { - name: "multiline blockquote", + name: "FencedCode/top-level/unclosed-runs-to-eof", input: joinLines( - "> hello, ", - "> world", - "> ", - "> again", + "```bash", + "echo \"hello\"", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "> ", "hello, \nworld\n"}, - {internal.BlockKindText, "> ", "\n"}, - {internal.BlockKindText, "> ", "again"}, + want: []wantBlock{ + code("", "```bash\necho \"hello\""), }, }, + + // FencedCode — inside containers. { - name: "blockquote with fenced code block", + name: "FencedCode/blockquote", input: joinLines( "> text", "> ```bash", @@ -343,18 +276,14 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> ```", "> text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "> ", "text\n"}, - {internal.BlockKindFencedCode, "> ", "```bash\necho \"hello\"\n```\n"}, - {internal.BlockKindText, "> ", "text"}, + want: []wantBlock{ + text("> ", "text\n"), + code("> ", "```bash\necho \"hello\"\n```\n"), + text("> ", "text"), }, }, { - name: "nested blockquote with fenced code block", + name: "FencedCode/blockquote/nested", input: joinLines( "> text", "> > ```bash", @@ -362,37 +291,14 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> > ```", "> text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "> ", "text\n"}, - {internal.BlockKindFencedCode, "> > ", "```bash\necho \"hello\"\n```\n"}, - {internal.BlockKindText, "> ", "text"}, - }, - }, - { - name: "nested blockquote with html comment block", - input: joinLines( - "> text", - "> > ", - "> text", - ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "> ", "text\n"}, - {internal.BlockKindHTMLComment, "> > ", "\n"}, - {internal.BlockKindText, "> ", "text"}, + want: []wantBlock{ + text("> ", "text\n"), + code("> > ", "```bash\necho \"hello\"\n```\n"), + text("> ", "text"), }, }, { - name: "list with fenced code block", + name: "FencedCode/list/dash", input: joinLines( "- text", "- ```bash", @@ -400,75 +306,89 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " ```", "- text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "- ", "text\n"}, - {internal.BlockKindFencedCode, "- ", "```bash\necho \"hello\"\n```\n"}, - {internal.BlockKindText, "- ", "text"}, + want: []wantBlock{ + text("- ", "text\n"), + code("- ", "```bash\necho \"hello\"\n```\n"), + text("- ", "text"), }, }, { - name: "list with html comment block", + name: "FencedCode/list/dash/nested", input: joinLines( "- text", - "- ", + " - ```bash", + " echo \"hello\"", + " ```", "- text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "- ", "text\n"}, - {internal.BlockKindHTMLComment, "- ", "\n"}, - {internal.BlockKindText, "- ", "text"}, + want: []wantBlock{ + text("- ", "text\n"), + code(" - ", "```bash\necho \"hello\"\n```\n"), + text("- ", "text"), }, }, { - name: "nested list with fenced code block", + name: "FencedCode/list/plus-tilde-fence", input: joinLines( - "- text", - " - ```bash", - " echo \"hello\"", - " ```", - "- text", + "+ text", + "+ ~~~bash", + " echo \"hello\"", + " ~~~", + "+ text", + ), + want: []wantBlock{ + text("+ ", "text\n"), + code("+ ", "~~~bash\necho \"hello\"\n~~~\n"), + text("+ ", "text"), + }, + }, + { + name: "FencedCode/list/ordered", + input: joinLines( + "1. text", + "2. ```bash", + " echo \"hello\"", + " ```", + "3. text", + ), + want: []wantBlock{ + text("1. ", "text\n"), + code("2. ", "```bash\necho \"hello\"\n```\n"), + text("3. ", "text"), + }, + }, + { + name: "FencedCode/list/ordered/nested", + input: joinLines( + "1. text", + " 1. ```bash", + " echo \"hello\"", + " ```", + "2. text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "- ", "text\n"}, - {internal.BlockKindFencedCode, " - ", "```bash\necho \"hello\"\n```\n"}, - {internal.BlockKindText, "- ", "text"}, + want: []wantBlock{ + text("1. ", "text\n"), + code(" 1. ", "```bash\necho \"hello\"\n```\n"), + text("2. ", "text"), }, }, { - name: "nested list with html comment block", + name: "FencedCode/list/blockquote", input: joinLines( "- text", - " - ", + " > ```bash", + " > echo \"hello\"", + " > ```", "- text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "- ", "text\n"}, - {internal.BlockKindHTMLComment, " - ", "\n"}, - {internal.BlockKindText, "- ", "text"}, + want: []wantBlock{ + text("- ", "text\n"), + code(" > ", "```bash\necho \"hello\"\n```\n"), + text("- ", "text"), }, }, { - name: "blockquote with nested list fenced code block", + name: "FencedCode/blockquote/list-nested", input: joinLines( "> text", "> - item", @@ -477,152 +397,181 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> ```", "> text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "> ", "text\n"}, - {internal.BlockKindText, "> - ", "item\n"}, - {internal.BlockKindFencedCode, "> - ", "```bash\necho \"hello\"\n```\n"}, - {internal.BlockKindText, "> ", "text"}, + want: []wantBlock{ + text("> ", "text\n"), + text("> - ", "item\n"), + code("> - ", "```bash\necho \"hello\"\n```\n"), + text("> ", "text"), }, }, { - name: "blockquote with list html comment block", + name: "FencedCode/blockquote/list-ordered", input: joinLines( "> text", - "> - ", + "> 1. ```bash", + "> echo \"hello\"", + "> ```", "> text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "> ", "text\n"}, - {internal.BlockKindHTMLComment, "> - ", "\n"}, - {internal.BlockKindText, "> ", "text"}, + want: []wantBlock{ + text("> ", "text\n"), + code("> 1. ", "```bash\necho \"hello\"\n```\n"), + text("> ", "text"), }, }, + + // HTMLComment — top-level (CommonMark §4.6 type 2). { - name: "list with blockquote fenced code block", - input: joinLines( - "- text", - " > ```bash", - " > echo \"hello\"", - " > ```", - "- text", - ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "- ", "text\n"}, - {internal.BlockKindFencedCode, " > ", "```bash\necho \"hello\"\n```\n"}, - {internal.BlockKindText, "- ", "text"}, + name: "HTMLComment/top-level/inline", + input: "text text", + want: []wantBlock{ + text("", "text "), + comment("", ""), + text("", " text"), + }, + }, + { + name: "HTMLComment/top-level/inline-multiple", + input: "text text", + want: []wantBlock{ + text("", "text "), + comment("", ""), + comment("", ""), + text("", " text"), }, }, { - name: "nested blockquote with list html comment block", + name: "HTMLComment/top-level/inline-at-line-end", + input: "text ", + want: []wantBlock{ + text("", "text "), + comment("", ""), + }, + }, + { + name: "HTMLComment/top-level/block", input: joinLines( - "> text", - "> > - ", - "> text", + "text", + "", + "text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "> ", "text\n"}, - {internal.BlockKindHTMLComment, "> > - ", "\n"}, - {internal.BlockKindText, "> ", "text"}, + want: []wantBlock{ + text("", "text\n"), + comment("", "\n"), + text("", "text"), }, }, { - name: "ordered list with fenced code block", + name: "HTMLComment/top-level/block-empty", + input: "\n", + want: []wantBlock{ + comment("", "\n"), + }, + }, + { + name: "HTMLComment/top-level/block-consecutive", input: joinLines( - "1. text", - "2. ```bash", - " echo \"hello\"", - " ```", - "3. text", + "", + "", + "", + "", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "1. ", "text\n"}, - {internal.BlockKindFencedCode, "2. ", "```bash\necho \"hello\"\n```\n"}, - {internal.BlockKindText, "3. ", "text"}, + want: []wantBlock{ + comment("", "\n"), + text("", "\n"), + comment("", "\n"), + }, + }, + { + name: "HTMLComment/top-level/in-heading-line", + input: "# Title \n", + want: []wantBlock{ + text("", "# Title "), + comment("", ""), + text("", "\n"), + }, + }, + + // HTMLComment — inside containers. + { + name: "HTMLComment/blockquote/inline-multiple", + input: "> text text", + want: []wantBlock{ + text("> ", "text "), + comment("", ""), + comment("", ""), + text("> ", " text"), }, }, { - name: "ordered list with html comment block", + name: "HTMLComment/blockquote/nested/block", input: joinLines( - "1. text", - "2. ", - "3. text", + "> text", + "> > ", + "> text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "1. ", "text\n"}, - {internal.BlockKindHTMLComment, "2. ", "\n"}, - {internal.BlockKindText, "3. ", "text"}, + want: []wantBlock{ + text("> ", "text\n"), + comment("> > ", "\n"), + text("> ", "text"), }, }, { - name: "nested ordered list with fenced code block", + name: "HTMLComment/list/dash/inline-multiple", + input: "- text text", + want: []wantBlock{ + text("- ", "text "), + comment("", ""), + comment("", ""), + text("- ", " text"), + }, + }, + { + name: "HTMLComment/list/dash/nested/inline-multiple", + input: " - text text", + want: []wantBlock{ + text(" - ", "text "), + comment("", ""), + comment("", ""), + text(" - ", " text"), + }, + }, + { + name: "HTMLComment/list/dash/block", input: joinLines( - "1. text", - " 1. ```bash", - " echo \"hello\"", - " ```", - "2. text", + "- text", + "- ", + "- text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "1. ", "text\n"}, - {internal.BlockKindFencedCode, " 1. ", "```bash\necho \"hello\"\n```\n"}, - {internal.BlockKindText, "2. ", "text"}, + want: []wantBlock{ + text("- ", "text\n"), + comment("- ", "\n"), + text("- ", "text"), }, }, { - name: "plus list with tilde fenced code block", + name: "HTMLComment/list/dash/nested/block", input: joinLines( - "+ text", - "+ ~~~bash", - " echo \"hello\"", - " ~~~", - "+ text", + "- text", + " - ", + "- text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "+ ", "text\n"}, - {internal.BlockKindFencedCode, "+ ", "~~~bash\necho \"hello\"\n~~~\n"}, - {internal.BlockKindText, "+ ", "text"}, + want: []wantBlock{ + text("- ", "text\n"), + comment(" - ", "\n"), + text("- ", "text"), }, }, { - name: "star list with html comment block", + name: "HTMLComment/list/star/block", input: joinLines( "* text", "* ", "* text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "* ", "text\n"}, - {internal.BlockKindHTMLComment, "* ", "\n"}, - {internal.BlockKindText, "* ", "text"}, + want: []wantBlock{ + text("* ", "text\n"), + comment("* ", "\n"), + text("* ", "text"), + }, + }, + { + name: "HTMLComment/list/ordered/block", + input: joinLines( + "1. text", + "2. ", + "3. text", + ), + want: []wantBlock{ + text("1. ", "text\n"), + comment("2. ", "\n"), + text("3. ", "text"), + }, + }, + { + name: "HTMLComment/blockquote/list/inline-multiple", + input: "> - text text", + want: []wantBlock{ + text("> - ", "text "), + comment("", ""), + comment("", ""), + text("> - ", " text"), }, }, { - name: "blockquote with ordered list fenced code block", + name: "HTMLComment/blockquote/list/block", input: joinLines( "> text", - "> 1. ```bash", - "> echo \"hello\"", - "> ```", + "> - ", + "> text", + ), + want: []wantBlock{ + text("> ", "text\n"), + comment("> - ", "\n"), + text("> ", "text"), + }, + }, + { + name: "HTMLComment/blockquote/nested/list/block", + input: joinLines( + "> text", + "> > - ", "> text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "> ", "text\n"}, - {internal.BlockKindFencedCode, "> 1. ", "```bash\necho \"hello\"\n```\n"}, - {internal.BlockKindText, "> ", "text"}, + want: []wantBlock{ + text("> ", "text\n"), + comment("> > - ", "\n"), + text("> ", "text"), }, }, { - name: "ordered list with blockquote html comment block", + name: "HTMLComment/list/ordered/blockquote/block", input: joinLines( "1. text", " > ", "2. text", ), - want: []struct { - kind internal.BlockKind - indent string - content string - }{ - {internal.BlockKindText, "1. ", "text\n"}, - {internal.BlockKindHTMLComment, " > ", "\n"}, - {internal.BlockKindText, "2. ", "text"}, + want: []wantBlock{ + text("1. ", "text\n"), + comment(" > ", "\n"), + text("2. ", "text"), }, }, } @@ -696,27 +673,9 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { len(blocks), ) for i, w := range tt.want { - assert.Equal( - t, - w.kind, - blocks[i].Kind(), - "block[%d] kind", - i, - ) - assert.Equal( - t, - w.indent, - string(blocks[i].Indent()), - "block[%d] indent", - i, - ) - assert.Equal( - t, - w.content, - string(blocks[i].Content()), - "block[%d] content", - i, - ) + assert.Equal(t, w.kind, blocks[i].Kind(), "block[%d] kind", i) + assert.Equal(t, w.indent, string(blocks[i].Indent()), "block[%d] indent", i) + assert.Equal(t, w.content, string(blocks[i].Content()), "block[%d] content", i) } }) } diff --git a/internal/cell_test.go b/internal/cell_test.go index 42a14c8..f298d9d 100644 --- a/internal/cell_test.go +++ b/internal/cell_test.go @@ -55,33 +55,37 @@ func TestParseInfoString(t *testing.T) { }{ { name: "text block", - block: internal.MakeBlockFromRaw( + block: internal.MakeBlock( internal.BlockKindText, []byte("hello"), + nil, ), want: internal.InfoString{}, }, { name: "fenced code without litdoc", - block: internal.MakeBlockFromRaw( + block: internal.MakeBlock( internal.BlockKindFencedCode, []byte("```bash\necho hello\n```\n"), + nil, ), want: internal.InfoString{Lang: "bash"}, }, { name: "fenced code with litdoc", - block: internal.MakeBlockFromRaw( + block: internal.MakeBlock( internal.BlockKindFencedCode, []byte("```bash | litdoc\necho hello\n```\n"), + nil, ), want: internal.InfoString{Lang: "bash", IsLitdoc: true}, }, { name: "html comment with litdoc", - block: internal.MakeBlockFromRaw( + block: internal.MakeBlock( internal.BlockKindHTMLComment, []byte("\n"), + nil, ), want: internal.InfoString{Lang: "bash", IsLitdoc: true}, }, @@ -219,10 +223,10 @@ func TestCompose(t *testing.T) { func TestClassify(t *testing.T) { textBlock := func(content string) internal.Block { - return internal.MakeBlockFromRaw(internal.BlockKindText, []byte(content)) + return internal.MakeBlock(internal.BlockKindText, []byte(content), nil) } bashLitdocBlock := func(content string) internal.Block { - return internal.MakeBlockFromRaw(internal.BlockKindFencedCode, []byte(content)) + return internal.MakeBlock(internal.BlockKindFencedCode, []byte(content), nil) } t.Run("single text block becomes StaticCell", func(t *testing.T) { @@ -289,7 +293,7 @@ func TestClassify(t *testing.T) { // given code := "```bash\necho hello\n```\n" blocks := []internal.Block{ - internal.MakeBlockFromRaw(internal.BlockKindFencedCode, []byte(code)), + internal.MakeBlock(internal.BlockKindFencedCode, []byte(code), nil), } // when From 21f057921d89b7fb3a0e6cee7084dc20801ddf83 Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Wed, 29 Apr 2026 11:17:13 -0600 Subject: [PATCH 08/18] Convert Block to use strings and use internal.Block in tests --- internal/block.go | 36 ++++++----- internal/block_test.go | 135 +++++++++++++++++++++-------------------- internal/cell.go | 37 ++++++----- internal/cell_test.go | 22 +++---- 4 files changed, 120 insertions(+), 110 deletions(-) diff --git a/internal/block.go b/internal/block.go index 20c44ee..a0d8364 100644 --- a/internal/block.go +++ b/internal/block.go @@ -5,6 +5,8 @@ import ( "context" "fmt" "regexp" + "strings" + "unicode/utf8" sitter "github.com/smacker/go-tree-sitter" "github.com/smacker/go-tree-sitter/markdown" @@ -20,19 +22,19 @@ const ( type Block struct { kind BlockKind - indent []byte - content []byte + indent string + content string } -func MakeBlock(kind BlockKind, content, indent []byte) Block { +func MakeBlock(kind BlockKind, content, indent string) Block { return Block{kind: kind, indent: indent, content: content} } func (b Block) Kind() BlockKind { return b.kind } -func (b Block) Indent() []byte { return b.indent } +func (b Block) Indent() string { return b.indent } -func (b Block) Content() []byte { return b.content } +func (b Block) Content() string { return b.content } func (b Block) String() string { return fmt.Sprintf("{%s %q}", b.kind, b.content) @@ -41,6 +43,10 @@ func (b Block) String() string { var htmlCommentRe = regexp.MustCompile(`(?s)`) func MakeBlocksFromMarkdown(content []byte) ([]Block, error) { + if !utf8.Valid(content) { + return nil, fmt.Errorf("markdown content is not valid UTF-8") + } + tree, err := markdown.ParseCtx(context.Background(), nil, content) if err != nil { return nil, fmt.Errorf("parsing markdown: %w", err) @@ -53,7 +59,7 @@ func MakeBlocksFromMarkdown(content []byte) ([]Block, error) { collectBlockNodes(root, content, nil, nil, &pos, &blocks) if pos < uint32(len(content)) { - blocks = append(blocks, MakeBlock(BlockKindText, content[pos:], nil)) + blocks = append(blocks, MakeBlock(BlockKindText, string(content[pos:]), "")) } return splitInlineHTMLComments(blocks), nil @@ -67,7 +73,7 @@ func splitInlineHTMLComments(blocks []Block) []Block { continue } content := b.Content() - locs := htmlCommentRe.FindAllIndex(content, -1) + locs := htmlCommentRe.FindAllStringIndex(content, -1) if len(locs) == 0 { result = append(result, b) continue @@ -77,7 +83,7 @@ func splitInlineHTMLComments(blocks []Block) []Block { if loc[0] > pos { result = append(result, MakeBlock(BlockKindText, content[pos:loc[0]], b.indent)) } - commentIndent := []byte(nil) + commentIndent := "" if isWholeTextBlock(content, loc) { commentIndent = b.indent } @@ -91,9 +97,9 @@ func splitInlineHTMLComments(blocks []Block) []Block { return result } -func isWholeTextBlock(content []byte, loc []int) bool { - return len(bytes.TrimSpace(content[:loc[0]])) == 0 && - len(bytes.TrimSpace(content[loc[1]:])) == 0 +func isWholeTextBlock(content string, loc []int) bool { + return len(strings.TrimSpace(content[:loc[0]])) == 0 && + len(strings.TrimSpace(content[loc[1]:])) == 0 } func collectBlockNodes( @@ -118,7 +124,7 @@ func collectBlockNodes( if child.StartByte() > *pos { gap := stripIndent(content[*pos:child.StartByte()], childStripPrefix) if len(gap) > 0 { - *blocks = append(*blocks, MakeBlock(BlockKindText, gap, childIndent)) + *blocks = append(*blocks, MakeBlock(BlockKindText, string(gap), string(childIndent))) } } *pos = child.EndByte() @@ -138,7 +144,7 @@ func collectBlockNodes( if child.StartByte() > *pos { gap := stripIndent(content[*pos:child.StartByte()], stripPrefix) if len(gap) > 0 { - *blocks = append(*blocks, MakeBlock(BlockKindText, gap, indent)) + *blocks = append(*blocks, MakeBlock(BlockKindText, string(gap), string(indent))) } } *pos = child.EndByte() @@ -175,13 +181,13 @@ func collectBlockNodes( if start > *pos { gap := stripIndent(content[*pos:start], stripPrefix) if len(gap) > 0 { - *blocks = append(*blocks, MakeBlock(BlockKindText, gap, indent)) + *blocks = append(*blocks, MakeBlock(BlockKindText, string(gap), string(indent))) } } raw := stripIndent(content[start:end], blockStripPrefix) if len(raw) > 0 { - *blocks = append(*blocks, MakeBlock(blockKind(node, content), raw, blockIndent)) + *blocks = append(*blocks, MakeBlock(blockKind(node, content), string(raw), string(blockIndent))) } *pos = end } diff --git a/internal/block_test.go b/internal/block_test.go index ff51fc4..8184d4a 100644 --- a/internal/block_test.go +++ b/internal/block_test.go @@ -10,22 +10,16 @@ import ( "github.com/stretchr/testify/require" ) -type wantBlock struct { - kind internal.BlockKind - indent string - content string +func text(indent, content string) internal.Block { + return internal.MakeBlock(internal.BlockKindText, content, indent) } -func text(indent, content string) wantBlock { - return wantBlock{internal.BlockKindText, indent, content} +func code(indent, content string) internal.Block { + return internal.MakeBlock(internal.BlockKindFencedCode, content, indent) } -func code(indent, content string) wantBlock { - return wantBlock{internal.BlockKindFencedCode, indent, content} -} - -func comment(indent, content string) wantBlock { - return wantBlock{internal.BlockKindHTMLComment, indent, content} +func comment(indent, content string) internal.Block { + return internal.MakeBlock(internal.BlockKindHTMLComment, content, indent) } func joinLines(lines ...string) string { @@ -35,8 +29,8 @@ func joinLines(lines ...string) string { func TestMakeBlock(t *testing.T) { // given kind := internal.BlockKindFencedCode - content := []byte("```bash\necho hello\n```\n") - indent := []byte("> ") + content := "```bash\necho hello\n```\n" + indent := "> " // when got := internal.MakeBlock(kind, content, indent) @@ -51,13 +45,13 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { tests := []struct { name string input string - want []wantBlock + want []internal.Block }{ // Text — baseline cases that should produce no FencedCode or HTMLComment blocks. { name: "Text/heading", input: "# Hello\n", - want: []wantBlock{ + want: []internal.Block{ text("", "# Hello\n"), }, }, @@ -73,7 +67,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "more text", "", ), - want: []wantBlock{ + want: []internal.Block{ text("", "# Level 1\n"), text("", "\n"), text("", "some text\n"), @@ -91,7 +85,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "", "last line", ), - want: []wantBlock{ + want: []internal.Block{ text("", "text\nand more text\n"), text("", "\n"), text("", "last line"), @@ -104,7 +98,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "and more text ", "after line break", ), - want: []wantBlock{ + want: []internal.Block{ text("", "text\nand more text \nafter line break"), }, }, @@ -114,21 +108,21 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " echo \"hello, \"", " echo \"world\"", ), - want: []wantBlock{ + want: []internal.Block{ text("", " echo \"hello, \"\n echo \"world\""), }, }, { name: "Text/inline-code-span", input: "text `echo \"hello, \" text`", - want: []wantBlock{ + want: []internal.Block{ text("", "text `echo \"hello, \" text`"), }, }, { name: "Text/html-block-not-comment", input: "
\nhello\n
\n", - want: []wantBlock{ + want: []internal.Block{ text("", "
\nhello\n
\n"), }, }, @@ -140,7 +134,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> ", "> again", ), - want: []wantBlock{ + want: []internal.Block{ text("> ", "hello, \nworld\n"), text("> ", "\n"), text("> ", "again"), @@ -156,7 +150,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "```", "", ), - want: []wantBlock{ + want: []internal.Block{ code("", "```bash\necho \"hello\"\n```\n"), }, }, @@ -167,7 +161,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "echo \"hello\"", "```", ), - want: []wantBlock{ + want: []internal.Block{ code("", "```bash\necho \"hello\"\n```"), }, }, @@ -178,7 +172,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "echo \"hello\"", "~~~", ), - want: []wantBlock{ + want: []internal.Block{ code("", "~~~bash\necho \"hello\"\n~~~"), }, }, @@ -191,7 +185,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "```", "````", ), - want: []wantBlock{ + want: []internal.Block{ code("", "````md\n```bash\necho \"hello\"\n```\n````"), }, }, @@ -202,7 +196,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "echo \"hello\"", "```", ), - want: []wantBlock{ + want: []internal.Block{ code("", "```\necho \"hello\"\n```"), }, }, @@ -212,7 +206,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "```bash", "```", ), - want: []wantBlock{ + want: []internal.Block{ code("", "```bash\n```"), }, }, @@ -227,7 +221,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "echo b", "```", ), - want: []wantBlock{ + want: []internal.Block{ code("", "```bash\necho a\n```\n"), text("", "\n"), code("", "```bash\necho b\n```"), @@ -240,7 +234,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " echo \"hello\"", " ```", ), - want: []wantBlock{ + want: []internal.Block{ code(" ", "```bash\necho \"hello\"\n```"), }, }, @@ -251,7 +245,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " echo \"hello\"", " ```", ), - want: []wantBlock{ + want: []internal.Block{ code(" ", "```bash\necho \"hello\"\n```"), }, }, @@ -261,7 +255,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "```bash", "echo \"hello\"", ), - want: []wantBlock{ + want: []internal.Block{ code("", "```bash\necho \"hello\""), }, }, @@ -276,7 +270,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> ```", "> text", ), - want: []wantBlock{ + want: []internal.Block{ text("> ", "text\n"), code("> ", "```bash\necho \"hello\"\n```\n"), text("> ", "text"), @@ -291,7 +285,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> > ```", "> text", ), - want: []wantBlock{ + want: []internal.Block{ text("> ", "text\n"), code("> > ", "```bash\necho \"hello\"\n```\n"), text("> ", "text"), @@ -306,7 +300,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " ```", "- text", ), - want: []wantBlock{ + want: []internal.Block{ text("- ", "text\n"), code("- ", "```bash\necho \"hello\"\n```\n"), text("- ", "text"), @@ -321,7 +315,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " ```", "- text", ), - want: []wantBlock{ + want: []internal.Block{ text("- ", "text\n"), code(" - ", "```bash\necho \"hello\"\n```\n"), text("- ", "text"), @@ -336,7 +330,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " ~~~", "+ text", ), - want: []wantBlock{ + want: []internal.Block{ text("+ ", "text\n"), code("+ ", "~~~bash\necho \"hello\"\n~~~\n"), text("+ ", "text"), @@ -351,7 +345,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " ```", "3. text", ), - want: []wantBlock{ + want: []internal.Block{ text("1. ", "text\n"), code("2. ", "```bash\necho \"hello\"\n```\n"), text("3. ", "text"), @@ -366,7 +360,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " ```", "2. text", ), - want: []wantBlock{ + want: []internal.Block{ text("1. ", "text\n"), code(" 1. ", "```bash\necho \"hello\"\n```\n"), text("2. ", "text"), @@ -381,7 +375,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " > ```", "- text", ), - want: []wantBlock{ + want: []internal.Block{ text("- ", "text\n"), code(" > ", "```bash\necho \"hello\"\n```\n"), text("- ", "text"), @@ -397,7 +391,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> ```", "> text", ), - want: []wantBlock{ + want: []internal.Block{ text("> ", "text\n"), text("> - ", "item\n"), code("> - ", "```bash\necho \"hello\"\n```\n"), @@ -413,7 +407,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> ```", "> text", ), - want: []wantBlock{ + want: []internal.Block{ text("> ", "text\n"), code("> 1. ", "```bash\necho \"hello\"\n```\n"), text("> ", "text"), @@ -424,7 +418,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { { name: "HTMLComment/top-level/inline", input: "text text", - want: []wantBlock{ + want: []internal.Block{ text("", "text "), comment("", ""), text("", " text"), @@ -433,7 +427,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { { name: "HTMLComment/top-level/inline-multiple", input: "text text", - want: []wantBlock{ + want: []internal.Block{ text("", "text "), comment("", ""), comment("", ""), @@ -443,7 +437,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { { name: "HTMLComment/top-level/inline-at-line-end", input: "text ", - want: []wantBlock{ + want: []internal.Block{ text("", "text "), comment("", ""), }, @@ -457,7 +451,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "-->", "text", ), - want: []wantBlock{ + want: []internal.Block{ text("", "text\n"), comment("", "\n"), text("", "text"), @@ -466,7 +460,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { { name: "HTMLComment/top-level/block-empty", input: "\n", - want: []wantBlock{ + want: []internal.Block{ comment("", "\n"), }, }, @@ -478,7 +472,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "", "", ), - want: []wantBlock{ + want: []internal.Block{ comment("", "\n"), text("", "\n"), comment("", "\n"), @@ -487,7 +481,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { { name: "HTMLComment/top-level/in-heading-line", input: "# Title \n", - want: []wantBlock{ + want: []internal.Block{ text("", "# Title "), comment("", ""), text("", "\n"), @@ -498,7 +492,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { { name: "HTMLComment/blockquote/inline-multiple", input: "> text text", - want: []wantBlock{ + want: []internal.Block{ text("> ", "text "), comment("", ""), comment("", ""), @@ -514,7 +508,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> > -->", "> text", ), - want: []wantBlock{ + want: []internal.Block{ text("> ", "text\n"), comment("> > ", "\n"), text("> ", "text"), @@ -523,7 +517,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { { name: "HTMLComment/list/dash/inline-multiple", input: "- text text", - want: []wantBlock{ + want: []internal.Block{ text("- ", "text "), comment("", ""), comment("", ""), @@ -533,7 +527,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { { name: "HTMLComment/list/dash/nested/inline-multiple", input: " - text text", - want: []wantBlock{ + want: []internal.Block{ text(" - ", "text "), comment("", ""), comment("", ""), @@ -549,7 +543,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " -->", "- text", ), - want: []wantBlock{ + want: []internal.Block{ text("- ", "text\n"), comment("- ", "\n"), text("- ", "text"), @@ -564,7 +558,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " -->", "- text", ), - want: []wantBlock{ + want: []internal.Block{ text("- ", "text\n"), comment(" - ", "\n"), text("- ", "text"), @@ -579,7 +573,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " -->", "* text", ), - want: []wantBlock{ + want: []internal.Block{ text("* ", "text\n"), comment("* ", "\n"), text("* ", "text"), @@ -594,7 +588,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " -->", "3. text", ), - want: []wantBlock{ + want: []internal.Block{ text("1. ", "text\n"), comment("2. ", "\n"), text("3. ", "text"), @@ -603,7 +597,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { { name: "HTMLComment/blockquote/list/inline-multiple", input: "> - text text", - want: []wantBlock{ + want: []internal.Block{ text("> - ", "text "), comment("", ""), comment("", ""), @@ -619,7 +613,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> -->", "> text", ), - want: []wantBlock{ + want: []internal.Block{ text("> ", "text\n"), comment("> - ", "\n"), text("> ", "text"), @@ -634,7 +628,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> > -->", "> text", ), - want: []wantBlock{ + want: []internal.Block{ text("> ", "text\n"), comment("> > - ", "\n"), text("> ", "text"), @@ -649,7 +643,7 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { " > -->", "2. text", ), - want: []wantBlock{ + want: []internal.Block{ text("1. ", "text\n"), comment(" > ", "\n"), text("2. ", "text"), @@ -673,10 +667,21 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { len(blocks), ) for i, w := range tt.want { - assert.Equal(t, w.kind, blocks[i].Kind(), "block[%d] kind", i) - assert.Equal(t, w.indent, string(blocks[i].Indent()), "block[%d] indent", i) - assert.Equal(t, w.content, string(blocks[i].Content()), "block[%d] content", i) + assert.Equal(t, w.Kind(), blocks[i].Kind(), "block[%d] kind", i) + assert.Equal(t, w.Indent(), blocks[i].Indent(), "block[%d] indent", i) + assert.Equal(t, w.Content(), blocks[i].Content(), "block[%d] content", i) } }) } + + t.Run("invalid-utf8", func(t *testing.T) { + // given + content := []byte{'#', ' ', 0xe9, '\n'} + + // when + _, err := internal.MakeBlocksFromMarkdown(content) + + // then + require.ErrorContains(t, err, "not valid UTF-8") + }) } diff --git a/internal/cell.go b/internal/cell.go index 55588cb..d564be9 100644 --- a/internal/cell.go +++ b/internal/cell.go @@ -1,7 +1,6 @@ package internal import ( - "bytes" "fmt" "strings" ) @@ -54,21 +53,21 @@ type InfoString struct { func ParseInfoString(b Block) InfoString { firstLine := b.content - if i := bytes.IndexByte(b.content, '\n'); i >= 0 { + if i := strings.IndexByte(b.content, '\n'); i >= 0 { firstLine = b.content[:i] } - var raw []byte + var raw string switch b.kind { case BlockKindFencedCode: - raw = bytes.TrimLeft(firstLine, "`~") + raw = strings.TrimLeft(firstLine, "`~") case BlockKindHTMLComment: - raw = bytes.TrimSpace(bytes.TrimPrefix(firstLine, []byte("\n"), - nil, + "\n", + "", ), want: internal.InfoString{Lang: "bash", IsLitdoc: true}, }, @@ -223,10 +223,10 @@ func TestCompose(t *testing.T) { func TestClassify(t *testing.T) { textBlock := func(content string) internal.Block { - return internal.MakeBlock(internal.BlockKindText, []byte(content), nil) + return internal.MakeBlock(internal.BlockKindText, content, "") } bashLitdocBlock := func(content string) internal.Block { - return internal.MakeBlock(internal.BlockKindFencedCode, []byte(content), nil) + return internal.MakeBlock(internal.BlockKindFencedCode, content, "") } t.Run("single text block becomes StaticCell", func(t *testing.T) { @@ -293,7 +293,7 @@ func TestClassify(t *testing.T) { // given code := "```bash\necho hello\n```\n" blocks := []internal.Block{ - internal.MakeBlock(internal.BlockKindFencedCode, []byte(code), nil), + internal.MakeBlock(internal.BlockKindFencedCode, code, ""), } // when From 1079f6f4120b259a96648fb5491ec5826524d21a Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Wed, 29 Apr 2026 11:19:21 -0600 Subject: [PATCH 09/18] Rename block types --- internal/block.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/block.go b/internal/block.go index a0d8364..04b79b4 100644 --- a/internal/block.go +++ b/internal/block.go @@ -15,9 +15,9 @@ import ( type BlockKind string const ( - BlockKindText BlockKind = "Text" - BlockKindFencedCode BlockKind = "fencedCode" - BlockKindHTMLComment BlockKind = "HTMLComment" + BlockKindText BlockKind = "text" + BlockKindFencedCode BlockKind = "fenced_code" + BlockKindHTMLComment BlockKind = "html_comment" ) type Block struct { From 580734e157d1e80cdfea36e074164409541f3efa Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Wed, 29 Apr 2026 11:30:34 -0600 Subject: [PATCH 10/18] Restructure block implementation --- internal/block.go | 252 +++++++++++++++++++++++++++++----------------- 1 file changed, 160 insertions(+), 92 deletions(-) diff --git a/internal/block.go b/internal/block.go index 04b79b4..e05e4e8 100644 --- a/internal/block.go +++ b/internal/block.go @@ -20,6 +20,18 @@ const ( BlockKindHTMLComment BlockKind = "html_comment" ) +const ( + markdownNodeDocument = "document" + markdownNodeSection = "section" + markdownNodeBlockQuote = "block_quote" + markdownNodeBlockQuoteMarker = "block_quote_marker" + markdownNodeList = "list" + markdownNodeListItem = "list_item" + markdownNodeFencedCodeBlock = "fenced_code_block" + markdownNodeHTMLBlock = "html_block" + markdownListMarkerPrefix = "list_marker_" +) + type Block struct { kind BlockKind indent string @@ -42,6 +54,9 @@ func (b Block) String() string { var htmlCommentRe = regexp.MustCompile(`(?s)`) +// MakeBlocksFromMarkdown parses UTF-8 Markdown into Blocks. +// Block content is stored without its surrounding quote/list prefix; Block.indent +// stores the prefix needed to render that Block.content back into the same container. func MakeBlocksFromMarkdown(content []byte) ([]Block, error) { if !utf8.Valid(content) { return nil, fmt.Errorf("markdown content is not valid UTF-8") @@ -53,16 +68,11 @@ func MakeBlocksFromMarkdown(content []byte) ([]Block, error) { } root := tree.BlockTree().RootNode() - var blocks []Block - pos := uint32(0) - - collectBlockNodes(root, content, nil, nil, &pos, &blocks) + collector := blockCollector{content: content} + collector.collect(root, nil, nil) + collector.emitTrailingText() - if pos < uint32(len(content)) { - blocks = append(blocks, MakeBlock(BlockKindText, string(content[pos:]), "")) - } - - return splitInlineHTMLComments(blocks), nil + return splitInlineHTMLComments(collector.blocks), nil } func splitInlineHTMLComments(blocks []Block) []Block { @@ -102,98 +112,153 @@ func isWholeTextBlock(content string, loc []int) bool { len(strings.TrimSpace(content[loc[1]:])) == 0 } -func collectBlockNodes( +type blockCollector struct { + content []byte + pos uint32 + blocks []Block +} + +// indent and stripPrefix describe the current markdown container. +// +// indent is the prefix used when rendering a Block back into its container. +// stripPrefix is the source prefix removed from every line before storing block +// content. They differ for list items: "- " is rendered on the first line, but +// continuation lines are prefixed by spaces. +func (c *blockCollector) collect(node *sitter.Node, indent, stripPrefix []byte) { + switch node.Type() { + case markdownNodeDocument, markdownNodeSection, markdownNodeList: + c.collectChildren(node, indent, stripPrefix) + case markdownNodeBlockQuote: + c.collectBlockQuote(node, indent, stripPrefix) + case markdownNodeListItem: + c.collectListItem(node, indent, stripPrefix) + default: + c.collectLeaf(node, indent, stripPrefix) + } +} + +func (c *blockCollector) collectChildren(node *sitter.Node, indent, stripPrefix []byte) { + for i := 0; i < int(node.ChildCount()); i++ { + c.collect(node.Child(i), indent, stripPrefix) + } +} + +func (c *blockCollector) collectBlockQuote( node *sitter.Node, - content []byte, - indent []byte, - stripPrefix []byte, - pos *uint32, - blocks *[]Block, + indent, stripPrefix []byte, ) { - switch node.Type() { - case "document", "section": - for i := 0; i < int(node.ChildCount()); i++ { - collectBlockNodes(node.Child(i), content, indent, stripPrefix, pos, blocks) - } - case "block_quote": - childIndent := append(append([]byte(nil), indent...), "> "...) - childStripPrefix := append(append([]byte(nil), stripPrefix...), "> "...) - for i := 0; i < int(node.ChildCount()); i++ { - child := node.Child(i) - if child.Type() == "block_quote_marker" { - if child.StartByte() > *pos { - gap := stripIndent(content[*pos:child.StartByte()], childStripPrefix) - if len(gap) > 0 { - *blocks = append(*blocks, MakeBlock(BlockKindText, string(gap), string(childIndent))) - } - } - *pos = child.EndByte() - continue - } - collectBlockNodes(child, content, childIndent, childStripPrefix, pos, blocks) - } - case "list": - for i := 0; i < int(node.ChildCount()); i++ { - collectBlockNodes(node.Child(i), content, indent, stripPrefix, pos, blocks) + childIndent := appendPrefix(indent, "> ") + childStripPrefix := appendPrefix(stripPrefix, "> ") + for i := 0; i < int(node.ChildCount()); i++ { + child := node.Child(i) + if child.Type() == markdownNodeBlockQuoteMarker { + c.emitTextGap(child.StartByte(), childStripPrefix, childIndent) + c.pos = child.EndByte() + continue } - case "list_item": - childIndent, childStripPrefix := listItemIndent(node, content) - for i := 0; i < int(node.ChildCount()); i++ { - child := node.Child(i) - if isListMarker(child) { - if child.StartByte() > *pos { - gap := stripIndent(content[*pos:child.StartByte()], stripPrefix) - if len(gap) > 0 { - *blocks = append(*blocks, MakeBlock(BlockKindText, string(gap), string(indent))) - } - } - *pos = child.EndByte() - continue - } - collectBlockNodes(child, content, childIndent, childStripPrefix, pos, blocks) + c.collect(child, childIndent, childStripPrefix) + } +} + +func (c *blockCollector) collectListItem(node *sitter.Node, indent, stripPrefix []byte) { + childIndent, childStripPrefix := listItemPrefixes(node, c.content) + for i := 0; i < int(node.ChildCount()); i++ { + child := node.Child(i) + if isListMarker(child) { + c.emitTextGap(child.StartByte(), stripPrefix, indent) + c.pos = child.EndByte() + continue } - default: - start := node.StartByte() - end := node.EndByte() - blockIndent := indent - blockStripPrefix := stripPrefix - linePrefix := linePrefixBefore(content, start) - if len(linePrefix) > 0 && !bytes.Equal(linePrefix, indent) { - blockIndent = linePrefix - blockStripPrefix = linePrefix - if isSpaceIndent(linePrefix) { - rawLeading := leadingLineSpaces(content[start:end]) - if len(rawLeading) > 0 { - blockIndent = append(append([]byte(nil), linePrefix...), rawLeading...) - blockStripPrefix = blockIndent - } - } - } else if node.Type() == "fenced_code_block" && len(indent) == 0 { - // CommonMark §4.5: a fence may be indented up to 3 spaces; that indent is - // part of the block's prefix, not the code content. + c.collect(child, childIndent, childStripPrefix) + } +} + +func (c *blockCollector) collectLeaf(node *sitter.Node, indent, stripPrefix []byte) { + start := node.StartByte() + end := node.EndByte() + blockIndent, blockStripPrefix := blockPrefixes(node, c.content, indent, stripPrefix) + + c.emitTextGap(start, stripPrefix, indent) + + raw := stripIndent(c.content[start:end], blockStripPrefix) + c.emitBlock(blockKind(node, c.content), raw, blockIndent) + c.pos = end +} + +func (c *blockCollector) emitTrailingText() { + if c.pos < uint32(len(c.content)) { + c.emitBlock(BlockKindText, c.content[c.pos:], nil) + } +} + +func (c *blockCollector) emitTextGap(end uint32, stripPrefix, indent []byte) { + if end <= c.pos { + return + } + gap := stripIndent(c.content[c.pos:end], stripPrefix) + c.emitBlock(BlockKindText, gap, indent) +} + +func (c *blockCollector) emitBlock(kind BlockKind, raw, indent []byte) { + if len(raw) == 0 { + return + } + c.blocks = append(c.blocks, MakeBlock(kind, string(raw), string(indent))) +} + +func blockPrefixes( + node *sitter.Node, + content []byte, + indent []byte, + stripPrefix []byte, +) ([]byte, []byte) { + start := node.StartByte() + end := node.EndByte() + blockIndent := indent + blockStripPrefix := stripPrefix + + // Tree-sitter nodes start after some markdown prefixes but include others. + // When a node begins after a prefix that differs from the current container, + // strip that prefix from stored content and keep it as the render indent. + linePrefix := linePrefixBefore(content, start) + if len(linePrefix) > 0 && !bytes.Equal(linePrefix, indent) { + blockIndent = linePrefix + blockStripPrefix = linePrefix + if isSpaceIndent(linePrefix) { rawLeading := leadingLineSpaces(content[start:end]) - if n := len(rawLeading); n > 0 && n <= 3 { - blockIndent = rawLeading - blockStripPrefix = rawLeading + if len(rawLeading) > 0 { + blockIndent = append(append([]byte(nil), linePrefix...), rawLeading...) + blockStripPrefix = blockIndent } } + return blockIndent, blockStripPrefix + } - if start > *pos { - gap := stripIndent(content[*pos:start], stripPrefix) - if len(gap) > 0 { - *blocks = append(*blocks, MakeBlock(BlockKindText, string(gap), string(indent))) - } - } + if node.Type() != markdownNodeFencedCodeBlock || len(indent) > 0 { + return blockIndent, blockStripPrefix + } - raw := stripIndent(content[start:end], blockStripPrefix) - if len(raw) > 0 { - *blocks = append(*blocks, MakeBlock(blockKind(node, content), string(raw), string(blockIndent))) - } - *pos = end + // CommonMark allows top-level fenced code blocks to be indented up to three + // spaces. Treat that as the block prefix rather than code content. + rawLeading := leadingLineSpaces(content[start:end]) + if n := len(rawLeading); n > 0 && n <= 3 { + blockIndent = rawLeading + blockStripPrefix = rawLeading } + return blockIndent, blockStripPrefix +} + +func appendPrefix(prefix []byte, suffix string) []byte { + result := make([]byte, 0, len(prefix)+len(suffix)) + result = append(result, prefix...) + result = append(result, suffix...) + return result } -func listItemIndent(node *sitter.Node, content []byte) ([]byte, []byte) { +// listItemPrefixes returns both views of a list marker: the marker itself is +// used when rendering the first line, while equivalent spaces are stripped from +// continuation lines in the source. +func listItemPrefixes(node *sitter.Node, content []byte) ([]byte, []byte) { for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) if !isListMarker(child) { @@ -209,7 +274,7 @@ func listItemIndent(node *sitter.Node, content []byte) ([]byte, []byte) { } func isListMarker(node *sitter.Node) bool { - return bytes.HasPrefix([]byte(node.Type()), []byte("list_marker_")) + return strings.HasPrefix(node.Type(), markdownListMarkerPrefix) } func linePrefixBefore(content []byte, pos uint32) []byte { @@ -226,6 +291,9 @@ func leadingLineSpaces(content []byte) []byte { return line[:len(line)-len(bytes.TrimLeft(line, " "))] } +// stripIndent removes a container prefix from each line before storing content +// in a Block. Blockquote-only blank lines are normalized to empty lines so the +// rendered output does not keep stray ">" markers as content. func stripIndent(content, indent []byte) []byte { if len(indent) == 0 { return content @@ -281,9 +349,9 @@ func quoteParentIndent(indent []byte) []byte { func blockKind(node *sitter.Node, content []byte) BlockKind { switch node.Type() { - case "fenced_code_block": + case markdownNodeFencedCodeBlock: return BlockKindFencedCode - case "html_block": + case markdownNodeHTMLBlock: if bytes.HasPrefix(content[node.StartByte():node.EndByte()], []byte("`) +var ( + htmlCommentRe = regexp.MustCompile(`(?s)`) + openingFenceRe = regexp.MustCompile(`^[ ]{0,3}(` + "`" + `{3,}|~{3,})`) +) // MakeBlocksFromMarkdown parses UTF-8 Markdown into Blocks. // Block content is stored without its surrounding quote/list prefix; Block.indent @@ -70,6 +73,9 @@ func MakeBlocksFromMarkdown(content []byte) ([]Block, error) { root := tree.BlockTree().RootNode() collector := blockCollector{content: content} collector.collect(root, nil, nil) + if collector.err != nil { + return nil, collector.err + } collector.emitTrailingText() return splitInlineHTMLComments(collector.blocks), nil @@ -116,6 +122,7 @@ type blockCollector struct { content []byte pos uint32 blocks []Block + err error } // indent and stripPrefix describe the current markdown container. @@ -125,6 +132,10 @@ type blockCollector struct { // content. They differ for list items: "- " is rendered on the first line, but // continuation lines are prefixed by spaces. func (c *blockCollector) collect(node *sitter.Node, indent, stripPrefix []byte) { + if c.err != nil { + return + } + switch node.Type() { case markdownNodeDocument, markdownNodeSection, markdownNodeList: c.collectChildren(node, indent, stripPrefix) @@ -181,7 +192,17 @@ func (c *blockCollector) collectLeaf(node *sitter.Node, indent, stripPrefix []by c.emitTextGap(start, stripPrefix, indent) raw := stripIndent(c.content[start:end], blockStripPrefix) - c.emitBlock(blockKind(node, c.content), raw, blockIndent) + kind := blockKind(node, c.content) + if kind == BlockKindFencedCode && !isClosedFencedCodeBlock(raw) { + c.err = fmt.Errorf("unclosed fenced code block at byte %d", start) + return + } + if kind == BlockKindHTMLComment && !isClosedHTMLComment(raw) { + c.err = fmt.Errorf("unclosed HTML comment at byte %d", start) + return + } + + c.emitBlock(kind, raw, blockIndent) c.pos = end } @@ -248,6 +269,82 @@ func blockPrefixes( return blockIndent, blockStripPrefix } +func isClosedFencedCodeBlock(content []byte) bool { + openLineEnd := bytes.IndexByte(content, '\n') + if openLineEnd < 0 { + return false + } + + fenceChar, fenceLen, ok := openingFence(content[:openLineEnd]) + if !ok { + return false + } + + rest := content[openLineEnd+1:] + if len(rest) == 0 { + return false + } + lastLine := lastContentLine(rest) + if len(lastLine) == 0 { + return false + } + return isClosingFence(lastLine, fenceChar, fenceLen) +} + +func openingFence(line []byte) (byte, int, bool) { + line = bytes.TrimSuffix(line, []byte("\r")) + match := openingFenceRe.FindSubmatch(line) + if match == nil { + return 0, 0, false + } + fence := match[1] + fenceChar := fence[0] + return fenceChar, len(fence), true +} + +func lastContentLine(content []byte) []byte { + content = bytes.TrimSuffix(content, []byte("\n")) + if i := bytes.LastIndexByte(content, '\n'); i >= 0 { + return content[i+1:] + } + return content +} + +func isClosingFence(line []byte, fenceChar byte, fenceLen int) bool { + fence, ok := trimFenceLineIndent(line) + if !ok || len(fence) < fenceLen { + return false + } + + closingLen := countLeadingByte(fence, fenceChar) + if closingLen < fenceLen { + return false + } + + return len(bytes.Trim(fence[closingLen:], " \t")) == 0 +} + +func trimFenceLineIndent(line []byte) ([]byte, bool) { + line = bytes.TrimSuffix(line, []byte("\r")) + n := countLeadingByte(line, ' ') + if n > 3 { + return nil, false + } + return line[n:], true +} + +func countLeadingByte(content []byte, b byte) int { + n := 0 + for n < len(content) && content[n] == b { + n++ + } + return n +} + +func isClosedHTMLComment(content []byte) bool { + return bytes.Contains(content, []byte("-->")) +} + func appendPrefix(prefix []byte, suffix string) []byte { result := make([]byte, 0, len(prefix)+len(suffix)) result = append(result, prefix...) diff --git a/internal/block_test.go b/internal/block_test.go index 8184d4a..126cbf1 100644 --- a/internal/block_test.go +++ b/internal/block_test.go @@ -249,17 +249,6 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { code(" ", "```bash\necho \"hello\"\n```"), }, }, - { - name: "FencedCode/top-level/unclosed-runs-to-eof", - input: joinLines( - "```bash", - "echo \"hello\"", - ), - want: []internal.Block{ - code("", "```bash\necho \"hello\""), - }, - }, - // FencedCode — inside containers. { name: "FencedCode/blockquote", @@ -674,14 +663,102 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { }) } - t.Run("invalid-utf8", func(t *testing.T) { - // given - content := []byte{'#', ' ', 0xe9, '\n'} + invalidTests := []struct { + name string + input []byte + wantErrText string + }{ + { + name: "invalid-utf8", + input: []byte{'#', ' ', 0xe9, '\n'}, + wantErrText: "not valid UTF-8", + }, + { + name: "FencedCode/top-level/unclosed-runs-to-eof", + input: []byte(joinLines( + "```bash", + "echo \"hello\"", + )), + wantErrText: "unclosed fenced code block", + }, + { + name: "FencedCode/top-level/opening-line-runs-to-eof", + input: []byte("```bash\n"), + wantErrText: "unclosed fenced code block", + }, + { + name: "FencedCode/top-level/wrong-closing-fence-char", + input: []byte(joinLines( + "```bash", + "echo \"hello\"", + "~~~", + )), + wantErrText: "unclosed fenced code block", + }, + { + name: "FencedCode/top-level/short-closing-fence", + input: []byte(joinLines( + "````bash", + "echo \"hello\"", + "```", + )), + wantErrText: "unclosed fenced code block", + }, + { + name: "FencedCode/top-level/closing-fence-with-trailing-text", + input: []byte(joinLines( + "```bash", + "echo \"hello\"", + "``` nope", + )), + wantErrText: "unclosed fenced code block", + }, + { + name: "FencedCode/blockquote/unclosed-runs-to-eof", + input: []byte(joinLines( + "> ```bash", + "> echo \"hello\"", + )), + wantErrText: "unclosed fenced code block", + }, + { + name: "HTMLComment/top-level/unclosed-block-runs-to-eof", + input: []byte(joinLines( + " text", want: []internal.Block{ - text("", "text "), - comment("", ""), - text("", " text"), + text("", "text ", false), + cmnt("", "", true), + text("", " text", true), }, }, { name: "HTMLComment/top-level/inline-multiple", input: "text text", want: []internal.Block{ - text("", "text "), - comment("", ""), - comment("", ""), - text("", " text"), + text("", "text ", false), + cmnt("", "", true), + cmnt("", "", true), + text("", " text", true), }, }, { name: "HTMLComment/top-level/inline-at-line-end", input: "text ", want: []internal.Block{ - text("", "text "), - comment("", ""), + text("", "text ", false), + cmnt("", "", true), }, }, { @@ -441,16 +443,16 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "text", ), want: []internal.Block{ - text("", "text\n"), - comment("", "\n"), - text("", "text"), + text("", "text\n", false), + cmnt("", "\n", false), + text("", "text", false), }, }, { name: "HTMLComment/top-level/block-empty", input: "\n", want: []internal.Block{ - comment("", "\n"), + cmnt("", "\n", false), }, }, { @@ -462,18 +464,18 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "", ), want: []internal.Block{ - comment("", "\n"), - text("", "\n"), - comment("", "\n"), + cmnt("", "\n", false), + text("", "\n", false), + cmnt("", "\n", false), }, }, { name: "HTMLComment/top-level/in-heading-line", input: "# Title \n", want: []internal.Block{ - text("", "# Title "), - comment("", ""), - text("", "\n"), + text("", "# Title ", false), + cmnt("", "", true), + text("", "\n", true), }, }, @@ -482,10 +484,10 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { name: "HTMLComment/blockquote/inline-multiple", input: "> text text", want: []internal.Block{ - text("> ", "text "), - comment("", ""), - comment("", ""), - text("> ", " text"), + text("> ", "text ", false), + cmnt("> ", "", true), + cmnt("> ", "", true), + text("> ", " text", true), }, }, { @@ -498,29 +500,29 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> text", ), want: []internal.Block{ - text("> ", "text\n"), - comment("> > ", "\n"), - text("> ", "text"), + text("> ", "text\n", false), + cmnt("> > ", "\n", false), + text("> ", "text", false), }, }, { name: "HTMLComment/list/dash/inline-multiple", input: "- text text", want: []internal.Block{ - text("- ", "text "), - comment("", ""), - comment("", ""), - text("- ", " text"), + text("- ", "text ", false), + cmnt("- ", "", true), + cmnt("- ", "", true), + text("- ", " text", true), }, }, { name: "HTMLComment/list/dash/nested/inline-multiple", input: " - text text", want: []internal.Block{ - text(" - ", "text "), - comment("", ""), - comment("", ""), - text(" - ", " text"), + text(" - ", "text ", false), + cmnt(" - ", "", true), + cmnt(" - ", "", true), + text(" - ", " text", true), }, }, { @@ -533,9 +535,9 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "- text", ), want: []internal.Block{ - text("- ", "text\n"), - comment("- ", "\n"), - text("- ", "text"), + text("- ", "text\n", false), + cmnt("- ", "\n", false), + text("- ", "text", false), }, }, { @@ -548,9 +550,9 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "- text", ), want: []internal.Block{ - text("- ", "text\n"), - comment(" - ", "\n"), - text("- ", "text"), + text("- ", "text\n", false), + cmnt(" - ", "\n", false), + text("- ", "text", false), }, }, { @@ -563,9 +565,9 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "* text", ), want: []internal.Block{ - text("* ", "text\n"), - comment("* ", "\n"), - text("* ", "text"), + text("* ", "text\n", false), + cmnt("* ", "\n", false), + text("* ", "text", false), }, }, { @@ -578,19 +580,19 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "3. text", ), want: []internal.Block{ - text("1. ", "text\n"), - comment("2. ", "\n"), - text("3. ", "text"), + text("1. ", "text\n", false), + cmnt("2. ", "\n", false), + text("3. ", "text", false), }, }, { name: "HTMLComment/blockquote/list/inline-multiple", input: "> - text text", want: []internal.Block{ - text("> - ", "text "), - comment("", ""), - comment("", ""), - text("> - ", " text"), + text("> - ", "text ", false), + cmnt("> - ", "", true), + cmnt("> - ", "", true), + text("> - ", " text", true), }, }, { @@ -603,9 +605,9 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> text", ), want: []internal.Block{ - text("> ", "text\n"), - comment("> - ", "\n"), - text("> ", "text"), + text("> ", "text\n", false), + cmnt("> - ", "\n", false), + text("> ", "text", false), }, }, { @@ -618,9 +620,9 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> text", ), want: []internal.Block{ - text("> ", "text\n"), - comment("> > - ", "\n"), - text("> ", "text"), + text("> ", "text\n", false), + cmnt("> > - ", "\n", false), + text("> ", "text", false), }, }, { @@ -633,9 +635,9 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "2. text", ), want: []internal.Block{ - text("1. ", "text\n"), - comment(" > ", "\n"), - text("2. ", "text"), + text("1. ", "text\n", false), + cmnt(" > ", "\n", false), + text("2. ", "text", false), }, }, } @@ -659,6 +661,13 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { assert.Equal(t, w.Kind(), blocks[i].Kind(), "block[%d] kind", i) assert.Equal(t, w.Indent(), blocks[i].Indent(), "block[%d] indent", i) assert.Equal(t, w.Content(), blocks[i].Content(), "block[%d] content", i) + assert.Equal( + t, + w.Continuation(), + blocks[i].Continuation(), + "block[%d] continuation", + i, + ) } }) } diff --git a/internal/cell.go b/internal/cell.go index d564be9..0d6a029 100644 --- a/internal/cell.go +++ b/internal/cell.go @@ -106,7 +106,7 @@ func renderStaticBlock(b Block) string { rendered.WriteString(continuationIndent) } } else { - if len(line) > 0 { + if len(line) > 0 && !b.continuation { rendered.WriteString(b.indent) } } @@ -119,8 +119,9 @@ func renderStaticBlock(b Block) string { } func blockContinuationIndent(indent string) string { - if strings.HasSuffix(indent, "> ") { - return indent + if idx := strings.LastIndex(indent, "> "); idx >= 0 { + prefixLen := idx + len("> ") + return indent[:prefixLen] + strings.Repeat(" ", len(indent)-prefixLen) } return strings.Repeat(" ", len(indent)) } diff --git a/internal/cell_test.go b/internal/cell_test.go index fd60af4..9c82ff2 100644 --- a/internal/cell_test.go +++ b/internal/cell_test.go @@ -59,6 +59,7 @@ func TestParseInfoString(t *testing.T) { internal.BlockKindText, "hello", "", + false, ), want: internal.InfoString{}, }, @@ -68,6 +69,7 @@ func TestParseInfoString(t *testing.T) { internal.BlockKindFencedCode, "```bash\necho hello\n```\n", "", + false, ), want: internal.InfoString{Lang: "bash"}, }, @@ -77,6 +79,7 @@ func TestParseInfoString(t *testing.T) { internal.BlockKindFencedCode, "```bash | litdoc\necho hello\n```\n", "", + false, ), want: internal.InfoString{Lang: "bash", IsLitdoc: true}, }, @@ -86,6 +89,7 @@ func TestParseInfoString(t *testing.T) { internal.BlockKindHTMLComment, "\n", "", + false, ), want: internal.InfoString{Lang: "bash", IsLitdoc: true}, }, @@ -221,12 +225,42 @@ func TestCompose(t *testing.T) { }) } +func TestComposePreservesInlineHTMLCommentContinuations(t *testing.T) { + tests := []struct { + name string + input string + }{ + { + name: "list", + input: "- text text", + }, + { + name: "blockquote list", + input: "> - text text", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + blocks, err := internal.MakeBlocksFromMarkdown([]byte(tt.input)) + require.NoError(t, err) + cells, err := internal.Classify(blocks) + require.NoError(t, err) + + got, err := internal.Compose(cells) + + require.NoError(t, err) + assert.Equal(t, tt.input, got) + }) + } +} + func TestClassify(t *testing.T) { textBlock := func(content string) internal.Block { - return internal.MakeBlock(internal.BlockKindText, content, "") + return internal.MakeBlock(internal.BlockKindText, content, "", false) } bashLitdocBlock := func(content string) internal.Block { - return internal.MakeBlock(internal.BlockKindFencedCode, content, "") + return internal.MakeBlock(internal.BlockKindFencedCode, content, "", false) } t.Run("single text block becomes StaticCell", func(t *testing.T) { @@ -293,7 +327,7 @@ func TestClassify(t *testing.T) { // given code := "```bash\necho hello\n```\n" blocks := []internal.Block{ - internal.MakeBlock(internal.BlockKindFencedCode, code, ""), + internal.MakeBlock(internal.BlockKindFencedCode, code, "", false), } // when From d52f342dcb269b7180da0f756504788543a83bdf Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Wed, 29 Apr 2026 12:44:06 -0600 Subject: [PATCH 13/18] Simplify function naming --- internal/cell.go | 6 ++-- internal/cell_test.go | 77 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/internal/cell.go b/internal/cell.go index 0d6a029..31b9430 100644 --- a/internal/cell.go +++ b/internal/cell.go @@ -95,7 +95,7 @@ func renderStaticBlock(b Block) string { lines := strings.Split(b.content, "\n") var rendered strings.Builder - continuationIndent := blockContinuationIndent(b.indent) + renderedIndent := renderIndent(b.indent) for i, line := range lines { if i == len(lines)-1 && len(line) == 0 { break @@ -103,7 +103,7 @@ func renderStaticBlock(b Block) string { if i > 0 { rendered.WriteByte('\n') if len(line) > 0 { - rendered.WriteString(continuationIndent) + rendered.WriteString(renderedIndent) } } else { if len(line) > 0 && !b.continuation { @@ -118,7 +118,7 @@ func renderStaticBlock(b Block) string { return rendered.String() } -func blockContinuationIndent(indent string) string { +func renderIndent(indent string) string { if idx := strings.LastIndex(indent, "> "); idx >= 0 { prefixLen := idx + len("> ") return indent[:prefixLen] + strings.Repeat(" ", len(indent)-prefixLen) diff --git a/internal/cell_test.go b/internal/cell_test.go index 9c82ff2..3329cad 100644 --- a/internal/cell_test.go +++ b/internal/cell_test.go @@ -67,7 +67,12 @@ func TestParseInfoString(t *testing.T) { name: "fenced code without litdoc", block: internal.MakeBlock( internal.BlockKindFencedCode, - "```bash\necho hello\n```\n", + joinLines( + "```bash", + "echo hello", + "```", + "", + ), "", false, ), @@ -77,7 +82,12 @@ func TestParseInfoString(t *testing.T) { name: "fenced code with litdoc", block: internal.MakeBlock( internal.BlockKindFencedCode, - "```bash | litdoc\necho hello\n```\n", + joinLines( + "```bash | litdoc", + "echo hello", + "```", + "", + ), "", false, ), @@ -87,7 +97,12 @@ func TestParseInfoString(t *testing.T) { name: "html comment with litdoc", block: internal.MakeBlock( internal.BlockKindHTMLComment, - "\n", + joinLines( + "", + "", + ), "", false, ), @@ -105,7 +120,12 @@ func TestParseInfoString(t *testing.T) { func TestMakeBashCellFromRaw(t *testing.T) { // given - code := "```bash\necho hello\n```\n" + code := joinLines( + "```bash", + "echo hello", + "```", + "", + ) // when gotCell := internal.MakeBashCellFromRaw(code, "") @@ -118,7 +138,12 @@ func TestMakeBashCellFromRaw(t *testing.T) { func TestBashCellExecute(t *testing.T) { // given - fencedCode := "```bash\necho hello\n```\n" + fencedCode := joinLines( + "```bash", + "echo hello", + "```", + "", + ) cell := internal.MakeBashCellFromRaw(fencedCode, "") // when @@ -134,7 +159,12 @@ func TestBashCellExecute(t *testing.T) { func TestBashCellRender(t *testing.T) { t.Run("without output", func(t *testing.T) { // given - code := "```bash\necho hello\n```\n" + code := joinLines( + "```bash", + "echo hello", + "```", + "", + ) cell := internal.MakeBashCellFromRaw(code, "") // when @@ -147,7 +177,12 @@ func TestBashCellRender(t *testing.T) { t.Run("with output", func(t *testing.T) { // given - fencedCode := "```bash\necho hello\n```\n" + fencedCode := joinLines( + "```bash", + "echo hello", + "```", + "", + ) cell := internal.MakeBashCellFromRaw(fencedCode, "") executed, err := cell.Execute() require.NoError(t, err) @@ -231,12 +266,18 @@ func TestComposePreservesInlineHTMLCommentContinuations(t *testing.T) { input string }{ { - name: "list", - input: "- text text", + name: "list", + input: joinLines( + "- text text", + ), }, { - name: "blockquote list", - input: "> - text text", + name: "blockquote list", + input: joinLines( + "> - text text", + ), }, } @@ -282,7 +323,12 @@ func TestClassify(t *testing.T) { t.Run("litdoc bash block becomes BashCell", func(t *testing.T) { // given - code := "```bash | litdoc\necho hello\n```\n" + code := joinLines( + "```bash | litdoc", + "echo hello", + "```", + "", + ) blocks := []internal.Block{bashLitdocBlock(code)} // when @@ -300,7 +346,12 @@ func TestClassify(t *testing.T) { t.Run("mixed block types are each classified independently", func(t *testing.T) { // given - code := "```bash | litdoc\necho hello\n```\n" + code := joinLines( + "```bash | litdoc", + "echo hello", + "```", + "", + ) blocks := []internal.Block{ textBlock("before"), bashLitdocBlock(code), From 56abf376221e012c45815c5c73777be56948a661 Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Wed, 29 Apr 2026 12:48:44 -0600 Subject: [PATCH 14/18] Move continuation tests --- internal/cell_test.go | 72 +++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/internal/cell_test.go b/internal/cell_test.go index 3329cad..329f872 100644 --- a/internal/cell_test.go +++ b/internal/cell_test.go @@ -260,42 +260,6 @@ func TestCompose(t *testing.T) { }) } -func TestComposePreservesInlineHTMLCommentContinuations(t *testing.T) { - tests := []struct { - name string - input string - }{ - { - name: "list", - input: joinLines( - "- text text", - ), - }, - { - name: "blockquote list", - input: joinLines( - "> - text text", - ), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - blocks, err := internal.MakeBlocksFromMarkdown([]byte(tt.input)) - require.NoError(t, err) - cells, err := internal.Classify(blocks) - require.NoError(t, err) - - got, err := internal.Compose(cells) - - require.NoError(t, err) - assert.Equal(t, tt.input, got) - }) - } -} - func TestClassify(t *testing.T) { textBlock := func(content string) internal.Block { return internal.MakeBlock(internal.BlockKindText, content, "", false) @@ -374,6 +338,42 @@ func TestClassify(t *testing.T) { assert.Equal(t, "after", rendered2) }) + t.Run("preserves inline HTML comment continuations when composing static cells", func(t *testing.T) { + tests := []struct { + name string + input string + }{ + { + name: "list", + input: joinLines( + "- text text", + ), + }, + { + name: "blockquote list", + input: joinLines( + "> - text text", + ), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + blocks, err := internal.MakeBlocksFromMarkdown([]byte(tt.input)) + require.NoError(t, err) + cells, err := internal.Classify(blocks) + require.NoError(t, err) + + got, err := internal.Compose(cells) + + require.NoError(t, err) + assert.Equal(t, tt.input, got) + }) + } + }) + t.Run("non-litdoc fenced code block becomes StaticCell", func(t *testing.T) { // given code := "```bash\necho hello\n```\n" From bb6fa6fddd0ac2bc009e43a0905d4b24be1edd3e Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Wed, 29 Apr 2026 13:26:20 -0600 Subject: [PATCH 15/18] Clean up cell_test.go tests --- internal/block.go | 15 +- internal/block_test.go | 21 +- internal/cell.go | 18 +- internal/cell_test.go | 630 +++++++++++++++++++++++++-------------- internal/helpers_test.go | 23 ++ 5 files changed, 444 insertions(+), 263 deletions(-) create mode 100644 internal/helpers_test.go diff --git a/internal/block.go b/internal/block.go index 9327c21..a059da4 100644 --- a/internal/block.go +++ b/internal/block.go @@ -41,7 +41,12 @@ type Block struct { continuation bool } -func MakeBlock(kind BlockKind, content, indent string, continuation bool) Block { +func MakeBlockFromRaw( + kind BlockKind, + content string, + indent string, + continuation bool, +) Block { return Block{ kind: kind, indent: indent, @@ -107,7 +112,7 @@ func splitInlineHTMLComments(blocks []Block) []Block { pos := 0 for _, loc := range locs { if loc[0] > pos { - result = append(result, MakeBlock( + result = append(result, MakeBlockFromRaw( BlockKindText, content[pos:loc[0]], b.indent, @@ -119,7 +124,7 @@ func splitInlineHTMLComments(blocks []Block) []Block { if !wholeBlock { commentContinuation = b.continuation || loc[0] > 0 } - result = append(result, MakeBlock( + result = append(result, MakeBlockFromRaw( BlockKindHTMLComment, content[loc[0]:loc[1]], b.indent, @@ -128,7 +133,7 @@ func splitInlineHTMLComments(blocks []Block) []Block { pos = loc[1] } if pos < len(content) { - result = append(result, MakeBlock(BlockKindText, content[pos:], b.indent, true)) + result = append(result, MakeBlockFromRaw(BlockKindText, content[pos:], b.indent, true)) } } return result @@ -245,7 +250,7 @@ func (c *blockCollector) emitBlock(kind BlockKind, raw, indent []byte) { if len(raw) == 0 { return } - c.blocks = append(c.blocks, MakeBlock(kind, string(raw), string(indent), false)) + c.blocks = append(c.blocks, MakeBlockFromRaw(kind, string(raw), string(indent), false)) } func blockPrefixes( diff --git a/internal/block_test.go b/internal/block_test.go index 83dcf9f..dc87591 100644 --- a/internal/block_test.go +++ b/internal/block_test.go @@ -1,7 +1,6 @@ package internal_test import ( - "strings" "testing" "litdoc/internal" @@ -10,23 +9,7 @@ import ( "github.com/stretchr/testify/require" ) -func text(indent, content string, continuation bool) internal.Block { - return internal.MakeBlock(internal.BlockKindText, content, indent, continuation) -} - -func code(indent, content string, continuation bool) internal.Block { - return internal.MakeBlock(internal.BlockKindFencedCode, content, indent, continuation) -} - -func cmnt(indent, content string, continuation bool) internal.Block { - return internal.MakeBlock(internal.BlockKindHTMLComment, content, indent, continuation) -} - -func joinLines(lines ...string) string { - return strings.Join(lines, "\n") -} - -func TestMakeBlock(t *testing.T) { +func TestMakeBlockFromRaw(t *testing.T) { // given kind := internal.BlockKindFencedCode content := "```bash\necho hello\n```\n" @@ -34,7 +17,7 @@ func TestMakeBlock(t *testing.T) { continuation := true // when - got := internal.MakeBlock(kind, content, indent, continuation) + got := internal.MakeBlockFromRaw(kind, content, indent, continuation) // then assert.Equal(t, kind, got.Kind()) diff --git a/internal/cell.go b/internal/cell.go index 31b9430..9d0baf5 100644 --- a/internal/cell.go +++ b/internal/cell.go @@ -14,8 +14,8 @@ type StaticCell struct { content string } -func MakeStaticCellFromRaw(raw string) StaticCell { - return StaticCell{content: raw} +func MakeStaticCellFromRaw(content string) StaticCell { + return StaticCell{content: content} } func (t StaticCell) Execute() (Cell, error) { @@ -47,8 +47,8 @@ func (c BashCell) Render() (string, error) { } type InfoString struct { - Lang string - IsLitdoc bool + Lang string + Litdoc bool } func ParseInfoString(b Block) InfoString { @@ -67,8 +67,8 @@ func ParseInfoString(b Block) InfoString { } parts := strings.SplitN(raw, " | ", 2) lang := strings.TrimSpace(parts[0]) - isLitdoc := len(parts) > 1 && strings.HasPrefix(strings.TrimSpace(parts[1]), "litdoc") - return InfoString{Lang: lang, IsLitdoc: isLitdoc} + litdoc := len(parts) > 1 && strings.HasPrefix(strings.TrimSpace(parts[1]), "litdoc") + return InfoString{Lang: lang, Litdoc: litdoc} } func Classify(blocks []Block) ([]Cell, error) { @@ -76,10 +76,10 @@ func Classify(blocks []Block) ([]Cell, error) { for _, b := range blocks { info := ParseInfoString(b) switch { - case info.IsLitdoc && info.Lang == "bash": - cell := MakeBashCellFromRaw(b.content, "") + case info.Litdoc && info.Lang == "bash": + cell := MakeBashCellFromRaw(renderStaticBlock(b), "") cells = append(cells, cell) - case info.IsLitdoc: + case info.Litdoc: return nil, fmt.Errorf("unsupported language: %q", info.Lang) default: cells = append(cells, MakeStaticCellFromRaw(renderStaticBlock(b))) diff --git a/internal/cell_test.go b/internal/cell_test.go index 329f872..a3aa4bb 100644 --- a/internal/cell_test.go +++ b/internal/cell_test.go @@ -9,42 +9,31 @@ import ( "github.com/stretchr/testify/require" ) -func TestMakeStaticCellFromRaw(t *testing.T) { - // given - content := "hello" - - // when - gotCell := internal.MakeStaticCellFromRaw(content) - - // then - got, err := gotCell.Render() - require.NoError(t, err) - assert.Equal(t, content, got) -} - -func TestStaticCellExecute(t *testing.T) { - // given - cell := internal.MakeStaticCellFromRaw("hello") +func TestStaticCell(t *testing.T) { + t.Run("renders raw content", func(t *testing.T) { + // given + content := "hello" + cell := internal.MakeStaticCellFromRaw(content) - // when - gotCell, err := cell.Execute() + // when + gotContent, err := cell.Render() - // then - require.NoError(t, err) - assert.Equal(t, cell, gotCell) -} + // then + require.NoError(t, err) + assert.Equal(t, content, gotContent) + }) -func TestStaticCellRender(t *testing.T) { - // given - content := "hello" - cell := internal.MakeStaticCellFromRaw(content) + t.Run("executes to itself", func(t *testing.T) { + // given + cell := internal.MakeStaticCellFromRaw("hello") - // when - gotContent, err := cell.Render() + // when + gotCell, err := cell.Execute() - // then - require.NoError(t, err) - assert.Equal(t, content, gotContent) + // then + require.NoError(t, err) + assert.Equal(t, cell, gotCell) + }) } func TestParseInfoString(t *testing.T) { @@ -54,59 +43,135 @@ func TestParseInfoString(t *testing.T) { want internal.InfoString }{ { - name: "text block", - block: internal.MakeBlock( - internal.BlockKindText, - "hello", + name: "text block", + block: text("", "hello", false), + want: internal.InfoString{}, + }, + { + name: "fenced code/backtick/without-litdoc", + block: code( "", + joinLines( + "```bash", + "echo hello", + "```", + "", + ), false, ), - want: internal.InfoString{}, + want: internal.InfoString{Lang: "bash"}, }, { - name: "fenced code without litdoc", - block: internal.MakeBlock( - internal.BlockKindFencedCode, + name: "fenced code/backtick/with-litdoc", + block: code( + "", joinLines( - "```bash", + "```bash | litdoc", "echo hello", "```", "", ), + false, + ), + want: internal.InfoString{Lang: "bash", Litdoc: true}, + }, + { + name: "fenced code/tilde/with-litdoc", + block: code( + "", + joinLines( + "~~~sh | litdoc", + "echo hello", + "~~~", + "", + ), + false, + ), + want: internal.InfoString{Lang: "sh", Litdoc: true}, + }, + { + name: "fenced code/no-info-string", + block: code( "", + joinLines( + "```", + "echo hello", + "```", + "", + ), false, ), - want: internal.InfoString{Lang: "bash"}, + want: internal.InfoString{}, }, { - name: "fenced code with litdoc", - block: internal.MakeBlock( - internal.BlockKindFencedCode, + name: "fenced code/trims-language", + block: code( + "", joinLines( - "```bash | litdoc", + "``` bash | litdoc", "echo hello", "```", "", ), + false, + ), + want: internal.InfoString{Lang: "bash", Litdoc: true}, + }, + { + name: "fenced code/litdoc-prefix", + block: code( "", + joinLines( + "```bash | litdoc-output", + "echo hello", + "```", + "", + ), false, ), - want: internal.InfoString{Lang: "bash", IsLitdoc: true}, + want: internal.InfoString{Lang: "bash", Litdoc: true}, }, { - name: "html comment with litdoc", - block: internal.MakeBlock( - internal.BlockKindHTMLComment, + name: "html comment/without-litdoc", + block: cmnt( + "", + joinLines( + "", + "", + ), + false, + ), + want: internal.InfoString{Lang: "bash"}, + }, + { + name: "html comment/with-litdoc", + block: cmnt( + "", joinLines( "", "", ), + false, + ), + want: internal.InfoString{Lang: "bash", Litdoc: true}, + }, + { + name: "html comment/unsupported-litdoc-language", + block: cmnt( "", + joinLines( + "", + "", + ), false, ), - want: internal.InfoString{Lang: "bash", IsLitdoc: true}, + want: internal.InfoString{Lang: "go", Litdoc: true}, }, } @@ -118,45 +183,7 @@ func TestParseInfoString(t *testing.T) { } } -func TestMakeBashCellFromRaw(t *testing.T) { - // given - code := joinLines( - "```bash", - "echo hello", - "```", - "", - ) - - // when - gotCell := internal.MakeBashCellFromRaw(code, "") - - // then - got, err := gotCell.Render() - require.NoError(t, err) - assert.Equal(t, code, got) -} - -func TestBashCellExecute(t *testing.T) { - // given - fencedCode := joinLines( - "```bash", - "echo hello", - "```", - "", - ) - cell := internal.MakeBashCellFromRaw(fencedCode, "") - - // when - gotCell, err := cell.Execute() - - // then - require.NoError(t, err) - rendered, err := gotCell.Render() - require.NoError(t, err) - assert.Equal(t, fencedCode, rendered) -} - -func TestBashCellRender(t *testing.T) { +func TestBashCell(t *testing.T) { t.Run("without output", func(t *testing.T) { // given code := joinLines( @@ -176,6 +203,24 @@ func TestBashCellRender(t *testing.T) { }) t.Run("with output", func(t *testing.T) { + // given + fencedCode := joinLines( + "```bash", + "echo hello", + "```", + ) + output := "hello" + cell := internal.MakeBashCellFromRaw(fencedCode, output) + + // when + gotContent, err := cell.Render() + + // then + require.NoError(t, err) + assert.Equal(t, fencedCode+"\n"+output, gotContent) + }) + + t.Run("executes to itself", func(t *testing.T) { // given fencedCode := joinLines( "```bash", @@ -184,15 +229,15 @@ func TestBashCellRender(t *testing.T) { "", ) cell := internal.MakeBashCellFromRaw(fencedCode, "") - executed, err := cell.Execute() - require.NoError(t, err) // when - gotContent, err := executed.Render() + gotCell, err := cell.Execute() // then require.NoError(t, err) - assert.Equal(t, fencedCode, gotContent) + rendered, err := gotCell.Render() + require.NoError(t, err) + assert.Equal(t, fencedCode, rendered) }) } @@ -261,147 +306,272 @@ func TestCompose(t *testing.T) { } func TestClassify(t *testing.T) { - textBlock := func(content string) internal.Block { - return internal.MakeBlock(internal.BlockKindText, content, "", false) + type wantCell struct { + kind string + rendered string } - bashLitdocBlock := func(content string) internal.Block { - return internal.MakeBlock(internal.BlockKindFencedCode, content, "", false) - } - - t.Run("single text block becomes StaticCell", func(t *testing.T) { - // given - blocks := []internal.Block{textBlock("hello")} - - // when - cells, err := internal.Classify(blocks) - - // then - require.NoError(t, err) - require.Len(t, cells, 1) - _, ok := cells[0].(internal.StaticCell) - require.True(t, ok, "expected StaticCell, got %T", cells[0]) - got, err := cells[0].Render() - require.NoError(t, err) - assert.Equal(t, "hello", got) - }) - t.Run("litdoc bash block becomes BashCell", func(t *testing.T) { - // given - code := joinLines( - "```bash | litdoc", - "echo hello", - "```", - "", - ) - blocks := []internal.Block{bashLitdocBlock(code)} - - // when - cells, err := internal.Classify(blocks) - - // then - require.NoError(t, err) - require.Len(t, cells, 1) - _, ok := cells[0].(internal.BashCell) - require.True(t, ok, "expected BashCell, got %T", cells[0]) - rendered, err := cells[0].Render() - require.NoError(t, err) - assert.Equal(t, code, rendered) - }) - - t.Run("mixed block types are each classified independently", func(t *testing.T) { - // given - code := joinLines( - "```bash | litdoc", - "echo hello", - "```", - "", - ) - blocks := []internal.Block{ - textBlock("before"), - bashLitdocBlock(code), - textBlock("after"), - } - - // when - cells, err := internal.Classify(blocks) - - // then - require.NoError(t, err) - require.Len(t, cells, 3) - rendered0, err := cells[0].Render() - require.NoError(t, err) - assert.Equal(t, "before", rendered0) - _, ok := cells[1].(internal.BashCell) - assert.True(t, ok, "expected BashCell, got %T", cells[1]) - rendered2, err := cells[2].Render() - require.NoError(t, err) - assert.Equal(t, "after", rendered2) - }) - - t.Run("preserves inline HTML comment continuations when composing static cells", func(t *testing.T) { - tests := []struct { - name string - input string - }{ - { - name: "list", - input: joinLines( - "- text text", - ), + tests := []struct { + name string + blocks []internal.Block + want []wantCell + wantErrText string + }{ + { + name: "Text/top-level", + blocks: []internal.Block{ + text("", "hello", false), }, - { - name: "blockquote list", - input: joinLines( - "> - text text", - ), + want: []wantCell{ + {kind: "StaticCell", rendered: "hello"}, }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - blocks, err := internal.MakeBlocksFromMarkdown([]byte(tt.input)) - require.NoError(t, err) - cells, err := internal.Classify(blocks) - require.NoError(t, err) - - got, err := internal.Compose(cells) - - require.NoError(t, err) - assert.Equal(t, tt.input, got) - }) - } - }) - - t.Run("non-litdoc fenced code block becomes StaticCell", func(t *testing.T) { - // given - code := "```bash\necho hello\n```\n" - blocks := []internal.Block{ - internal.MakeBlock(internal.BlockKindFencedCode, code, "", false), - } - - // when - cells, err := internal.Classify(blocks) - - // then - require.NoError(t, err) - require.Len(t, cells, 1) - rendered, err := cells[0].Render() - require.NoError(t, err) - assert.Equal(t, code, rendered) - }) - - t.Run("litdoc block with unsupported language", func(t *testing.T) { - // given - blocks := []internal.Block{ - bashLitdocBlock("```go | litdoc\nfmt.Println()\n```\n"), - } + }, + { + name: "Text/blockquote/multiline", + blocks: []internal.Block{ + text("> ", "hello, \nworld\n", false), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "> hello, \n> world\n"}, + }, + }, + { + name: "Text/list/dash", + blocks: []internal.Block{ + text("- ", "text\ncontinued", false), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "- text\n continued"}, + }, + }, + { + name: "Text/list/ordered", + blocks: []internal.Block{ + text("2. ", "text\ncontinued", false), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "2. text\n continued"}, + }, + }, + { + name: "Text/blockquote/list", + blocks: []internal.Block{ + text("> - ", "text\ncontinued", false), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "> - text\n> continued"}, + }, + }, + { + name: "FencedCode/non-litdoc/top-level", + blocks: []internal.Block{ + code("", joinLines( + "```bash", + "echo hello", + "```", + "", + ), false), + }, + want: []wantCell{ + { + kind: "StaticCell", rendered: joinLines( + "```bash", + "echo hello", + "```", + "", + ), + }, + }, + }, + { + name: "FencedCode/non-litdoc/list", + blocks: []internal.Block{ + code("- ", joinLines( + "```bash", + "echo hello", + "```", + "", + ), false), + }, + want: []wantCell{ + { + kind: "StaticCell", rendered: joinLines( + "- ```bash", + " echo hello", + " ```", + "", + ), + }, + }, + }, + { + name: "FencedCode/litdoc/bash", + blocks: []internal.Block{ + code("", joinLines( + "```bash | litdoc", + "echo hello", + "```", + "", + ), false), + }, + want: []wantCell{ + { + kind: "BashCell", rendered: joinLines( + "```bash | litdoc", + "echo hello", + "```", + "", + ), + }, + }, + }, + { + name: "FencedCode/litdoc/unsupported-language", + blocks: []internal.Block{ + code("", joinLines( + "```go | litdoc", + "fmt.Println()", + "```", + "", + ), false), + }, + wantErrText: "unsupported language", + }, + { + name: "HTMLComment/block/list", + blocks: []internal.Block{ + cmnt("- ", joinLines( + "", + "", + ), false), + }, + want: []wantCell{ + { + kind: "StaticCell", rendered: joinLines( + "- ", + "", + ), + }, + }, + }, + { + name: "HTMLComment/litdoc/bash", + blocks: []internal.Block{ + cmnt("", joinLines( + "", + "", + ), false), + }, + want: []wantCell{ + { + kind: "BashCell", rendered: joinLines( + "", + "", + ), + }, + }, + }, + { + name: "HTMLComment/inline-continuation/list", + blocks: []internal.Block{ + text("- ", "text ", false), + cmnt("- ", "", true), + cmnt("- ", "", true), + text("- ", " text", true), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "- text "}, + {kind: "StaticCell", rendered: ""}, + {kind: "StaticCell", rendered: ""}, + {kind: "StaticCell", rendered: " text"}, + }, + }, + { + name: "HTMLComment/inline-continuation/blockquote-list", + blocks: []internal.Block{ + text("> - ", "text ", false), + cmnt("> - ", "", true), + cmnt("> - ", "", true), + text("> - ", " text", true), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "> - text "}, + {kind: "StaticCell", rendered: ""}, + {kind: "StaticCell", rendered: ""}, + {kind: "StaticCell", rendered: " text"}, + }, + }, + { + name: "Mixed/independent-blocks", + blocks: []internal.Block{ + text("", "before", false), + code("", joinLines( + "```bash | litdoc", + "echo hello", + "```", + "", + ), false), + text("", "after", false), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "before"}, + { + kind: "BashCell", rendered: joinLines( + "```bash | litdoc", + "echo hello", + "```", + "", + ), + }, + {kind: "StaticCell", rendered: "after"}, + }, + }, + } - // when - _, err := internal.Classify(blocks) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // when + cells, err := internal.Classify(tt.blocks) + + // then + if tt.wantErrText != "" { + require.ErrorContains(t, err, tt.wantErrText) + return + } + + require.NoError(t, err) + require.Len(t, cells, len(tt.want)) + wantComposed := "" + for i, w := range tt.want { + assert.Equal(t, w.kind, cellKind(cells[i]), "cell[%d] kind", i) + rendered, err := cells[i].Render() + require.NoError(t, err, "cell[%d] render", i) + assert.Equal(t, w.rendered, rendered, "cell[%d] rendered", i) + wantComposed += w.rendered + } + + gotComposed, err := internal.Compose(cells) + require.NoError(t, err) + assert.Equal(t, wantComposed, gotComposed) + }) + } +} - // then - require.ErrorContains(t, err, "unsupported language") - }) +func cellKind(cell internal.Cell) string { + switch cell.(type) { + case internal.StaticCell: + return "StaticCell" + case internal.BashCell: + return "BashCell" + default: + return "unknown" + } } diff --git a/internal/helpers_test.go b/internal/helpers_test.go new file mode 100644 index 0000000..c593e01 --- /dev/null +++ b/internal/helpers_test.go @@ -0,0 +1,23 @@ +package internal_test + +import ( + "strings" + + "litdoc/internal" +) + +func text(indent, content string, continuation bool) internal.Block { + return internal.MakeBlockFromRaw(internal.BlockKindText, content, indent, continuation) +} + +func code(indent, content string, continuation bool) internal.Block { + return internal.MakeBlockFromRaw(internal.BlockKindFencedCode, content, indent, continuation) +} + +func cmnt(indent, content string, continuation bool) internal.Block { + return internal.MakeBlockFromRaw(internal.BlockKindHTMLComment, content, indent, continuation) +} + +func joinLines(lines ...string) string { + return strings.Join(lines, "\n") +} From e50e8c3e415f814ce919fd3c930eb951932d6187 Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Wed, 29 Apr 2026 13:30:36 -0600 Subject: [PATCH 16/18] Classify blocks based on their kind first --- internal/cell.go | 19 ++++++++++++------- internal/cell_test.go | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/internal/cell.go b/internal/cell.go index 9d0baf5..72aeb3c 100644 --- a/internal/cell.go +++ b/internal/cell.go @@ -74,13 +74,18 @@ func ParseInfoString(b Block) InfoString { func Classify(blocks []Block) ([]Cell, error) { var cells []Cell for _, b := range blocks { - info := ParseInfoString(b) - switch { - case info.Litdoc && info.Lang == "bash": - cell := MakeBashCellFromRaw(renderStaticBlock(b), "") - cells = append(cells, cell) - case info.Litdoc: - return nil, fmt.Errorf("unsupported language: %q", info.Lang) + switch b.kind { + case BlockKindFencedCode, BlockKindHTMLComment: + info := ParseInfoString(b) + switch { + case info.Litdoc && info.Lang == "bash": + cell := MakeBashCellFromRaw(renderStaticBlock(b), "") + cells = append(cells, cell) + case info.Litdoc: + return nil, fmt.Errorf("unsupported language: %q", info.Lang) + default: + cells = append(cells, MakeStaticCellFromRaw(renderStaticBlock(b))) + } default: cells = append(cells, MakeStaticCellFromRaw(renderStaticBlock(b))) } diff --git a/internal/cell_test.go b/internal/cell_test.go index a3aa4bb..6f56870 100644 --- a/internal/cell_test.go +++ b/internal/cell_test.go @@ -362,6 +362,27 @@ func TestClassify(t *testing.T) { {kind: "StaticCell", rendered: "> - text\n> continued"}, }, }, + { + name: "Text/litdoc-looking-content", + blocks: []internal.Block{ + text("", joinLines( + "", + "", + ), false), + }, + want: []wantCell{ + { + kind: "StaticCell", rendered: joinLines( + "", + "", + ), + }, + }, + }, { name: "FencedCode/non-litdoc/top-level", blocks: []internal.Block{ From 212f97eb71a82d919eb99ffde28ca3d676cb8fa3 Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Thu, 30 Apr 2026 11:51:05 -0600 Subject: [PATCH 17/18] Don't merge blockquote multiline into a single block --- internal/block.go | 233 +++++++++++++++++++++++------------------ internal/block_test.go | 3 +- internal/cell.go | 1 + 3 files changed, 134 insertions(+), 103 deletions(-) diff --git a/internal/block.go b/internal/block.go index a059da4..8d64c5d 100644 --- a/internal/block.go +++ b/internal/block.go @@ -20,24 +20,12 @@ const ( BlockKindHTMLComment BlockKind = "html_comment" ) -const ( - markdownNodeDocument = "document" - markdownNodeSection = "section" - markdownNodeBlockQuote = "block_quote" - markdownNodeBlockQuoteMarker = "block_quote_marker" - markdownNodeList = "list" - markdownNodeListItem = "list_item" - markdownNodeFencedCodeBlock = "fenced_code_block" - markdownNodeHTMLBlock = "html_block" - markdownListMarkerPrefix = "list_marker_" -) - type Block struct { - kind BlockKind + kind BlockKind + // indent characters potentially nested blockquotes and lists indent string content string - // continuation blocks keep their container indent for embedded newlines, - // but do not render that indent before their first line. + // continuation when preceding block is on the smame line continuation bool } @@ -67,14 +55,6 @@ func (b Block) String() string { return fmt.Sprintf("{%s %q}", b.kind, b.content) } -var ( - htmlCommentRe = regexp.MustCompile(`(?s)`) - openingFenceRe = regexp.MustCompile(`^[ ]{0,3}(` + "`" + `{3,}|~{3,})`) -) - -// MakeBlocksFromMarkdown parses UTF-8 Markdown into Blocks. -// Block content is stored without its surrounding quote/list prefix; Block.indent -// stores the prefix needed to render that Block.content back into the same container. func MakeBlocksFromMarkdown(content []byte) ([]Block, error) { if !utf8.Valid(content) { return nil, fmt.Errorf("markdown content is not valid UTF-8") @@ -91,59 +71,12 @@ func MakeBlocksFromMarkdown(content []byte) ([]Block, error) { if collector.err != nil { return nil, collector.err } - collector.emitTrailingText() + collector.appendTrailingText() + // inline comments come in a text block. Split them out in a linear pass. return splitInlineHTMLComments(collector.blocks), nil } -func splitInlineHTMLComments(blocks []Block) []Block { - result := make([]Block, 0, len(blocks)) - for _, b := range blocks { - if b.Kind() != BlockKindText { - result = append(result, b) - continue - } - content := b.Content() - locs := htmlCommentRe.FindAllStringIndex(content, -1) - if len(locs) == 0 { - result = append(result, b) - continue - } - pos := 0 - for _, loc := range locs { - if loc[0] > pos { - result = append(result, MakeBlockFromRaw( - BlockKindText, - content[pos:loc[0]], - b.indent, - b.continuation || pos > 0, - )) - } - wholeBlock := isWholeTextBlock(content, loc) - commentContinuation := b.continuation - if !wholeBlock { - commentContinuation = b.continuation || loc[0] > 0 - } - result = append(result, MakeBlockFromRaw( - BlockKindHTMLComment, - content[loc[0]:loc[1]], - b.indent, - commentContinuation, - )) - pos = loc[1] - } - if pos < len(content) { - result = append(result, MakeBlockFromRaw(BlockKindText, content[pos:], b.indent, true)) - } - } - return result -} - -func isWholeTextBlock(content string, loc []int) bool { - return len(strings.TrimSpace(content[:loc[0]])) == 0 && - len(strings.TrimSpace(content[loc[1]:])) == 0 -} - type blockCollector struct { content []byte pos uint32 @@ -151,23 +84,17 @@ type blockCollector struct { err error } -// indent and stripPrefix describe the current markdown container. -// -// indent is the prefix used when rendering a Block back into its container. -// stripPrefix is the source prefix removed from every line before storing block -// content. They differ for list items: "- " is rendered on the first line, but -// continuation lines are prefixed by spaces. func (c *blockCollector) collect(node *sitter.Node, indent, stripPrefix []byte) { if c.err != nil { return } switch node.Type() { - case markdownNodeDocument, markdownNodeSection, markdownNodeList: + case "document", "section", "list": c.collectChildren(node, indent, stripPrefix) - case markdownNodeBlockQuote: + case "block_quote": c.collectBlockQuote(node, indent, stripPrefix) - case markdownNodeListItem: + case "list_item": c.collectListItem(node, indent, stripPrefix) default: c.collectLeaf(node, indent, stripPrefix) @@ -182,14 +109,15 @@ func (c *blockCollector) collectChildren(node *sitter.Node, indent, stripPrefix func (c *blockCollector) collectBlockQuote( node *sitter.Node, - indent, stripPrefix []byte, + indent []byte, + stripPrefix []byte, ) { - childIndent := appendPrefix(indent, "> ") - childStripPrefix := appendPrefix(stripPrefix, "> ") + childIndent := concatenate(indent, []byte("> ")) + childStripPrefix := concatenate(stripPrefix, []byte("> ")) for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) - if child.Type() == markdownNodeBlockQuoteMarker { - c.emitTextGap(child.StartByte(), childStripPrefix, childIndent) + if child.Type() == "block_quote_marker" { + c.appendTextGap(child.StartByte(), childStripPrefix, childIndent) c.pos = child.EndByte() continue } @@ -202,7 +130,7 @@ func (c *blockCollector) collectListItem(node *sitter.Node, indent, stripPrefix for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) if isListMarker(child) { - c.emitTextGap(child.StartByte(), stripPrefix, indent) + c.appendTextGap(child.StartByte(), stripPrefix, indent) c.pos = child.EndByte() continue } @@ -215,7 +143,7 @@ func (c *blockCollector) collectLeaf(node *sitter.Node, indent, stripPrefix []by end := node.EndByte() blockIndent, blockStripPrefix := blockPrefixes(node, c.content, indent, stripPrefix) - c.emitTextGap(start, stripPrefix, indent) + c.appendTextGap(start, stripPrefix, indent) raw := stripIndent(c.content[start:end], blockStripPrefix) kind := blockKind(node, c.content) @@ -228,31 +156,74 @@ func (c *blockCollector) collectLeaf(node *sitter.Node, indent, stripPrefix []by return } - c.emitBlock(kind, raw, blockIndent) + c.appendBlock(kind, raw, blockIndent) c.pos = end } -func (c *blockCollector) emitTrailingText() { +func (c *blockCollector) appendTrailingText() { if c.pos < uint32(len(c.content)) { - c.emitBlock(BlockKindText, c.content[c.pos:], nil) + c.appendBlock(BlockKindText, c.content[c.pos:], nil) } } -func (c *blockCollector) emitTextGap(end uint32, stripPrefix, indent []byte) { +func (c *blockCollector) appendTextGap(end uint32, stripPrefix, indent []byte) { if end <= c.pos { return } gap := stripIndent(c.content[c.pos:end], stripPrefix) - c.emitBlock(BlockKindText, gap, indent) + c.appendBlock(BlockKindText, gap, indent) } -func (c *blockCollector) emitBlock(kind BlockKind, raw, indent []byte) { +func (c *blockCollector) appendBlock(kind BlockKind, raw, indent []byte) { if len(raw) == 0 { return } + if kind == BlockKindText && isBlockQuoteOnlyIndent(indent) && !bytes.Contains(raw, []byte("")) } -func appendPrefix(prefix []byte, suffix string) []byte { - result := make([]byte, 0, len(prefix)+len(suffix)) - result = append(result, prefix...) - result = append(result, suffix...) +func concatenate(left, right []byte) []byte { + result := make([]byte, 0, len(left)+len(right)) + result = append(result, left...) + result = append(result, right...) return result } @@ -397,7 +370,7 @@ func listItemPrefixes(node *sitter.Node, content []byte) ([]byte, []byte) { } func isListMarker(node *sitter.Node) bool { - return strings.HasPrefix(node.Type(), markdownListMarkerPrefix) + return strings.HasPrefix(node.Type(), "list_marker_") } func linePrefixBefore(content []byte, pos uint32) []byte { @@ -472,9 +445,9 @@ func quoteParentIndent(indent []byte) []byte { func blockKind(node *sitter.Node, content []byte) BlockKind { switch node.Type() { - case markdownNodeFencedCodeBlock: + case "fenced_code_block": return BlockKindFencedCode - case markdownNodeHTMLBlock: + case "html_block": if bytes.HasPrefix(content[node.StartByte():node.EndByte()], []byte("`) + +func splitInlineHTMLComments(blocks []Block) []Block { + result := make([]Block, 0, len(blocks)) + for _, b := range blocks { + if b.Kind() != BlockKindText { + result = append(result, b) + continue + } + content := b.Content() + locs := htmlCommentRe.FindAllStringIndex(content, -1) + if len(locs) == 0 { + result = append(result, b) + continue + } + pos := 0 + for _, loc := range locs { + if loc[0] > pos { + result = append( + result, + MakeBlockFromRaw( + BlockKindText, + content[pos:loc[0]], + b.indent, + b.continuation || pos > 0, + ), + ) + } + wholeBlock := isWholeTextBlock(content, loc) + commentContinuation := b.continuation + if !wholeBlock { + commentContinuation = b.continuation || loc[0] > 0 + } + result = append(result, MakeBlockFromRaw( + BlockKindHTMLComment, + content[loc[0]:loc[1]], + b.indent, + commentContinuation, + )) + pos = loc[1] + } + if pos < len(content) { + result = append( + result, + MakeBlockFromRaw(BlockKindText, content[pos:], b.indent, true), + ) + } + } + return result +} + +func isWholeTextBlock(content string, loc []int) bool { + return len(strings.TrimSpace(content[:loc[0]])) == 0 && + len(strings.TrimSpace(content[loc[1]:])) == 0 +} diff --git a/internal/block_test.go b/internal/block_test.go index dc87591..20cf9a8 100644 --- a/internal/block_test.go +++ b/internal/block_test.go @@ -120,7 +120,8 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { "> again", ), want: []internal.Block{ - text("> ", "hello, \nworld\n", false), + text("> ", "hello, \n", false), + text("> ", "world\n", false), text("> ", "\n", false), text("> ", "again", false), }, diff --git a/internal/cell.go b/internal/cell.go index 72aeb3c..00da053 100644 --- a/internal/cell.go +++ b/internal/cell.go @@ -71,6 +71,7 @@ func ParseInfoString(b Block) InfoString { return InfoString{Lang: lang, Litdoc: litdoc} } +// Classify todo: review this function func Classify(blocks []Block) ([]Cell, error) { var cells []Cell for _, b := range blocks { From dc0f659e604bafe894d8ab6b8457134537931c10 Mon Sep 17 00:00:00 2001 From: Mike Wittie Date: Thu, 30 Apr 2026 12:02:58 -0600 Subject: [PATCH 18/18] Move TLD testscript tests into cmd and remove the dependency on make build --- cmd/block_test.go | 7 ++++ cmd/file.go | 10 +++--- cmd/file_test.go | 34 +++++++++++++++++++ {testdata => cmd/testdata}/script/block.txtar | 0 {testdata => cmd/testdata}/script/file.txtar | 3 +- main_test.go | 23 ------------- 6 files changed, 48 insertions(+), 29 deletions(-) create mode 100644 cmd/block_test.go create mode 100644 cmd/file_test.go rename {testdata => cmd/testdata}/script/block.txtar (100%) rename {testdata => cmd/testdata}/script/file.txtar (56%) delete mode 100644 main_test.go diff --git a/cmd/block_test.go b/cmd/block_test.go new file mode 100644 index 0000000..001509b --- /dev/null +++ b/cmd/block_test.go @@ -0,0 +1,7 @@ +package cmd_test + +import "testing" + +func TestBlockCommand(t *testing.T) { + runScript(t, "testdata/script/block.txtar") +} diff --git a/cmd/file.go b/cmd/file.go index 4cbddca..d0c78d4 100644 --- a/cmd/file.go +++ b/cmd/file.go @@ -2,22 +2,24 @@ package cmd import ( "fmt" - "io" "os" + "litdoc/internal" + "github.com/spf13/cobra" ) var fileCmd = &cobra.Command{ - Use: "file", + Use: "file ", Short: "Process a file", + Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { - data, err := io.ReadAll(os.Stdin) + data, err := internal.ProcessFile(args[0]) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - fmt.Print(string(data)) + fmt.Print(data) }, } diff --git a/cmd/file_test.go b/cmd/file_test.go new file mode 100644 index 0000000..62200d7 --- /dev/null +++ b/cmd/file_test.go @@ -0,0 +1,34 @@ +package cmd_test + +import ( + "fmt" + "os" + "testing" + + "litdoc/cmd" + + "github.com/rogpeppe/go-internal/testscript" +) + +func TestMain(m *testing.M) { + testscript.Main(m, map[string]func(){ + "litdoc": func() { + if err := cmd.Execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + }, + }) +} + +func TestFileCommand(t *testing.T) { + runScript(t, "testdata/script/file.txtar") +} + +func runScript(t *testing.T, script string) { + t.Helper() + + testscript.Run(t, testscript.Params{ + Files: []string{script}, + }) +} diff --git a/testdata/script/block.txtar b/cmd/testdata/script/block.txtar similarity index 100% rename from testdata/script/block.txtar rename to cmd/testdata/script/block.txtar diff --git a/testdata/script/file.txtar b/cmd/testdata/script/file.txtar similarity index 56% rename from testdata/script/file.txtar rename to cmd/testdata/script/file.txtar index 5809cf5..d8aafae 100644 --- a/testdata/script/file.txtar +++ b/cmd/testdata/script/file.txtar @@ -1,5 +1,4 @@ -stdin input.md -exec litdoc file +exec litdoc file input.md stdout '# Hello' -- input.md -- diff --git a/main_test.go b/main_test.go deleted file mode 100644 index 36bbea9..0000000 --- a/main_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package main_test - -import ( - "os" - "path/filepath" - "testing" - - "github.com/rogpeppe/go-internal/testscript" -) - -func TestScript(t *testing.T) { - binPath, err := filepath.Abs("bin") - if err != nil { - t.Fatal(err) - } - testscript.Run(t, testscript.Params{ - Dir: "testdata/script", - Setup: func(env *testscript.Env) error { - env.Setenv("PATH", binPath+string(os.PathListSeparator)+env.Getenv("PATH")) - return nil - }, - }) -}