diff --git a/internal/block.go b/internal/block.go index 1c09b53..8d64c5d 100644 --- a/internal/block.go +++ b/internal/block.go @@ -4,6 +4,9 @@ import ( "bytes" "context" "fmt" + "regexp" + "strings" + "unicode/utf8" sitter "github.com/smacker/go-tree-sitter" "github.com/smacker/go-tree-sitter/markdown" @@ -12,73 +15,434 @@ import ( type BlockKind string const ( - BlockKindText BlockKind = "Text" - BlockKindFencedCode BlockKind = "fencedCode" - BlockKindHTMLComment BlockKind = "HTMLComment" + BlockKindText BlockKind = "text" + BlockKindFencedCode BlockKind = "fenced_code" + BlockKindHTMLComment BlockKind = "html_comment" ) type Block struct { - kind BlockKind - content []byte + kind BlockKind + // indent characters potentially nested blockquotes and lists + indent string + content string + // continuation when preceding block is on the smame line + continuation bool } -func MakeBlockFromRaw(kind BlockKind, raw []byte) Block { - return Block{kind: kind, content: raw} +func MakeBlockFromRaw( + kind BlockKind, + content string, + indent string, + continuation bool, +) Block { + return Block{ + kind: kind, + indent: indent, + content: content, + continuation: continuation, + } } func (b Block) Kind() BlockKind { return b.kind } -func (b Block) Content() []byte { return b.content } +func (b Block) Indent() string { return b.indent } + +func (b Block) Content() string { return b.content } + +func (b Block) Continuation() bool { return b.continuation } + +func (b Block) String() string { + return fmt.Sprintf("{%s %q}", b.kind, b.content) +} func MakeBlocksFromMarkdown(content []byte) ([]Block, error) { + if !utf8.Valid(content) { + return nil, fmt.Errorf("markdown content is not valid UTF-8") + } + tree, err := markdown.ParseCtx(context.Background(), nil, content) if err != nil { return nil, fmt.Errorf("parsing markdown: %w", err) } root := tree.BlockTree().RootNode() - var blocks []Block - pos := uint32(0) + collector := blockCollector{content: content} + collector.collect(root, nil, nil) + if collector.err != nil { + return nil, collector.err + } + collector.appendTrailingText() - collectBlockNodes(root, content, &pos, &blocks) + // inline comments come in a text block. Split them out in a linear pass. + return splitInlineHTMLComments(collector.blocks), nil +} + +type blockCollector struct { + content []byte + pos uint32 + blocks []Block + err error +} - if pos < uint32(len(content)) { - blocks = append(blocks, Block{kind: BlockKindText, content: content[pos:]}) +func (c *blockCollector) collect(node *sitter.Node, indent, stripPrefix []byte) { + if c.err != nil { + return + } + + switch node.Type() { + case "document", "section", "list": + c.collectChildren(node, indent, stripPrefix) + case "block_quote": + c.collectBlockQuote(node, indent, stripPrefix) + case "list_item": + c.collectListItem(node, indent, stripPrefix) + default: + c.collectLeaf(node, indent, stripPrefix) } +} - return blocks, nil +func (c *blockCollector) collectChildren(node *sitter.Node, indent, stripPrefix []byte) { + for i := 0; i < int(node.ChildCount()); i++ { + c.collect(node.Child(i), indent, stripPrefix) + } } -func collectBlockNodes( +func (c *blockCollector) collectBlockQuote( node *sitter.Node, - content []byte, - pos *uint32, - blocks *[]Block, + indent []byte, + stripPrefix []byte, ) { - switch node.Type() { - case "document", "section": - for i := 0; i < int(node.ChildCount()); i++ { - collectBlockNodes(node.Child(i), content, pos, blocks) + childIndent := concatenate(indent, []byte("> ")) + childStripPrefix := concatenate(stripPrefix, []byte("> ")) + for i := 0; i < int(node.ChildCount()); i++ { + child := node.Child(i) + if child.Type() == "block_quote_marker" { + c.appendTextGap(child.StartByte(), childStripPrefix, childIndent) + c.pos = child.EndByte() + continue } - default: - start := node.StartByte() - end := node.EndByte() + c.collect(child, childIndent, childStripPrefix) + } +} - if start > *pos { - *blocks = append( - *blocks, - MakeBlockFromRaw(BlockKindText, content[*pos:start]), - ) +func (c *blockCollector) collectListItem(node *sitter.Node, indent, stripPrefix []byte) { + childIndent, childStripPrefix := listItemPrefixes(node, c.content) + for i := 0; i < int(node.ChildCount()); i++ { + child := node.Child(i) + if isListMarker(child) { + c.appendTextGap(child.StartByte(), stripPrefix, indent) + c.pos = child.EndByte() + continue } + c.collect(child, childIndent, childStripPrefix) + } +} + +func (c *blockCollector) collectLeaf(node *sitter.Node, indent, stripPrefix []byte) { + start := node.StartByte() + end := node.EndByte() + blockIndent, blockStripPrefix := blockPrefixes(node, c.content, indent, stripPrefix) + + c.appendTextGap(start, stripPrefix, indent) + + raw := stripIndent(c.content[start:end], blockStripPrefix) + kind := blockKind(node, c.content) + if kind == BlockKindFencedCode && !isClosedFencedCodeBlock(raw) { + c.err = fmt.Errorf("unclosed fenced code block at byte %d", start) + return + } + if kind == BlockKindHTMLComment && !isClosedHTMLComment(raw) { + c.err = fmt.Errorf("unclosed HTML comment at byte %d", start) + return + } + + c.appendBlock(kind, raw, blockIndent) + c.pos = end +} - *blocks = append( - *blocks, - MakeBlockFromRaw(blockKind(node, content), content[start:end]), - ) - *pos = end +func (c *blockCollector) appendTrailingText() { + if c.pos < uint32(len(c.content)) { + c.appendBlock(BlockKindText, c.content[c.pos:], nil) } } +func (c *blockCollector) appendTextGap(end uint32, stripPrefix, indent []byte) { + if end <= c.pos { + return + } + gap := stripIndent(c.content[c.pos:end], stripPrefix) + c.appendBlock(BlockKindText, gap, indent) +} + +func (c *blockCollector) appendBlock(kind BlockKind, raw, indent []byte) { + if len(raw) == 0 { + return + } + if kind == BlockKindText && isBlockQuoteOnlyIndent(indent) && !bytes.Contains(raw, []byte("")) +} + +func concatenate(left, right []byte) []byte { + result := make([]byte, 0, len(left)+len(right)) + result = append(result, left...) + result = append(result, right...) + return result +} + +// listItemPrefixes returns both views of a list marker: the marker itself is +// used when rendering the first line, while equivalent spaces are stripped from +// continuation lines in the source. +func listItemPrefixes(node *sitter.Node, content []byte) ([]byte, []byte) { + for i := 0; i < int(node.ChildCount()); i++ { + child := node.Child(i) + if !isListMarker(child) { + continue + } + linePrefix := linePrefixBefore(content, child.StartByte()) + marker := content[child.StartByte():child.EndByte()] + indent := append(append([]byte(nil), linePrefix...), marker...) + stripPrefix := append(append([]byte(nil), linePrefix...), bytes.Repeat([]byte(" "), len(marker))...) + return indent, stripPrefix + } + return nil, nil +} + +func isListMarker(node *sitter.Node) bool { + return strings.HasPrefix(node.Type(), "list_marker_") +} + +func linePrefixBefore(content []byte, pos uint32) []byte { + lineStart := bytes.LastIndexByte(content[:pos], '\n') + 1 + return content[lineStart:pos] +} + +func leadingLineSpaces(content []byte) []byte { + end := bytes.IndexByte(content, '\n') + if end < 0 { + end = len(content) + } + line := content[:end] + return line[:len(line)-len(bytes.TrimLeft(line, " "))] +} + +// stripIndent removes a container prefix from each line before storing content +// in a Block. Blockquote-only blank lines are normalized to empty lines so the +// rendered output does not keep stray ">" markers as content. +func stripIndent(content, indent []byte) []byte { + if len(indent) == 0 { + return content + } + lines := bytes.Split(content, []byte("\n")) + parentIndent := quoteParentIndent(indent) + for i, line := range lines { + switch { + case bytes.HasPrefix(line, indent): + lines[i] = bytes.TrimPrefix(line, indent) + case isSpaceIndent(indent): + lines[i] = trimSpaceIndent(line, len(indent)) + case bytes.Contains(indent, []byte("> ")) && isQuoteOnlyLine(line): + lines[i] = nil + case len(parentIndent) > 0 && bytes.Equal(line, parentIndent): + lines[i] = nil + } + } + return bytes.Join(lines, []byte("\n")) +} + +func isSpaceIndent(indent []byte) bool { + return len(bytes.Trim(indent, " ")) == 0 +} + +func trimSpaceIndent(line []byte, width int) []byte { + i := 0 + for i < len(line) && i < width && line[i] == ' ' { + i++ + } + return line[i:] +} + +func isQuoteOnlyLine(line []byte) bool { + trimmed := bytes.TrimSpace(line) + if len(trimmed) == 0 { + return false + } + for _, b := range trimmed { + if b != '>' && b != ' ' { + return false + } + } + return true +} + +func quoteParentIndent(indent []byte) []byte { + if len(indent) < len("> ") || !bytes.HasSuffix(indent, []byte("> ")) { + return nil + } + return indent[:len(indent)-len("> ")] +} + func blockKind(node *sitter.Node, content []byte) BlockKind { switch node.Type() { case "fenced_code_block": @@ -92,3 +456,59 @@ func blockKind(node *sitter.Node, content []byte) BlockKind { return BlockKindText } } + +var htmlCommentRe = regexp.MustCompile(`(?s)`) + +func splitInlineHTMLComments(blocks []Block) []Block { + result := make([]Block, 0, len(blocks)) + for _, b := range blocks { + if b.Kind() != BlockKindText { + result = append(result, b) + continue + } + content := b.Content() + locs := htmlCommentRe.FindAllStringIndex(content, -1) + if len(locs) == 0 { + result = append(result, b) + continue + } + pos := 0 + for _, loc := range locs { + if loc[0] > pos { + result = append( + result, + MakeBlockFromRaw( + BlockKindText, + content[pos:loc[0]], + b.indent, + b.continuation || pos > 0, + ), + ) + } + wholeBlock := isWholeTextBlock(content, loc) + commentContinuation := b.continuation + if !wholeBlock { + commentContinuation = b.continuation || loc[0] > 0 + } + result = append(result, MakeBlockFromRaw( + BlockKindHTMLComment, + content[loc[0]:loc[1]], + b.indent, + commentContinuation, + )) + pos = loc[1] + } + if pos < len(content) { + result = append( + result, + MakeBlockFromRaw(BlockKindText, content[pos:], b.indent, true), + ) + } + } + return result +} + +func isWholeTextBlock(content string, loc []int) bool { + return len(strings.TrimSpace(content[:loc[0]])) == 0 && + len(strings.TrimSpace(content[loc[1]:])) == 0 +} diff --git a/internal/block_test.go b/internal/block_test.go index 6f0bab3..20cf9a8 100644 --- a/internal/block_test.go +++ b/internal/block_test.go @@ -12,130 +12,616 @@ import ( func TestMakeBlockFromRaw(t *testing.T) { // given kind := internal.BlockKindFencedCode - content := []byte("```bash\necho hello\n```\n") + content := "```bash\necho hello\n```\n" + indent := "> " + continuation := true // when - got := internal.MakeBlockFromRaw(kind, content) + got := internal.MakeBlockFromRaw(kind, content, indent, continuation) // then assert.Equal(t, kind, got.Kind()) assert.Equal(t, content, got.Content()) + assert.Equal(t, indent, got.Indent()) + assert.Equal(t, continuation, got.Continuation()) } func TestMakeBlocksFromMarkdown(t *testing.T) { tests := []struct { name string input string - want []struct { - kind internal.BlockKind - content string - } + want []internal.Block }{ + // Text — baseline cases that should produce no FencedCode or HTMLComment blocks. { - name: "heading", + name: "Text/heading", input: "# Hello\n", - want: []struct { - kind internal.BlockKind - content string - }{ - {internal.BlockKindText, "# Hello\n"}, + want: []internal.Block{ + text("", "# Hello\n", false), }, }, { - name: "text", - input: "text\nand more text", - want: []struct { - kind internal.BlockKind - content string - }{ - {internal.BlockKindText, "text\nand more text"}, + name: "Text/heading/nested", + input: joinLines( + "# Level 1", + "", + "some text", + "", + "## Level 2", + "", + "more text", + "", + ), + want: []internal.Block{ + text("", "# Level 1\n", false), + text("", "\n", false), + text("", "some text\n", false), + text("", "\n", false), + text("", "## Level 2\n", false), + text("", "\n", false), + text("", "more text\n", false), }, }, { - name: "fenced code block", - input: "```bash\necho \"hello\"\n```\n", - want: []struct { - kind internal.BlockKind - content string - }{ - {internal.BlockKindFencedCode, "```bash\necho \"hello\"\n```\n"}, + name: "Text/paragraph/grouped", + input: joinLines( + "text", + "and more text", + "", + "last line", + ), + want: []internal.Block{ + text("", "text\nand more text\n", false), + text("", "\n", false), + text("", "last line", false), }, }, { - name: "tilde fenced code block", - input: "~~~bash\necho \"hello\"\n~~~\n", - want: []struct { - kind internal.BlockKind - content string - }{ - {internal.BlockKindFencedCode, "~~~bash\necho \"hello\"\n~~~\n"}, + name: "Text/paragraph/line-break", + input: joinLines( + "text", + "and more text ", + "after line break", + ), + want: []internal.Block{ + text("", "text\nand more text \nafter line break", false), }, }, { - name: "longer backtick fence", - input: "````bash\necho \"hello\"\n````\n", - want: []struct { - kind internal.BlockKind - content string - }{ - {internal.BlockKindFencedCode, "````bash\necho \"hello\"\n````\n"}, + name: "Text/indented-code-as-text", + input: joinLines( + " echo \"hello, \"", + " echo \"world\"", + ), + want: []internal.Block{ + text("", " echo \"hello, \"\n echo \"world\"", false), }, }, { - name: "quad fenced verbatim block containing litdoc source", - input: "````md\n```bash | litdoc\necho \"hello\"\n```\n````", - want: []struct { - kind internal.BlockKind - content string - }{ - { - internal.BlockKindFencedCode, - "````md\n```bash | litdoc\necho \"hello\"\n```\n````", - }, + name: "Text/inline-code-span", + input: "text `echo \"hello, \" text`", + want: []internal.Block{ + text("", "text `echo \"hello, \" text`", false), }, }, { - name: "indented code block", - input: " echo \"hello\"\n", - want: []struct { - kind internal.BlockKind - content string - }{ - {internal.BlockKindText, " echo \"hello\"\n"}, + name: "Text/html-block-not-comment", + input: "
\nhello\n
\n", + want: []internal.Block{ + text("", "
\nhello\n
\n", false), }, }, { - name: "html comment block", - input: "\n", - want: []struct { - kind internal.BlockKind - content string - }{ - {internal.BlockKindHTMLComment, "\n"}, + name: "Text/blockquote/multiline", + input: joinLines( + "> hello, ", + "> world", + "> ", + "> again", + ), + want: []internal.Block{ + text("> ", "hello, \n", false), + text("> ", "world\n", false), + text("> ", "\n", false), + text("> ", "again", false), }, }, + + // FencedCode — top-level (CommonMark §4.5). { - name: "html block is not a comment", - input: "
\nhello\n
\n", - want: []struct { - kind internal.BlockKind - content string - }{ - {internal.BlockKindText, "
\nhello\n
\n"}, + name: "FencedCode/top-level/backtick", + input: joinLines( + "```bash", + "echo \"hello\"", + "```", + "", + ), + want: []internal.Block{ + code("", "```bash\necho \"hello\"\n```\n", false), + }, + }, + { + name: "FencedCode/top-level/backtick-no-trailing-newline", + input: joinLines( + "```bash", + "echo \"hello\"", + "```", + ), + want: []internal.Block{ + code("", "```bash\necho \"hello\"\n```", false), + }, + }, + { + name: "FencedCode/top-level/tilde", + input: joinLines( + "~~~bash", + "echo \"hello\"", + "~~~", + ), + want: []internal.Block{ + code("", "~~~bash\necho \"hello\"\n~~~", false), + }, + }, + { + name: "FencedCode/top-level/longer-fence-around-fence", + input: joinLines( + "````md", + "```bash", + "echo \"hello\"", + "```", + "````", + ), + want: []internal.Block{ + code("", "````md\n```bash\necho \"hello\"\n```\n````", false), + }, + }, + { + name: "FencedCode/top-level/no-info-string", + input: joinLines( + "```", + "echo \"hello\"", + "```", + ), + want: []internal.Block{ + code("", "```\necho \"hello\"\n```", false), + }, + }, + { + name: "FencedCode/top-level/empty", + input: joinLines( + "```bash", + "```", + ), + want: []internal.Block{ + code("", "```bash\n```", false), + }, + }, + { + name: "FencedCode/top-level/consecutive", + input: joinLines( + "```bash", + "echo a", + "```", + "", + "```bash", + "echo b", + "```", + ), + want: []internal.Block{ + code("", "```bash\necho a\n```\n", false), + text("", "\n", false), + code("", "```bash\necho b\n```", false), + }, + }, + { + name: "FencedCode/top-level/indented-1-space", + input: joinLines( + " ```bash", + " echo \"hello\"", + " ```", + ), + want: []internal.Block{ + code(" ", "```bash\necho \"hello\"\n```", false), }, }, { - name: "mixed content", - input: "# Title\n\n```go\nfmt.Println()\n```\n\n\n", - want: []struct { - kind internal.BlockKind - content string - }{ - {internal.BlockKindText, "# Title\n"}, - {internal.BlockKindText, "\n"}, - {internal.BlockKindFencedCode, "```go\nfmt.Println()\n```\n"}, - {internal.BlockKindText, "\n"}, - {internal.BlockKindHTMLComment, "\n"}, + name: "FencedCode/top-level/indented-3-spaces", + input: joinLines( + " ```bash", + " echo \"hello\"", + " ```", + ), + want: []internal.Block{ + code(" ", "```bash\necho \"hello\"\n```", false), + }, + }, + // FencedCode — inside containers. + { + name: "FencedCode/blockquote", + input: joinLines( + "> text", + "> ```bash", + "> echo \"hello\"", + "> ```", + "> text", + ), + want: []internal.Block{ + text("> ", "text\n", false), + code("> ", "```bash\necho \"hello\"\n```\n", false), + text("> ", "text", false), + }, + }, + { + name: "FencedCode/blockquote/nested", + input: joinLines( + "> text", + "> > ```bash", + "> > echo \"hello\"", + "> > ```", + "> text", + ), + want: []internal.Block{ + text("> ", "text\n", false), + code("> > ", "```bash\necho \"hello\"\n```\n", false), + text("> ", "text", false), + }, + }, + { + name: "FencedCode/list/dash", + input: joinLines( + "- text", + "- ```bash", + " echo \"hello\"", + " ```", + "- text", + ), + want: []internal.Block{ + text("- ", "text\n", false), + code("- ", "```bash\necho \"hello\"\n```\n", false), + text("- ", "text", false), + }, + }, + { + name: "FencedCode/list/dash/nested", + input: joinLines( + "- text", + " - ```bash", + " echo \"hello\"", + " ```", + "- text", + ), + want: []internal.Block{ + text("- ", "text\n", false), + code(" - ", "```bash\necho \"hello\"\n```\n", false), + text("- ", "text", false), + }, + }, + { + name: "FencedCode/list/plus-tilde-fence", + input: joinLines( + "+ text", + "+ ~~~bash", + " echo \"hello\"", + " ~~~", + "+ text", + ), + want: []internal.Block{ + text("+ ", "text\n", false), + code("+ ", "~~~bash\necho \"hello\"\n~~~\n", false), + text("+ ", "text", false), + }, + }, + { + name: "FencedCode/list/ordered", + input: joinLines( + "1. text", + "2. ```bash", + " echo \"hello\"", + " ```", + "3. text", + ), + want: []internal.Block{ + text("1. ", "text\n", false), + code("2. ", "```bash\necho \"hello\"\n```\n", false), + text("3. ", "text", false), + }, + }, + { + name: "FencedCode/list/ordered/nested", + input: joinLines( + "1. text", + " 1. ```bash", + " echo \"hello\"", + " ```", + "2. text", + ), + want: []internal.Block{ + text("1. ", "text\n", false), + code(" 1. ", "```bash\necho \"hello\"\n```\n", false), + text("2. ", "text", false), + }, + }, + { + name: "FencedCode/list/blockquote", + input: joinLines( + "- text", + " > ```bash", + " > echo \"hello\"", + " > ```", + "- text", + ), + want: []internal.Block{ + text("- ", "text\n", false), + code(" > ", "```bash\necho \"hello\"\n```\n", false), + text("- ", "text", false), + }, + }, + { + name: "FencedCode/blockquote/list-nested", + input: joinLines( + "> text", + "> - item", + "> - ```bash", + "> echo \"hello\"", + "> ```", + "> text", + ), + want: []internal.Block{ + text("> ", "text\n", false), + text("> - ", "item\n", false), + code("> - ", "```bash\necho \"hello\"\n```\n", false), + text("> ", "text", false), + }, + }, + { + name: "FencedCode/blockquote/list-ordered", + input: joinLines( + "> text", + "> 1. ```bash", + "> echo \"hello\"", + "> ```", + "> text", + ), + want: []internal.Block{ + text("> ", "text\n", false), + code("> 1. ", "```bash\necho \"hello\"\n```\n", false), + text("> ", "text", false), + }, + }, + + // HTMLComment — top-level (CommonMark §4.6 type 2). + { + name: "HTMLComment/top-level/inline", + input: "text text", + want: []internal.Block{ + text("", "text ", false), + cmnt("", "", true), + text("", " text", true), + }, + }, + { + name: "HTMLComment/top-level/inline-multiple", + input: "text text", + want: []internal.Block{ + text("", "text ", false), + cmnt("", "", true), + cmnt("", "", true), + text("", " text", true), + }, + }, + { + name: "HTMLComment/top-level/inline-at-line-end", + input: "text ", + want: []internal.Block{ + text("", "text ", false), + cmnt("", "", true), + }, + }, + { + name: "HTMLComment/top-level/block", + input: joinLines( + "text", + "", + "text", + ), + want: []internal.Block{ + text("", "text\n", false), + cmnt("", "\n", false), + text("", "text", false), + }, + }, + { + name: "HTMLComment/top-level/block-empty", + input: "\n", + want: []internal.Block{ + cmnt("", "\n", false), + }, + }, + { + name: "HTMLComment/top-level/block-consecutive", + input: joinLines( + "", + "", + "", + "", + ), + want: []internal.Block{ + cmnt("", "\n", false), + text("", "\n", false), + cmnt("", "\n", false), + }, + }, + { + name: "HTMLComment/top-level/in-heading-line", + input: "# Title \n", + want: []internal.Block{ + text("", "# Title ", false), + cmnt("", "", true), + text("", "\n", true), + }, + }, + + // HTMLComment — inside containers. + { + name: "HTMLComment/blockquote/inline-multiple", + input: "> text text", + want: []internal.Block{ + text("> ", "text ", false), + cmnt("> ", "", true), + cmnt("> ", "", true), + text("> ", " text", true), + }, + }, + { + name: "HTMLComment/blockquote/nested/block", + input: joinLines( + "> text", + "> > ", + "> text", + ), + want: []internal.Block{ + text("> ", "text\n", false), + cmnt("> > ", "\n", false), + text("> ", "text", false), + }, + }, + { + name: "HTMLComment/list/dash/inline-multiple", + input: "- text text", + want: []internal.Block{ + text("- ", "text ", false), + cmnt("- ", "", true), + cmnt("- ", "", true), + text("- ", " text", true), + }, + }, + { + name: "HTMLComment/list/dash/nested/inline-multiple", + input: " - text text", + want: []internal.Block{ + text(" - ", "text ", false), + cmnt(" - ", "", true), + cmnt(" - ", "", true), + text(" - ", " text", true), + }, + }, + { + name: "HTMLComment/list/dash/block", + input: joinLines( + "- text", + "- ", + "- text", + ), + want: []internal.Block{ + text("- ", "text\n", false), + cmnt("- ", "\n", false), + text("- ", "text", false), + }, + }, + { + name: "HTMLComment/list/dash/nested/block", + input: joinLines( + "- text", + " - ", + "- text", + ), + want: []internal.Block{ + text("- ", "text\n", false), + cmnt(" - ", "\n", false), + text("- ", "text", false), + }, + }, + { + name: "HTMLComment/list/star/block", + input: joinLines( + "* text", + "* ", + "* text", + ), + want: []internal.Block{ + text("* ", "text\n", false), + cmnt("* ", "\n", false), + text("* ", "text", false), + }, + }, + { + name: "HTMLComment/list/ordered/block", + input: joinLines( + "1. text", + "2. ", + "3. text", + ), + want: []internal.Block{ + text("1. ", "text\n", false), + cmnt("2. ", "\n", false), + text("3. ", "text", false), + }, + }, + { + name: "HTMLComment/blockquote/list/inline-multiple", + input: "> - text text", + want: []internal.Block{ + text("> - ", "text ", false), + cmnt("> - ", "", true), + cmnt("> - ", "", true), + text("> - ", " text", true), + }, + }, + { + name: "HTMLComment/blockquote/list/block", + input: joinLines( + "> text", + "> - ", + "> text", + ), + want: []internal.Block{ + text("> ", "text\n", false), + cmnt("> - ", "\n", false), + text("> ", "text", false), + }, + }, + { + name: "HTMLComment/blockquote/nested/list/block", + input: joinLines( + "> text", + "> > - ", + "> text", + ), + want: []internal.Block{ + text("> ", "text\n", false), + cmnt("> > - ", "\n", false), + text("> ", "text", false), + }, + }, + { + name: "HTMLComment/list/ordered/blockquote/block", + input: joinLines( + "1. text", + " > ", + "2. text", + ), + want: []internal.Block{ + text("1. ", "text\n", false), + cmnt(" > ", "\n", false), + text("2. ", "text", false), }, }, } @@ -147,23 +633,125 @@ func TestMakeBlocksFromMarkdown(t *testing.T) { // then require.NoError(t, err) - require.Len(t, blocks, len(tt.want)) + require.Len( + t, + blocks, + len(tt.want), + "expected %d blocks, got %d", + len(tt.want), + len(blocks), + ) for i, w := range tt.want { + assert.Equal(t, w.Kind(), blocks[i].Kind(), "block[%d] kind", i) + assert.Equal(t, w.Indent(), blocks[i].Indent(), "block[%d] indent", i) + assert.Equal(t, w.Content(), blocks[i].Content(), "block[%d] content", i) assert.Equal( t, - w.kind, - blocks[i].Kind(), - "block[%d] kind", - i, - ) - assert.Equal( - t, - w.content, - string(blocks[i].Content()), - "block[%d] content", + w.Continuation(), + blocks[i].Continuation(), + "block[%d] continuation", i, ) } }) } + + invalidTests := []struct { + name string + input []byte + wantErrText string + }{ + { + name: "invalid-utf8", + input: []byte{'#', ' ', 0xe9, '\n'}, + wantErrText: "not valid UTF-8", + }, + { + name: "FencedCode/top-level/unclosed-runs-to-eof", + input: []byte(joinLines( + "```bash", + "echo \"hello\"", + )), + wantErrText: "unclosed fenced code block", + }, + { + name: "FencedCode/top-level/opening-line-runs-to-eof", + input: []byte("```bash\n"), + wantErrText: "unclosed fenced code block", + }, + { + name: "FencedCode/top-level/wrong-closing-fence-char", + input: []byte(joinLines( + "```bash", + "echo \"hello\"", + "~~~", + )), + wantErrText: "unclosed fenced code block", + }, + { + name: "FencedCode/top-level/short-closing-fence", + input: []byte(joinLines( + "````bash", + "echo \"hello\"", + "```", + )), + wantErrText: "unclosed fenced code block", + }, + { + name: "FencedCode/top-level/closing-fence-with-trailing-text", + input: []byte(joinLines( + "```bash", + "echo \"hello\"", + "``` nope", + )), + wantErrText: "unclosed fenced code block", + }, + { + name: "FencedCode/blockquote/unclosed-runs-to-eof", + input: []byte(joinLines( + "> ```bash", + "> echo \"hello\"", + )), + wantErrText: "unclosed fenced code block", + }, + { + name: "HTMLComment/top-level/unclosed-block-runs-to-eof", + input: []byte(joinLines( + "", + "", + ), + false, ), want: internal.InfoString{Lang: "bash"}, }, { - name: "fenced code with litdoc", - block: internal.MakeBlockFromRaw( - internal.BlockKindFencedCode, - []byte("```bash | litdoc\necho hello\n```\n"), + name: "html comment/with-litdoc", + block: cmnt( + "", + joinLines( + "", + "", + ), + false, ), - want: internal.InfoString{Lang: "bash", IsLitdoc: true}, + want: internal.InfoString{Lang: "bash", Litdoc: true}, }, { - name: "html comment with litdoc", - block: internal.MakeBlockFromRaw( - internal.BlockKindHTMLComment, - []byte("\n"), + name: "html comment/unsupported-litdoc-language", + block: cmnt( + "", + joinLines( + "", + "", + ), + false, ), - want: internal.InfoString{Lang: "bash", IsLitdoc: true}, + want: internal.InfoString{Lang: "go", Litdoc: true}, }, } @@ -95,38 +183,15 @@ func TestParseInfoString(t *testing.T) { } } -func TestMakeBashCellFromRaw(t *testing.T) { - // given - code := "```bash\necho hello\n```\n" - - // when - gotCell := internal.MakeBashCellFromRaw(code, "") - - // then - got, err := gotCell.Render() - require.NoError(t, err) - assert.Equal(t, code, got) -} - -func TestBashCellExecute(t *testing.T) { - // given - fencedCode := "```bash\necho hello\n```\n" - cell := internal.MakeBashCellFromRaw(fencedCode, "") - - // when - gotCell, err := cell.Execute() - - // then - require.NoError(t, err) - rendered, err := gotCell.Render() - require.NoError(t, err) - assert.Equal(t, fencedCode, rendered) -} - -func TestBashCellRender(t *testing.T) { +func TestBashCell(t *testing.T) { t.Run("without output", func(t *testing.T) { // given - code := "```bash\necho hello\n```\n" + code := joinLines( + "```bash", + "echo hello", + "```", + "", + ) cell := internal.MakeBashCellFromRaw(code, "") // when @@ -139,17 +204,40 @@ func TestBashCellRender(t *testing.T) { t.Run("with output", func(t *testing.T) { // given - fencedCode := "```bash\necho hello\n```\n" - cell := internal.MakeBashCellFromRaw(fencedCode, "") - executed, err := cell.Execute() + fencedCode := joinLines( + "```bash", + "echo hello", + "```", + ) + output := "hello" + cell := internal.MakeBashCellFromRaw(fencedCode, output) + + // when + gotContent, err := cell.Render() + + // then require.NoError(t, err) + assert.Equal(t, fencedCode+"\n"+output, gotContent) + }) + + t.Run("executes to itself", func(t *testing.T) { + // given + fencedCode := joinLines( + "```bash", + "echo hello", + "```", + "", + ) + cell := internal.MakeBashCellFromRaw(fencedCode, "") // when - gotContent, err := executed.Render() + gotCell, err := cell.Execute() // then require.NoError(t, err) - assert.Equal(t, fencedCode, gotContent) + rendered, err := gotCell.Render() + require.NoError(t, err) + assert.Equal(t, fencedCode, rendered) }) } @@ -218,101 +306,293 @@ func TestCompose(t *testing.T) { } func TestClassify(t *testing.T) { - textBlock := func(content string) internal.Block { - return internal.MakeBlockFromRaw(internal.BlockKindText, []byte(content)) + type wantCell struct { + kind string + rendered string } - bashLitdocBlock := func(content string) internal.Block { - return internal.MakeBlockFromRaw(internal.BlockKindFencedCode, []byte(content)) - } - - t.Run("single text block becomes StaticCell", func(t *testing.T) { - // given - blocks := []internal.Block{textBlock("hello")} - - // when - cells, err := internal.Classify(blocks) - - // then - require.NoError(t, err) - require.Len(t, cells, 1) - _, ok := cells[0].(internal.StaticCell) - require.True(t, ok, "expected StaticCell, got %T", cells[0]) - got, err := cells[0].Render() - require.NoError(t, err) - assert.Equal(t, "hello", got) - }) - - t.Run("litdoc bash block becomes BashCell", func(t *testing.T) { - // given - code := "```bash | litdoc\necho hello\n```\n" - blocks := []internal.Block{bashLitdocBlock(code)} - - // when - cells, err := internal.Classify(blocks) - // then - require.NoError(t, err) - require.Len(t, cells, 1) - _, ok := cells[0].(internal.BashCell) - require.True(t, ok, "expected BashCell, got %T", cells[0]) - rendered, err := cells[0].Render() - require.NoError(t, err) - assert.Equal(t, code, rendered) - }) - - t.Run("mixed block types are each classified independently", func(t *testing.T) { - // given - code := "```bash | litdoc\necho hello\n```\n" - blocks := []internal.Block{ - textBlock("before"), - bashLitdocBlock(code), - textBlock("after"), - } - - // when - cells, err := internal.Classify(blocks) - - // then - require.NoError(t, err) - require.Len(t, cells, 3) - rendered0, err := cells[0].Render() - require.NoError(t, err) - assert.Equal(t, "before", rendered0) - _, ok := cells[1].(internal.BashCell) - assert.True(t, ok, "expected BashCell, got %T", cells[1]) - rendered2, err := cells[2].Render() - require.NoError(t, err) - assert.Equal(t, "after", rendered2) - }) - - t.Run("non-litdoc fenced code block becomes StaticCell", func(t *testing.T) { - // given - code := "```bash\necho hello\n```\n" - blocks := []internal.Block{ - internal.MakeBlockFromRaw(internal.BlockKindFencedCode, []byte(code)), - } - - // when - cells, err := internal.Classify(blocks) - - // then - require.NoError(t, err) - require.Len(t, cells, 1) - rendered, err := cells[0].Render() - require.NoError(t, err) - assert.Equal(t, code, rendered) - }) - - t.Run("litdoc block with unsupported language", func(t *testing.T) { - // given - blocks := []internal.Block{ - bashLitdocBlock("```go | litdoc\nfmt.Println()\n```\n"), - } + tests := []struct { + name string + blocks []internal.Block + want []wantCell + wantErrText string + }{ + { + name: "Text/top-level", + blocks: []internal.Block{ + text("", "hello", false), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "hello"}, + }, + }, + { + name: "Text/blockquote/multiline", + blocks: []internal.Block{ + text("> ", "hello, \nworld\n", false), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "> hello, \n> world\n"}, + }, + }, + { + name: "Text/list/dash", + blocks: []internal.Block{ + text("- ", "text\ncontinued", false), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "- text\n continued"}, + }, + }, + { + name: "Text/list/ordered", + blocks: []internal.Block{ + text("2. ", "text\ncontinued", false), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "2. text\n continued"}, + }, + }, + { + name: "Text/blockquote/list", + blocks: []internal.Block{ + text("> - ", "text\ncontinued", false), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "> - text\n> continued"}, + }, + }, + { + name: "Text/litdoc-looking-content", + blocks: []internal.Block{ + text("", joinLines( + "", + "", + ), false), + }, + want: []wantCell{ + { + kind: "StaticCell", rendered: joinLines( + "", + "", + ), + }, + }, + }, + { + name: "FencedCode/non-litdoc/top-level", + blocks: []internal.Block{ + code("", joinLines( + "```bash", + "echo hello", + "```", + "", + ), false), + }, + want: []wantCell{ + { + kind: "StaticCell", rendered: joinLines( + "```bash", + "echo hello", + "```", + "", + ), + }, + }, + }, + { + name: "FencedCode/non-litdoc/list", + blocks: []internal.Block{ + code("- ", joinLines( + "```bash", + "echo hello", + "```", + "", + ), false), + }, + want: []wantCell{ + { + kind: "StaticCell", rendered: joinLines( + "- ```bash", + " echo hello", + " ```", + "", + ), + }, + }, + }, + { + name: "FencedCode/litdoc/bash", + blocks: []internal.Block{ + code("", joinLines( + "```bash | litdoc", + "echo hello", + "```", + "", + ), false), + }, + want: []wantCell{ + { + kind: "BashCell", rendered: joinLines( + "```bash | litdoc", + "echo hello", + "```", + "", + ), + }, + }, + }, + { + name: "FencedCode/litdoc/unsupported-language", + blocks: []internal.Block{ + code("", joinLines( + "```go | litdoc", + "fmt.Println()", + "```", + "", + ), false), + }, + wantErrText: "unsupported language", + }, + { + name: "HTMLComment/block/list", + blocks: []internal.Block{ + cmnt("- ", joinLines( + "", + "", + ), false), + }, + want: []wantCell{ + { + kind: "StaticCell", rendered: joinLines( + "- ", + "", + ), + }, + }, + }, + { + name: "HTMLComment/litdoc/bash", + blocks: []internal.Block{ + cmnt("", joinLines( + "", + "", + ), false), + }, + want: []wantCell{ + { + kind: "BashCell", rendered: joinLines( + "", + "", + ), + }, + }, + }, + { + name: "HTMLComment/inline-continuation/list", + blocks: []internal.Block{ + text("- ", "text ", false), + cmnt("- ", "", true), + cmnt("- ", "", true), + text("- ", " text", true), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "- text "}, + {kind: "StaticCell", rendered: ""}, + {kind: "StaticCell", rendered: ""}, + {kind: "StaticCell", rendered: " text"}, + }, + }, + { + name: "HTMLComment/inline-continuation/blockquote-list", + blocks: []internal.Block{ + text("> - ", "text ", false), + cmnt("> - ", "", true), + cmnt("> - ", "", true), + text("> - ", " text", true), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "> - text "}, + {kind: "StaticCell", rendered: ""}, + {kind: "StaticCell", rendered: ""}, + {kind: "StaticCell", rendered: " text"}, + }, + }, + { + name: "Mixed/independent-blocks", + blocks: []internal.Block{ + text("", "before", false), + code("", joinLines( + "```bash | litdoc", + "echo hello", + "```", + "", + ), false), + text("", "after", false), + }, + want: []wantCell{ + {kind: "StaticCell", rendered: "before"}, + { + kind: "BashCell", rendered: joinLines( + "```bash | litdoc", + "echo hello", + "```", + "", + ), + }, + {kind: "StaticCell", rendered: "after"}, + }, + }, + } - // when - _, err := internal.Classify(blocks) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // when + cells, err := internal.Classify(tt.blocks) + + // then + if tt.wantErrText != "" { + require.ErrorContains(t, err, tt.wantErrText) + return + } + + require.NoError(t, err) + require.Len(t, cells, len(tt.want)) + wantComposed := "" + for i, w := range tt.want { + assert.Equal(t, w.kind, cellKind(cells[i]), "cell[%d] kind", i) + rendered, err := cells[i].Render() + require.NoError(t, err, "cell[%d] render", i) + assert.Equal(t, w.rendered, rendered, "cell[%d] rendered", i) + wantComposed += w.rendered + } + + gotComposed, err := internal.Compose(cells) + require.NoError(t, err) + assert.Equal(t, wantComposed, gotComposed) + }) + } +} - // then - require.ErrorContains(t, err, "unsupported language") - }) +func cellKind(cell internal.Cell) string { + switch cell.(type) { + case internal.StaticCell: + return "StaticCell" + case internal.BashCell: + return "BashCell" + default: + return "unknown" + } } diff --git a/internal/helpers_test.go b/internal/helpers_test.go new file mode 100644 index 0000000..c593e01 --- /dev/null +++ b/internal/helpers_test.go @@ -0,0 +1,23 @@ +package internal_test + +import ( + "strings" + + "litdoc/internal" +) + +func text(indent, content string, continuation bool) internal.Block { + return internal.MakeBlockFromRaw(internal.BlockKindText, content, indent, continuation) +} + +func code(indent, content string, continuation bool) internal.Block { + return internal.MakeBlockFromRaw(internal.BlockKindFencedCode, content, indent, continuation) +} + +func cmnt(indent, content string, continuation bool) internal.Block { + return internal.MakeBlockFromRaw(internal.BlockKindHTMLComment, content, indent, continuation) +} + +func joinLines(lines ...string) string { + return strings.Join(lines, "\n") +}