From eaff79961d3e6bc23cd23e0689cb8556d5d19a87 Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Wed, 18 Mar 2026 10:29:09 -0600 Subject: [PATCH 1/5] ci: linter & fmt config to follow tom's code patterns --- Cargo.toml | 87 +++++++++++++++++++ ...lt-output-with-pretty-json-array-option.md | 6 +- dprint.json | 22 ++--- 3 files changed, 94 insertions(+), 21 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6c249e7..0b82c27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,3 +14,90 @@ repository = "https://github.com/tomdavidson/slash-parser" serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "2" + +[workspace.lints.rust] +# Safety baseline: no unsafe unless explicitly justified +unsafe_code = "forbid" +# Debug is cheap observability; helpful at boundaries +missing_debug_implementations = "warn" +# Keep APIs intentionally small; helps encapsulation +unreachable_pub = "warn" + +[workspace.lints.clippy] +# ── Baseline coverage (from lang-rust.md) ─────────────────────── +cargo = { level = "warn", priority = -1 } +# Broad coverage; per-lint overrides below win +pedantic = { level = "warn", priority = -1 } +# Feature modules + re-exports make this noisy +module_name_repetitions = "allow" +# You use #[must_use] intentionally, not everywhere +must_use_candidate = "allow" +# Reduce doc-noise; rely on clear types + thiserror +missing_errors_doc = "allow" +# You deny panics anyway (below) +missing_panics_doc = "allow" + +# ── String & borrowing conventions (lang-rust.md) ─────────────── +# Prefer &str over &String; &[T] over &Vec Date: Wed, 18 Mar 2026 10:34:11 -0600 Subject: [PATCH 2/5] add application & domain layers --- parser/src/application/command_accumulate.rs | 249 ++++++++++ parser/src/application/command_finalize.rs | 268 +++++++++++ parser/src/application/document_parse.rs | 452 +++++++++++++++++++ parser/src/application/line_classify.rs | 281 ++++++++++++ parser/src/application/mod.rs | 10 + parser/src/application/tests/mod.rs | 1 + parser/src/application/tests/proptest.rs | 62 +++ parser/src/application/text_collect.rs | 170 +++++++ parser/src/lib.rs | 5 + parser/tests/mod.rs | 21 + 10 files changed, 1519 insertions(+) create mode 100644 parser/src/application/command_accumulate.rs create mode 100644 parser/src/application/command_finalize.rs create mode 100644 parser/src/application/document_parse.rs create mode 100644 parser/src/application/line_classify.rs create mode 100644 parser/src/application/mod.rs create mode 100644 parser/src/application/tests/mod.rs create mode 100644 parser/src/application/tests/proptest.rs create mode 100644 parser/src/application/text_collect.rs create mode 100644 parser/src/lib.rs create mode 100644 parser/tests/mod.rs diff --git a/parser/src/application/command_accumulate.rs b/parser/src/application/command_accumulate.rs new file mode 100644 index 0000000..d89f76f --- /dev/null +++ b/parser/src/application/command_accumulate.rs @@ -0,0 +1,249 @@ +use super::line_classify::{CommandHeader, LineKind, classify_line}; +use crate::domain::ArgumentMode; + +/// Result of offering a line to an in-progress command. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AcceptResult { + Consumed, + Completed, + Rejected, +} + +/// In-progress command being assembled from consecutive lines. +#[derive(Debug, Clone)] +pub struct PendingCommand { + pub name: String, + pub raw_header: String, + pub header_text: String, + pub mode: ArgumentMode, + pub fence_lang: Option, + pub start_line: usize, + pub end_line: usize, + pub payload_lines: Vec, + pub is_open: bool, +} + +pub fn start_command(header: CommandHeader, line_index: usize) -> PendingCommand { + let initial_lines = if header.header_text.is_empty() { + vec![] + } else { + vec![header.header_text.clone()] + }; + + let (payload_lines, is_open) = match &header.mode { + ArgumentMode::SingleLine => (initial_lines, false), + ArgumentMode::Continuation => (initial_lines, true), + ArgumentMode::Fence => (vec![], true), + }; + + PendingCommand { + name: header.name, + raw_header: header.raw, + header_text: header.header_text, + mode: header.mode, + fence_lang: header.fence_lang, + start_line: line_index, + end_line: line_index, + payload_lines, + is_open, + } +} + +pub fn accept_line( + cmd: PendingCommand, + line_index: usize, + line: &str, +) -> (PendingCommand, AcceptResult) { + if !cmd.is_open { + return (cmd, AcceptResult::Rejected); + } + + match cmd.mode { + ArgumentMode::SingleLine => (cmd, AcceptResult::Rejected), + ArgumentMode::Continuation => accept_continuation(cmd, line_index, line), + ArgumentMode::Fence => accept_fence(cmd, line_index, line), + } +} + +fn accept_continuation( + mut cmd: PendingCommand, + line_index: usize, + line: &str, +) -> (PendingCommand, AcceptResult) { + if line.trim().is_empty() { + cmd.is_open = false; + return (cmd, AcceptResult::Rejected); + } + + if matches!(classify_line(line), LineKind::Command(_)) { + cmd.is_open = false; + return (cmd, AcceptResult::Rejected); + } + + if line.trim_end().ends_with('\\') { + let content = line.trim_end().trim_end_matches('\\').trim_end(); + cmd.payload_lines.push(content.to_string()); + cmd.end_line = line_index; + (cmd, AcceptResult::Consumed) + } else { + cmd.payload_lines.push(line.to_string()); + cmd.end_line = line_index; + cmd.is_open = false; + (cmd, AcceptResult::Completed) + } +} + +fn accept_fence( + mut cmd: PendingCommand, + line_index: usize, + line: &str, +) -> (PendingCommand, AcceptResult) { + if line.trim_start().starts_with("```") { + cmd.end_line = line_index; + cmd.is_open = false; + (cmd, AcceptResult::Completed) + } else { + cmd.end_line = line_index; + cmd.payload_lines.push(line.to_string()); + (cmd, AcceptResult::Consumed) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn header_from(line: &str) -> CommandHeader { + match classify_line(line) { + LineKind::Command(h) => h, + _ => panic!("expected command header"), + } + } + + #[test] + fn single_line_is_immediately_closed() { + let cmd = start_command(header_from("/help"), 0); + assert!(!cmd.is_open); + } + + #[test] + fn fence_stays_open_until_closing_ticks() { + let cmd = start_command(header_from("/code ```rust"), 0); + assert!(cmd.is_open); + + let (cmd, res) = accept_line(cmd, 1, "fn main() {}"); + assert_eq!(res, AcceptResult::Consumed); + assert!(cmd.is_open); + + let (cmd, res) = accept_line(cmd, 2, "```"); + assert_eq!(res, AcceptResult::Completed); + assert!(!cmd.is_open); + assert_eq!(cmd.payload_lines, vec!["fn main() {}"]); + } + + #[test] + fn continuation_ends_on_line_without_backslash() { + let cmd = start_command(header_from("/cmd first \\"), 0); + assert!(cmd.is_open); + + let (cmd, res) = accept_line(cmd, 1, "second \\"); + assert_eq!(res, AcceptResult::Consumed); + + let (cmd, res) = accept_line(cmd, 2, "last line"); + assert_eq!(res, AcceptResult::Completed); + assert!(!cmd.is_open); + } + + #[test] + fn continuation_rejects_blank_line() { + let cmd = start_command(header_from("/cmd start \\"), 0); + let (cmd, res) = accept_line(cmd, 1, ""); + assert_eq!(res, AcceptResult::Rejected); + assert!(!cmd.is_open); + } + + #[test] + fn continuation_rejects_new_command() { + let cmd = start_command(header_from("/cmd start \\"), 0); + let (cmd, res) = accept_line(cmd, 1, "/other"); + assert_eq!(res, AcceptResult::Rejected); + assert!(!cmd.is_open); + } + + #[test] + fn continuation_two_lines_produces_continuation_mode() { + let header = header_from("/mcp call_tool read_file \\"); + let cmd = start_command(header, 0); + + assert_eq!(cmd.mode, ArgumentMode::Continuation); + assert!(cmd.is_open); + } + + #[test] + fn continuation_header_strips_trailing_marker() { + let header = header_from("/mcp call_tool read_file \\"); + let cmd = start_command(header, 0); + + assert_eq!(cmd.header_text, "call_tool read_file"); + assert_eq!(cmd.payload_lines, vec!["call_tool read_file"]); + } + + #[test] + fn continuation_payload_preserves_newlines() { + let cmd = start_command(header_from("/mcp call_tool read_file \\"), 0); + + let line1 = r#"{"path": "src/index.ts"}"#; + let (cmd, res) = accept_line(cmd, 1, line1); + assert_eq!(res, AcceptResult::Completed); + assert!(!cmd.is_open); + + assert_eq!( + cmd.payload_lines, + vec!["call_tool read_file".to_string(), line1.to_string()] + ); + + assert_eq!(cmd.start_line, 0); + assert_eq!(cmd.end_line, 1); + } + + #[test] + fn continuation_three_lines() { + let cmd = start_command(header_from("/cmd first \\"), 0); + + let (cmd, res) = accept_line(cmd, 1, "second \\"); + assert_eq!(res, AcceptResult::Consumed); + let (cmd, res) = accept_line(cmd, 2, "third"); + assert_eq!(res, AcceptResult::Completed); + assert!(!cmd.is_open); + + assert_eq!( + cmd.payload_lines, + vec![ + "first".to_string(), + "second".to_string(), + "third".to_string() + ] + ); + } + + #[test] + fn continuation_range_spans_all_lines() { + let cmd = start_command(header_from("/cmd first \\"), 0); + + let (cmd, _) = accept_line(cmd, 1, "second \\"); + let (cmd, _) = accept_line(cmd, 2, "third"); + + assert_eq!(cmd.start_line, 0); + assert_eq!(cmd.end_line, 2); + } + + #[test] + fn slash_at_end_without_space_is_not_continuation() { + let header = header_from("/path /var/log/"); + let cmd = start_command(header, 0); + + assert_eq!(cmd.mode, ArgumentMode::SingleLine); + assert!(!cmd.is_open); + assert_eq!(cmd.payload_lines, vec!["/var/log/"]); + } +} diff --git a/parser/src/application/command_finalize.rs b/parser/src/application/command_finalize.rs new file mode 100644 index 0000000..7c3ca49 --- /dev/null +++ b/parser/src/application/command_finalize.rs @@ -0,0 +1,268 @@ +use super::command_accumulate::PendingCommand; +use crate::domain::{ArgumentMode, Command, CommandArguments, LineRange, ParseWarning}; + +#[derive(Debug)] +pub struct FinalizedCommand { + pub command: Command, + pub warnings: Vec, +} + +pub fn finalize_command(pending: PendingCommand) -> FinalizedCommand { + let mut warnings = Vec::new(); + + match pending.mode { + ArgumentMode::Fence if pending.is_open => { + warnings.push(ParseWarning::UnclosedFence { + start_line: pending.start_line, + }); + } + ArgumentMode::Continuation if pending.is_open => { + warnings.push(ParseWarning::UnclosedContinuation { + start_line: pending.start_line, + }); + } + _ => {} + } + + let payload = pending.payload_lines.join("\n"); + + let command = Command { + name: pending.name, + raw: pending.raw_header, + range: LineRange { + start_line: pending.start_line, + end_line: pending.end_line, + }, + arguments: CommandArguments { + header: pending.header_text, + mode: pending.mode, + fence_lang: pending.fence_lang, + payload, + }, + }; + + FinalizedCommand { command, warnings } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + application::{ + command_accumulate::{accept_line, start_command}, + line_classify::{CommandHeader, LineKind, classify_line}, + }, + domain::ArgumentMode, + }; + + fn header_from(line: &str) -> CommandHeader { + match classify_line(line) { + LineKind::Command(h) => h, + _ => panic!("expected command header"), + } + } + + // --- Warning tests --- + + #[test] + fn unclosed_fence_at_eof_warns() { + let cmd = start_command(header_from("/cmd ```rust"), 0); + let (cmd, _) = accept_line(cmd, 1, "line1"); + let (cmd, _) = accept_line(cmd, 2, "line2"); + + let result = finalize_command(cmd); + + assert_eq!(result.warnings.len(), 1); + assert!(matches!( + result.warnings[0], + ParseWarning::UnclosedFence { start_line: 0 } + )); + } + + #[test] + fn unclosed_continuation_at_eof_warns() { + let cmd = start_command(header_from("/cmd first \\"), 0); + + let result = finalize_command(cmd); + + assert_eq!(result.warnings.len(), 1); + assert!(matches!( + result.warnings[0], + ParseWarning::UnclosedContinuation { start_line: 0 } + )); + } + + #[test] + fn closed_fence_has_no_warning() { + let cmd = start_command(header_from("/cmd ```"), 0); + let (cmd, _) = accept_line(cmd, 1, "content"); + let (cmd, _) = accept_line(cmd, 2, "```"); + + let result = finalize_command(cmd); + + assert!(result.warnings.is_empty()); + } + + #[test] + fn closed_continuation_has_no_warning() { + let cmd = start_command(header_from("/cmd first \\"), 0); + let (cmd, _) = accept_line(cmd, 1, "last line"); + + let result = finalize_command(cmd); + + assert!(result.warnings.is_empty()); + } + + #[test] + fn single_line_has_no_warning() { + let cmd = start_command(header_from("/help"), 0); + + let result = finalize_command(cmd); + + assert!(result.warnings.is_empty()); + } + + // --- Finalized command structure --- + + #[test] + fn finalized_command_has_correct_name() { + let cmd = start_command(header_from("/deploy production"), 0); + + let result = finalize_command(cmd); + + assert_eq!(result.command.name, "deploy"); + } + + #[test] + fn finalized_command_has_correct_range() { + let cmd = start_command(header_from("/cmd ```"), 0); + let (cmd, _) = accept_line(cmd, 1, "body"); + let (cmd, _) = accept_line(cmd, 2, "```"); + + let result = finalize_command(cmd); + + assert_eq!(result.command.range.start_line, 0); + assert_eq!(result.command.range.end_line, 2); + } + + #[test] + fn finalized_fence_payload_is_joined_lines() { + let cmd = start_command(header_from("/cmd ```"), 0); + let (cmd, _) = accept_line(cmd, 1, "line one"); + let (cmd, _) = accept_line(cmd, 2, "line two"); + let (cmd, _) = accept_line(cmd, 3, "```"); + + let result = finalize_command(cmd); + + assert_eq!(result.command.arguments.payload, "line one\nline two"); + assert_eq!(result.command.arguments.mode, ArgumentMode::Fence); + } + + #[test] + fn finalized_continuation_payload_is_joined_lines() { + let cmd = start_command(header_from("/cmd first \\"), 0); + let (cmd, _) = accept_line(cmd, 1, "second \\"); + let (cmd, _) = accept_line(cmd, 2, "third"); + + let result = finalize_command(cmd); + + assert_eq!(result.command.arguments.payload, "first\nsecond\nthird"); + assert_eq!(result.command.arguments.mode, ArgumentMode::Continuation); + } + + #[test] + fn finalized_single_line_payload_matches_header() { + let cmd = start_command(header_from("/hello world"), 0); + + let result = finalize_command(cmd); + + assert_eq!(result.command.arguments.payload, "world"); + assert_eq!(result.command.arguments.mode, ArgumentMode::SingleLine); + } + + #[test] + fn finalized_fence_captures_language() { + let cmd = start_command(header_from("/code ```rust"), 0); + let (cmd, _) = accept_line(cmd, 1, "fn main() {}"); + let (cmd, _) = accept_line(cmd, 2, "```"); + + let result = finalize_command(cmd); + + assert_eq!( + result.command.arguments.fence_lang, + Some("rust".to_string()) + ); + } + + use proptest::prelude::*; + + fn valid_command_name() -> impl Strategy { + "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + } + + proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn finalized_name_matches_pending_name(name in valid_command_name()) { + let input = format!("/{name} arg"); + let cmd = start_command(header_from(&input), 0); + let result = finalize_command(cmd); + prop_assert_eq!(result.command.name, name); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn closed_commands_never_warn( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,30}", 1..8) + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0); + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); + + let result = finalize_command(cmd); + prop_assert!(result.warnings.is_empty()); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn unclosed_fence_always_warns( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 1..5) + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0); + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + + let result = finalize_command(cmd); + prop_assert_eq!(result.warnings.len(), 1); + let is_unclosed_fence = matches!(result.warnings[0], ParseWarning::UnclosedFence { .. }); + prop_assert!(is_unclosed_fence); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn payload_is_payload_lines_joined( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 1..8) + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0); + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); + + let result = finalize_command(cmd); + prop_assert_eq!(result.command.arguments.payload, body_lines.join("\n")); + } + } +} diff --git a/parser/src/application/document_parse.rs b/parser/src/application/document_parse.rs new file mode 100644 index 0000000..a9da645 --- /dev/null +++ b/parser/src/application/document_parse.rs @@ -0,0 +1,452 @@ +use super::{ + command_accumulate::{AcceptResult, PendingCommand, accept_line, start_command}, + command_finalize::finalize_command, + line_classify::{LineKind, classify_line}, + text_collect::{PendingText, append_text, finalize_text, start_text}, +}; +use crate::domain::{Command, ParseResult, ParseWarning, ParserContext, SPEC_VERSION, TextBlock}; + +struct ParseState { + current_cmd: Option, + current_text: Option, + commands: Vec, + text_blocks: Vec, + warnings: Vec, +} + +pub fn parse_document(input: &str) -> ParseResult { + let state = input + .lines() + .enumerate() + .fold(empty_state(), |state, (i, line)| step(state, i, line)); + + finalize(state) +} + +fn empty_state() -> ParseState { + ParseState { + current_cmd: None, + current_text: None, + commands: Vec::new(), + text_blocks: Vec::new(), + warnings: Vec::new(), + } +} + +fn step(mut state: ParseState, line_index: usize, line: &str) -> ParseState { + if try_feed_command(&mut state, line_index, line) { + return state; + } + + classify_fresh_line(&mut state, line_index, line); + state +} + +fn try_feed_command(state: &mut ParseState, line_index: usize, line: &str) -> bool { + let Some(cmd) = state.current_cmd.take() else { + return false; + }; + + if !cmd.is_open { + absorb_command(state, cmd); + return false; + } + + let (updated_cmd, result) = accept_line(cmd, line_index, line); + match result { + AcceptResult::Consumed => { + state.current_cmd = Some(updated_cmd); + true + } + AcceptResult::Completed => { + absorb_command(state, updated_cmd); + true + } + AcceptResult::Rejected => { + absorb_command(state, updated_cmd); + false + } + } +} + +fn classify_fresh_line(state: &mut ParseState, line_index: usize, line: &str) { + match classify_line(line) { + LineKind::Command(header) => { + flush_text(state); + state.current_cmd = Some(start_command(header, line_index)); + } + LineKind::Text => { + accumulate_text(state, line_index, line); + } + } +} + +fn absorb_command(state: &mut ParseState, cmd: PendingCommand) { + let finalized = finalize_command(cmd); + state.commands.push(finalized.command); + state.warnings.extend(finalized.warnings); +} + +fn flush_text(state: &mut ParseState) { + if let Some(text) = state.current_text.take() { + state.text_blocks.push(finalize_text(text)); + } +} + +fn accumulate_text(state: &mut ParseState, line_index: usize, line: &str) { + state.current_text = match state.current_text.take() { + Some(text) => Some(append_text(text, line_index, line)), + None => Some(start_text(line_index, line)), + }; +} + +fn finalize(mut state: ParseState) -> ParseResult { + if let Some(cmd) = state.current_cmd.take() { + absorb_command(&mut state, cmd); + } + + flush_text(&mut state); + + ParseResult { + commands: state.commands, + text_blocks: state.text_blocks, + warnings: state.warnings, + version: SPEC_VERSION.to_owned(), + context: ParserContext::default(), + } +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use crate::{application::document_parse::parse_document, domain::ArgumentMode}; + + fn parse(input: &str) -> crate::domain::ParseResult { + parse_document(input) + } + + fn assert_single_command(input: &str) -> crate::domain::Command { + let result = parse(input); + assert_eq!(result.commands.len(), 1, "expected exactly 1 command"); + result.commands.into_iter().next().unwrap() + } + + #[test] + fn single_line_simple_command_parses_name() { + let cmd = assert_single_command("/hello world"); + assert_eq!(cmd.name, "hello"); + } + + #[test] + fn trailing_newline_does_not_create_empty_text_block() { + let result = parse("/cmd arg\n"); + assert_eq!(result.text_blocks.len(), 0); + } + + #[test] + fn single_line_mode_threads_through_from_classify() { + let cmd = assert_single_command("/hello world"); + assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); + } + + #[test] + fn single_line_payload_threads_through_from_classify() { + let cmd = assert_single_command("/hello world"); + assert_eq!(cmd.arguments.payload, "world"); + } + + #[test] + fn unclosed_continuation_produces_warning() { + let input = "/cmd start \\"; + let result = parse(input); + assert_eq!(result.commands.len(), 1); + assert_eq!(result.warnings.len(), 1); + } + + #[test] + fn leading_whitespace_command_threads_through() { + let cmd = assert_single_command(" /hello world"); + assert_eq!(cmd.name, "hello"); + } + + #[test] + fn single_line_command_with_no_args_has_empty_header() { + let cmd = assert_single_command("/ping"); + assert_eq!(cmd.arguments.header, ""); + assert_eq!(cmd.arguments.payload, ""); + } + + #[test] + fn single_line_command_with_complex_args() { + let cmd = assert_single_command(r#"/mcp call_tool read_file {"path": "src/index.ts"}"#); + assert_eq!(cmd.name, "mcp"); + assert_eq!( + cmd.arguments.header, + r#"call_tool read_file {"path": "src/index.ts"}"# + ); + } + + #[test] + fn single_line_command_range_is_same_line() { + let cmd = assert_single_command("/test"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 0); + } + + #[test] + fn parses_text_and_single_command() { + let input = "intro\n/cmd arg\noutro"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + assert_eq!(result.text_blocks.len(), 2); + assert_eq!(result.commands[0].name, "cmd"); + assert_eq!(result.commands[0].arguments.mode, ArgumentMode::SingleLine); + } + + #[test] + fn command_on_line_two_has_correct_range() { + let input = "text\nmore text\n/cmd arg"; + let result = parse(input); + assert_eq!(result.commands[0].range.start_line, 2); + assert_eq!(result.commands[0].range.end_line, 2); + } + + #[test] + fn two_single_line_commands_both_parse() { + let result = parse("/first a\n/second b"); + assert_eq!(result.commands.len(), 2); + assert_eq!(result.commands[0].name, "first"); + assert_eq!(result.commands[1].name, "second"); + } + + #[test] + fn empty_input_produces_empty_result() { + let result = parse(""); + assert_eq!(result.commands.len(), 0); + assert_eq!(result.text_blocks.len(), 0); + assert_eq!(result.warnings.len(), 0); + } + + #[test] + fn text_only_produces_one_text_block() { + let result = parse("just some text"); + assert_eq!(result.commands.len(), 0); + assert_eq!(result.text_blocks.len(), 1); + } + + #[test] + fn adjacent_text_lines_merge_into_one_block() { + let result = parse("line one\nline two\nline three"); + assert_eq!(result.text_blocks.len(), 1); + assert_eq!(result.text_blocks[0].range.start_line, 0); + assert_eq!(result.text_blocks[0].range.end_line, 2); + } + + #[test] + fn text_block_before_command_has_correct_content() { + let result = parse("intro line\n/cmd arg"); + assert_eq!(result.text_blocks.len(), 1); + assert_eq!(result.text_blocks[0].content, "intro line"); + } + + #[test] + fn text_block_after_command_has_correct_range() { + let result = parse("/cmd arg\noutro line"); + assert_eq!(result.text_blocks.len(), 1); + assert_eq!(result.text_blocks[0].range.start_line, 1); + } + + #[test] + fn continuation_command_parses_through_document() { + let input = "/cmd first \\\nsecond \\\nthird"; + let cmd = assert_single_command(input); + assert_eq!(cmd.arguments.mode, ArgumentMode::Continuation); + assert_eq!(cmd.arguments.payload, "first\nsecond\nthird"); + } + + #[test] + fn fenced_command_parses_through_document() { + let input = "/cmd ```\nline one\nline two\n```"; + let cmd = assert_single_command(input); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.payload, "line one\nline two"); + } + + #[test] + fn unclosed_fence_produces_warning() { + let input = "/cmd ```\nline one\nline two"; + let result = parse(input); + assert_eq!(result.commands.len(), 1); + assert_eq!(result.warnings.len(), 1); + } + + #[test] + fn fence_followed_by_new_command() { + let input = "/first ```\nbody\n```\n/second arg"; + let result = parse(input); + assert_eq!(result.commands.len(), 2); + assert_eq!(result.commands[0].name, "first"); + assert_eq!(result.commands[1].name, "second"); + } + + #[test] + fn continuation_followed_by_new_command() { + let input = "/first start \\\nend\n/second arg"; + let result = parse(input); + assert_eq!(result.commands.len(), 2); + assert_eq!(result.commands[0].name, "first"); + assert_eq!(result.commands[1].name, "second"); + } + + #[test] + fn version_is_set() { + let result = parse(""); + assert_eq!(result.version, crate::domain::SPEC_VERSION); + } + + // --- Property tests --- + + fn valid_command_name() -> impl Strategy { + "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + } + + fn command_line() -> impl Strategy { + (valid_command_name(), "[a-z0-9 ]{0,30}").prop_map(|(name, args)| format!("/{name} {args}")) + } + + fn text_line() -> impl Strategy { + "[a-zA-Z0-9 !.,]{1,60}".prop_filter("must not start with slash", |s| { + !s.trim_start().starts_with('/') + }) + } + + proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn never_panics_on_arbitrary_input(input in "[\\x00-\\x7F]{0,500}") { + let _ = parse_document(&input); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn command_count_matches_slash_lines( + commands in prop::collection::vec(command_line(), 1..10) + ) { + let input = commands.join("\n"); + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), commands.len()); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_only_input_produces_zero_commands( + lines in prop::collection::vec(text_line(), 1..20) + ) { + let input = lines.join("\n"); + let result = parse_document(&input); + prop_assert!(result.commands.is_empty()); + prop_assert!(!result.text_blocks.is_empty()); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn version_is_always_spec_version(input in "[\\x20-\\x7E]{0,200}") { + let result = parse_document(&input); + prop_assert_eq!(&result.version, crate::domain::SPEC_VERSION); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fenced_content_preserved_verbatim( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9 !]{1,50}", 1..5) + ) { + let body = body_lines.join("\n"); + let input = format!("/{name} ```\n{body}\n```"); + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), 1); + prop_assert_eq!(&result.commands[0].arguments.payload, &body); + prop_assert_eq!(&result.commands[0].arguments.mode, &ArgumentMode::Fence); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn interleaved_text_and_commands_both_captured( + cmd_count in 1usize..5 + ) { + let mut lines = Vec::new(); + for i in 0..cmd_count { + lines.push(format!("text line {i}")); + lines.push(format!("/cmd{i} arg")); + } + let input = lines.join("\n"); + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), cmd_count); + prop_assert_eq!(result.text_blocks.len(), cmd_count); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn unclosed_fence_always_produces_warning( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,30}", 1..5) + ) { + let body = body_lines.join("\n"); + let input = format!("/{name} ```\n{body}"); + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), 1); + prop_assert_eq!(result.warnings.len(), 1); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_block_content_matches_non_command_lines( + lines in prop::collection::vec(text_line(), 1..10) + ) { + let input = lines.join("\n"); + let result = parse_document(&input); + prop_assert_eq!(result.text_blocks.len(), 1); + prop_assert_eq!(&result.text_blocks[0].content, &input); + } + + fn continuation_payload_matches_joined_lines( + name in valid_command_name(), + middle_lines in prop::collection::vec("[a-z0-9]{1,20}", 0..5), + final_line in "[a-z0-9]{1,20}" + ) { + let mut input_lines = vec![format!("/{name} start \\")]; + let mut expected_parts = vec!["start".to_string()]; + for line in &middle_lines { + input_lines.push(format!("{line} \\")); + expected_parts.push(line.clone()); + } + input_lines.push(final_line.clone()); + expected_parts.push(final_line); + let input = input_lines.join("\n"); + + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), 1); + prop_assert_eq!(&result.commands[0].arguments.mode, &ArgumentMode::Continuation); + prop_assert_eq!(&result.commands[0].arguments.payload, &expected_parts.join("\n")); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn unclosed_continuation_always_produces_warning( + name in valid_command_name(), + middle_lines in prop::collection::vec("[a-z0-9]{1,20}", 0..3) + ) { + let mut input_lines = vec![format!("/{name} start \\")]; + for line in &middle_lines { + input_lines.push(format!("{line} \\")); + } + let input = input_lines.join("\n"); + + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), 1); + prop_assert_eq!(result.warnings.len(), 1); + } + + } +} diff --git a/parser/src/application/line_classify.rs b/parser/src/application/line_classify.rs new file mode 100644 index 0000000..c95be82 --- /dev/null +++ b/parser/src/application/line_classify.rs @@ -0,0 +1,281 @@ +use crate::domain::ArgumentMode; + +/// Raw classification of a single input line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LineKind { + Command(CommandHeader), + Text, +} + +/// Extracted header fields from a line that starts a slash command. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommandHeader { + pub raw: String, + pub name: String, + pub header_text: String, + pub mode: ArgumentMode, + pub fence_lang: Option, +} + +/// Classify a single line as either a command header or plain text. +pub fn classify_line(line: &str) -> LineKind { + match try_parse_command(line) { + Some(header) => LineKind::Command(header), + None => LineKind::Text, + } +} + +fn try_parse_command(line: &str) -> Option { + let trimmed = line.trim_start(); + if !trimmed.starts_with('/') { + return None; + } + + let without_slash = &trimmed[1..]; + + // Command name is the first contiguous non-whitespace run. + let mut parts = without_slash.splitn(2, char::is_whitespace); + let name_raw = parts.next().filter(|n| !n.is_empty())?; + + // Command must begin with a lowercase ASCII letter and may contain lowercase letters, digits, and hyphens + if !name_raw.starts_with(|c: char| c.is_ascii_lowercase()) { + return None; + } + + if !name_raw + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-' || c == '_') + { + return None; + } + let name = name_raw.to_string(); + let rest = parts.next().unwrap_or("").trim_start(); + + // Fence mode: rest starts with ``` + if let Some(stripped) = rest.strip_prefix("```") { + let after_ticks = stripped.trim(); + let lang = after_ticks + .split_whitespace() + .next() + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + + return Some(CommandHeader { + raw: line.to_string(), + name, + header_text: rest.to_string(), + mode: ArgumentMode::Fence, + fence_lang: lang, + }); + } + + // Continuation mode: rest ends with trailing backslash. + if rest.ends_with('\\') { + let header_text = rest.trim_end_matches('\\').trim_end().to_string(); + + return Some(CommandHeader { + raw: line.to_string(), + name, + header_text, + mode: ArgumentMode::Continuation, + fence_lang: None, + }); + } + + // Single-line mode (default). + Some(CommandHeader { + raw: line.to_string(), + name, + header_text: rest.to_string(), + mode: ArgumentMode::SingleLine, + fence_lang: None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plain_text_line() { + assert_eq!(classify_line("hello world"), LineKind::Text); + } + + #[test] + fn bare_slash_is_text() { + assert_eq!(classify_line("/ "), LineKind::Text); + } + + #[test] + fn simple_command() { + let result = classify_line("/cmd some args"); + match result { + LineKind::Command(h) => { + assert_eq!(h.name, "cmd"); + assert_eq!(h.header_text, "some args"); + assert_eq!(h.mode, ArgumentMode::SingleLine); + } + _ => panic!("expected Command"), + } + } + + #[test] + fn fence_command_with_lang() { + let result = classify_line("/code ```rust"); + match result { + LineKind::Command(h) => { + assert_eq!(h.name, "code"); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang.as_deref(), Some("rust")); + } + _ => panic!("expected Command"), + } + } + + #[test] + fn continuation_command() { + let result = classify_line("/cmd first part \\"); + match result { + LineKind::Command(h) => { + assert_eq!(h.name, "cmd"); + assert_eq!(h.header_text, "first part"); + assert_eq!(h.mode, ArgumentMode::Continuation); + } + _ => panic!("expected Command"), + } + } + + #[test] + fn command_with_no_args() { + let result = classify_line("/help"); + match result { + LineKind::Command(h) => { + assert_eq!(h.name, "help"); + assert_eq!(h.header_text, ""); + assert_eq!(h.mode, ArgumentMode::SingleLine); + } + _ => panic!("expected Command"), + } + } + #[test] + fn slash_alone_no_space_is_text() { + assert_eq!(classify_line("/"), LineKind::Text); + } + + #[test] + fn command_name_starting_with_uppercase_is_text() { + assert_eq!(classify_line("/Hello"), LineKind::Text); + } + + #[test] + fn command_name_starting_with_digit_is_text() { + assert_eq!(classify_line("/2fast"), LineKind::Text); + } + + #[test] + fn leading_spaces_still_detected_as_command() { + match classify_line(" /cmd arg") { + LineKind::Command(h) => assert_eq!(h.name, "cmd"), + _ => panic!("expected Command"), + } + } + + #[test] + fn fence_without_language() { + match classify_line("/cmd ```") { + LineKind::Command(h) => { + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, None); + } + _ => panic!("expected Command"), + } + } + + #[test] + fn raw_preserves_original_line() { + match classify_line(" /cmd arg") { + LineKind::Command(h) => assert_eq!(h.raw, " /cmd arg"), + _ => panic!("expected Command"), + } + } + + // --- Property tests --- + + use proptest::prelude::*; + + fn valid_command_name() -> impl Strategy { + "[a-z][a-z0-9\\-]{0,20}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + } + + fn arbitrary_line() -> impl Strategy { + prop_oneof![ + // plain text + "[a-zA-Z0-9 !.,]{0,80}", + // command-like + valid_command_name().prop_flat_map(|name| { + "[a-zA-Z0-9 ]{0,40}".prop_map(move |args| format!("/{name} {args}")) + }), + // leading whitespace + command + valid_command_name().prop_flat_map(|name| { + (1usize..5, "[a-zA-Z0-9 ]{0,40}").prop_map(move |(spaces, args)| { + format!("{}/{} {}", " ".repeat(spaces), name, args) + }) + }), + ] + } + + proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn classify_never_panics(line in "[\\x00-\\x7F]{0,200}") { + let _ = classify_line(&line); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn valid_name_always_produces_command(name in valid_command_name(), args in "[a-z0-9 ]{0,40}") { + let input = format!("/{name} {args}"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.name, name), + LineKind::Text => panic!("expected Command for input: {input}"), + } + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_without_slash_is_never_command(line in "[a-zA-Z0-9 !.,]{1,80}") { + prop_assert!(matches!(classify_line(&line), LineKind::Text)); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn raw_field_preserves_original_input(line in arbitrary_line()) { + if let LineKind::Command(h) = classify_line(&line) { + prop_assert_eq!(h.raw, line); + } + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fence_mode_iff_backticks_present(name in valid_command_name(), lang in "[a-z]{0,10}") { + let input = format!("/{name} ```{lang}"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Fence), + _ => panic!("expected Command"), + } + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn continuation_mode_iff_trailing_backslash( + name in valid_command_name(), + args in "[a-z0-9 ]{1,30}" + ) { + let input = format!("/{name} {args} \\"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Continuation), + _ => panic!("expected Command"), + } + } + } +} diff --git a/parser/src/application/mod.rs b/parser/src/application/mod.rs new file mode 100644 index 0000000..19c9b5b --- /dev/null +++ b/parser/src/application/mod.rs @@ -0,0 +1,10 @@ +mod command_accumulate; +mod command_finalize; +mod document_parse; +mod line_classify; +mod text_collect; + +pub use document_parse::parse_document; + +#[cfg(test)] +mod tests; diff --git a/parser/src/application/tests/mod.rs b/parser/src/application/tests/mod.rs new file mode 100644 index 0000000..cd2f278 --- /dev/null +++ b/parser/src/application/tests/mod.rs @@ -0,0 +1 @@ +mod proptest; diff --git a/parser/src/application/tests/proptest.rs b/parser/src/application/tests/proptest.rs new file mode 100644 index 0000000..c994daa --- /dev/null +++ b/parser/src/application/tests/proptest.rs @@ -0,0 +1,62 @@ +use proptest::prelude::*; + +use crate::{ + application::{ + command_accumulate::{accept_line, start_command}, + command_finalize::finalize_command, + line_classify::{LineKind, classify_line}, + text_collect::{append_text, finalize_text, start_text}, + }, + domain::ArgumentMode, +}; + +fn valid_command_name() -> impl Strategy { + "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) +} + +proptest! { + /// classify -> accumulate -> finalize roundtrip never panics. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn classify_accumulate_finalize_roundtrip( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,30}", 0..8) + ) { + let input = format!("/{name} ```"); + let header = match classify_line(&input) { + LineKind::Command(h) => h, + LineKind::Text => panic!("expected command"), + }; + + let cmd = start_command(header, 0); + + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); + + let finalized = finalize_command(cmd); + prop_assert_eq!(finalized.command.name, name); + prop_assert_eq!(finalized.command.arguments.mode, ArgumentMode::Fence); + prop_assert!(finalized.warnings.is_empty()); + } + + /// Text lines classified as Text always produce valid text blocks. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_lines_through_classify_and_collect( + lines in prop::collection::vec("[a-zA-Z0-9 !.,]{1,40}", 1..15) + ) { + for line in &lines { + prop_assert!(matches!(classify_line(line), LineKind::Text)); + } + + let pending = lines.iter().enumerate().skip(1).fold( + start_text(0, &lines[0]), + |text, (i, line)| append_text(text, i, line), + ); + let block = finalize_text(pending); + prop_assert_eq!(block.content, lines.join("\n")); + } +} diff --git a/parser/src/application/text_collect.rs b/parser/src/application/text_collect.rs new file mode 100644 index 0000000..21a212b --- /dev/null +++ b/parser/src/application/text_collect.rs @@ -0,0 +1,170 @@ +use crate::domain::{LineRange, TextBlock}; + +/// In-progress accumulation of contiguous non-command lines. +#[derive(Debug, Clone)] +pub struct PendingText { + pub start_line: usize, + pub end_line: usize, + pub lines: Vec, +} + +pub fn start_text(line_index: usize, line: &str) -> PendingText { + PendingText { + start_line: line_index, + end_line: line_index, + lines: vec![line.to_string()], + } +} + +pub fn append_text(mut text: PendingText, line_index: usize, line: &str) -> PendingText { + text.end_line = line_index; + text.lines.push(line.to_string()); + text +} + +pub fn finalize_text(text: PendingText) -> TextBlock { + let content = text.lines.join("\n"); + TextBlock { + range: LineRange { + start_line: text.start_line, + end_line: text.end_line, + }, + content, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn single_line_text_block() { + let pending = start_text(0, "hello"); + let block = finalize_text(pending); + assert_eq!(block.content, "hello"); + assert_eq!(block.range.start_line, 0); + assert_eq!(block.range.end_line, 0); + } + + #[test] + fn multi_line_text_block() { + let text = start_text(2, "first"); + let text = append_text(text, 3, "second"); + let text = append_text(text, 4, "third"); + let block = finalize_text(text); + assert_eq!(block.content, "first\nsecond\nthird"); + assert_eq!(block.range.start_line, 2); + assert_eq!(block.range.end_line, 4); + } + + #[test] + fn empty_line_preserved_in_content() { + let text = start_text(0, "before"); + let text = append_text(text, 1, ""); + let text = append_text(text, 2, "after"); + let block = finalize_text(text); + assert_eq!(block.content, "before\n\nafter"); + } + + #[test] + fn whitespace_only_line_preserved() { + let pending = start_text(5, " "); + let block = finalize_text(pending); + assert_eq!(block.content, " "); + assert_eq!(block.range.start_line, 5); + } + + #[test] + fn append_updates_end_line_only() { + let text = start_text(3, "first"); + let text = append_text(text, 4, "second"); + assert_eq!(text.start_line, 3); + assert_eq!(text.end_line, 4); + } + + // --- Property tests --- + + use proptest::prelude::*; + + fn valid_command_name() -> impl Strategy { + "[a-z][a-z0-9\\-]{0,20}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + } + + fn arbitrary_line() -> impl Strategy { + prop_oneof![ + // plain text + "[a-zA-Z0-9 !.,]{0,80}", + // command-like + valid_command_name().prop_flat_map(|name| { + "[a-zA-Z0-9 ]{0,40}".prop_map(move |args| format!("/{name} {args}")) + }), + // leading whitespace + command + valid_command_name().prop_flat_map(|name| { + (1usize..5, "[a-zA-Z0-9 ]{0,40}").prop_map(move |(spaces, args)| { + format!("{}/{} {}", " ".repeat(spaces), name, args) + }) + }), + ] + } + use crate::{ + application::line_classify::{LineKind, classify_line}, + domain::ArgumentMode, + }; + proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + + + + fn classify_never_panics(line in "[\\x00-\\x7F]{0,200}") { + let _ = classify_line(&line); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn valid_name_always_produces_command(name in valid_command_name(), args in "[a-z0-9 ]{0,40}") { + let input = format!("/{name} {args}"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.name, name), + LineKind::Text => panic!("expected Command for input: {input}"), + } + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_without_slash_is_never_command(line in "[a-zA-Z0-9 !.,]{1,80}") { + prop_assert!(matches!(classify_line(&line), LineKind::Text)); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn raw_field_preserves_original_input(line in arbitrary_line()) { + if let LineKind::Command(h) = classify_line(&line) { + prop_assert_eq!(h.raw, line); + } + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fence_mode_iff_backticks_present(name in valid_command_name(), lang in "[a-z]{0,10}") { + let input = format!("/{name} ```{lang}"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Fence), + _ => panic!("expected Command"), + } + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn continuation_mode_iff_trailing_backslash( + name in valid_command_name(), + args in "[a-z0-9 ]{1,30}" + ) { + let input = format!("/{name} {args} \\"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Continuation), + _ => panic!("expected Command"), + } + } + } +} diff --git a/parser/src/lib.rs b/parser/src/lib.rs new file mode 100644 index 0000000..0001c4b --- /dev/null +++ b/parser/src/lib.rs @@ -0,0 +1,5 @@ +mod application; +pub mod domain; + +// Public API re-export +pub use application::parse_document; diff --git a/parser/tests/mod.rs b/parser/tests/mod.rs new file mode 100644 index 0000000..be37b0f --- /dev/null +++ b/parser/tests/mod.rs @@ -0,0 +1,21 @@ +// use super::*; +// use crate::domain::{ArgumentMode, ParserContext}; + +// mod proptest; + +// // --- Test helpers --- + +// fn default_context() -> ParserContext { ParserContext::default() } + +// fn parse(input: &str) -> SlashParseResult { parse_to_domain(input, default_context()) } + +// fn assert_single_command(input: &str) -> Command { +// let result = parse(input); +// assert_eq!(result.commands.len(), 1, "expected exactly 1 command"); +// result.commands.into_iter().next().unwrap() +// } + +// fn assert_no_commands(input: &str) { +// let result = parse(input); +// assert!(result.commands.is_empty(), "expected no commands, got {}", result.commands.len()); +// } From 11d1fb9034ae8be1d5108b88a12cc91ee262e6e7 Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Wed, 18 Mar 2026 20:46:33 -0600 Subject: [PATCH 3/5] doc: new 0.3.0 specification, clean slated on ADRs --- ...ward-slash-as-command-trigger-character.md | 62 - ...-continuation-with-trailing-space-slash.md | 98 -- ...lock-syntax-for-raw-multi-line-payloads.md | 89 -- .../0004-state-machine-parser-with-no-ast.md | 78 -- ...ment-payload-with-command-level-parsing.md | 87 -- ...sitory-structure-and-project-boundaries.md | 116 -- ...-build-tooling-and-toolchain-management.md | 84 -- .../0008-sequential-zero-based-command-ids.md | 61 - ...lt-output-with-pretty-json-array-option.md | 109 -- ...put-model-and-line-ending-normalization.md | 57 - docs/slash-cmd parseing sematic.md | 314 ----- docs/slash-parser CLI Specification.md | 19 +- docs/slash-parser-spec-v0.3.0.md | 1096 +++++++++++++++++ docs/slash-parser.md | 333 ----- 14 files changed, 1098 insertions(+), 1505 deletions(-) delete mode 100644 docs/adrs/0001-use-forward-slash-as-command-trigger-character.md delete mode 100644 docs/adrs/0002-line-continuation-with-trailing-space-slash.md delete mode 100644 docs/adrs/0003-fenced-block-syntax-for-raw-multi-line-payloads.md delete mode 100644 docs/adrs/0004-state-machine-parser-with-no-ast.md delete mode 100644 docs/adrs/0005-opaque-argument-payload-with-command-level-parsing.md delete mode 100644 docs/adrs/0006-repository-structure-and-project-boundaries.md delete mode 100644 docs/adrs/0007-build-tooling-and-toolchain-management.md delete mode 100644 docs/adrs/0008-sequential-zero-based-command-ids.md delete mode 100644 docs/adrs/0009-jsonl-default-output-with-pretty-json-array-option.md delete mode 100644 docs/adrs/0010-input-model-and-line-ending-normalization.md delete mode 100644 docs/slash-cmd parseing sematic.md create mode 100644 docs/slash-parser-spec-v0.3.0.md delete mode 100644 docs/slash-parser.md diff --git a/docs/adrs/0001-use-forward-slash-as-command-trigger-character.md b/docs/adrs/0001-use-forward-slash-as-command-trigger-character.md deleted file mode 100644 index e3c78e0..0000000 --- a/docs/adrs/0001-use-forward-slash-as-command-trigger-character.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -number: 1 -title: Use Forward Slash as Command Trigger Character -date: 2026-03-13 -status: accepted ---- - -# 1. Use Forward Slash as Command Trigger Character - -Date: 2026-03-13 - -## Status - -Accepted - -## Context - -The parser needs a reliable way to distinguish command lines from free-form text in a mixed-content -input stream. The trigger character must be unambiguous in the first column, easy to type, and -familiar to users of chat-style slash command interfaces (Discord, Slack, etc.). It must also allow -the parser to detect commands in a single left-to-right scan without backtracking. - -## Decision - -A command line is any line whose first non-whitespace character is `/` (forward slash, U+002F). - -Command line structure: - -```text -/[] -``` - -Command name rules: - -- Regex: `[a-z][a-z0-9-]*` -- Starts immediately after the leading `/` with no intervening space. -- Ends at the first whitespace character or end-of-line. -- Must begin with a lowercase ASCII letter and may contain lowercase letters, digits, and hyphens. - -Arguments prefix (optional): - -- Everything after the first whitespace following the command name. -- May contain inline arguments, an inline fence opener (see ADR-0003), or a continuation marker (see - ADR-0002). - -Any line that does not begin (after optional leading whitespace) with `/` is a non-command line. -Non-command lines are collected into text blocks (see ADR-0009). - -A line where the first non-whitespace character is `/` but the text after `/` does not match the -command name regex (e.g., a bare `/` or `/123`) is treated context-dependently: during accumulation -it becomes a literal payload line; in idle state it is treated as a non-command line. - -## Consequences - -- Command detection is a single character check on each line after skipping leading whitespace, - making it O(1) per line. -- The forward slash convention aligns with established chat/CLI slash command patterns, reducing - user learning curve. -- The strict command name regex prevents ambiguity with path-like content (e.g., `/usr/bin` fails - the regex because `usr/bin` contains `/`). -- Optional leading whitespace tolerance means indented commands (e.g., in nested contexts) are still - recognized. diff --git a/docs/adrs/0002-line-continuation-with-trailing-space-slash.md b/docs/adrs/0002-line-continuation-with-trailing-space-slash.md deleted file mode 100644 index d92896c..0000000 --- a/docs/adrs/0002-line-continuation-with-trailing-space-slash.md +++ /dev/null @@ -1,98 +0,0 @@ ---- -number: 2 -title: Line Continuation with Trailing Space-Slash -date: 2026-03-13 -status: accepted ---- - -# 2. Line Continuation with Trailing Space-Slash - -Date: 2026-03-13 - -## Status - -Accepted - -## Context - -Commands often need arguments that are too long or too complex for a single line. The parser needs -an explicit, opt-in mechanism for spanning a logical command across multiple physical lines. The -marker must be unambiguous (not confused with path separators or content containing slashes) and -must compose cleanly with the other argument modes (single-line and fenced blocks). - -## Decision - -Continuation mode is activated by a trailing space-slash (`/`) marker at the end of a line. - -### Marker definition - -A continuation marker is a line (after normalization per ADR-0010) whose content ends with a space -followed by a slash (`/`) immediately before the line terminator, with nothing after the slash -except optional trailing whitespace. - -Formally, a line is a continuation marker if the last two non-whitespace-trailing characters are `/` -(space + slash). - -Optional leading whitespace is allowed. For example, `/echo /` and `/` (space+slash with -indentation) both qualify. - -A special case: a line that, aside from leading whitespace, is exactly `/` represents a blank -payload line in continuation mode. - -### First command line with continuation - -When a command's first line ends with `/`: - -1. The parser strips the final `/` from that line's content. -2. The remaining content after the command name (which may be empty) becomes the first segment of - `arguments.payload`, followed by a newline separator. -3. `arguments.mode` is set to `continuation`. -4. The parser transitions to the `accumulating` state (see ADR-0004). - -If the first line does not end with `/` and has no fence opener, the command is single-line (see -ADR-0005). - -### Lines in accumulating state - -For each subsequent line while in `accumulating`: - -- Continuation marker line (ends in `/`): represents a blank payload line. The parser appends a - newline to `arguments.payload` and remains in `accumulating`. -- Empty line (zero characters after normalization): the parser finalizes the command and transitions - to `idle`. This blank line is not appended to the payload. -- Any other non-empty line: the parser appends the line content plus a newline to - `arguments.payload` and remains in `accumulating`. - -A fence opener encountered on a continuation line transitions the command from `accumulating` to -`inFence` (see ADR-0003). - -### Bare slash is not continuation - -A bare `/` at the end of a line (no preceding space) is not a continuation marker. If the line's -first non-whitespace character is `/`, it is processed via command detection (ADR-0001). If the `/` -appears elsewhere, it is literal content. Users needing complex or ambiguous multi-line payloads -should use fenced blocks (ADR-0003). - -Example: the input - -```text -/echo / -ooga booga -/ -testing 123 -``` - -does not include `testing 123` in `/echo`'s payload because the bare `/` line is not a `/` marker. - -## Consequences - -- Continuation is explicit and opt-in; lines without the marker never accidentally join. -- The space-slash marker avoids ambiguity with path-like content (e.g., `/usr/bin/` has no preceding - space before the trailing slash). -- Empty lines serve as a natural, visible terminator for multi-line commands. -- Blank payload lines can be represented by the `/` special case, so continuation mode can express - payloads containing empty lines. -- Round-trip fidelity of original whitespace at continuation boundaries is intentionally lost; the - canonical form uses newline separators between payload segments. -- Fenced blocks (ADR-0003) remain available as the preferred alternative when payloads are complex - or contain content that could be confused with markers. diff --git a/docs/adrs/0003-fenced-block-syntax-for-raw-multi-line-payloads.md b/docs/adrs/0003-fenced-block-syntax-for-raw-multi-line-payloads.md deleted file mode 100644 index a9b907b..0000000 --- a/docs/adrs/0003-fenced-block-syntax-for-raw-multi-line-payloads.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -number: 3 -title: Fenced Block Syntax for Raw Multi-line Payloads -date: 2026-03-13 -status: accepted ---- - -# 3. Fenced Block Syntax for Raw Multi-line Payloads - -Date: 2026-03-13 - -## Status - -Accepted - -## Context - -Some commands need to carry large, raw, multi-line payloads (JSON documents, code snippets, -configuration blocks) where the content itself may contain sequences that would otherwise be -interpreted as continuation markers (`/`) or command triggers (`/`). Continuation mode (ADR-0002) -cannot safely transport such content because every line is subject to marker detection. A verbatim -transport mechanism is needed that suspends all parsing rules until an explicit closing delimiter. - -## Decision - -Fenced blocks use markdown-style backtick fences to wrap raw payloads attached to a command. - -### Fence openers - -There are two ways to enter fence mode: - -1. Inline fence on the command line: - - ````text - / ```[lang] - - ```` - ``` - The first occurrence of three or more consecutive backticks in the arguments prefix is treated as the fence opener. Text before the opener is kept in `arguments.header`. The parser records the fence marker length (number of backticks) and the optional language identifier (e.g., `jsonl`). - ``` - -2. Fence on the next line after continuation: - - ````text - / / - ```[lang] - - ```` - ``` - The first line ends with ` /` (entering continuation per ADR-0002), but the next line is recognized as a fence opener. The parser transitions from `accumulating` to `inFence` for this command. - ``` - -### Fence mode semantics - -While in `inFence` state (see ADR-0004): - -- All lines (including blank lines) are appended to `arguments.payload` with newline separators. -- Continuation markers (`/`) inside a fence are treated as literal content, not syntax. -- Command triggers (`/`) at the start of a line inside a fence are treated as literal content. - -### Closing fence - -A line is a closing fence if, after trimming leading and trailing whitespace: - -- It consists solely of backticks. -- The number of backticks is greater than or equal to the opener's count. - -The closing fence line itself is not appended to `arguments.payload`. - -### Resulting command state - -Once the closing fence is found: - -- `arguments.mode` is set to `fence`. -- `arguments.fence_lang` is the captured language tag or null. -- The command is finalized and the parser transitions back to `idle`. - -## Consequences - -- Payloads inside fenced blocks are completely verbatim; no parsing rules apply to content lines, - eliminating escaping concerns. -- The variable-length backtick fence (3 or more) means content containing triple backticks can be - fenced with four or more backticks, avoiding collision. -- The inline and continuation-then-fence entry paths give authors flexibility in how they structure - command lines. -- Language tags (e.g., `jsonl`, `yaml`) are captured as metadata, enabling downstream consumers to - apply format-specific parsing to the payload. -- Fenced blocks have a clear, mandatory closing delimiter, so the parser always knows when the - payload ends (unlike continuation mode which relies on empty line termination). diff --git a/docs/adrs/0004-state-machine-parser-with-no-ast.md b/docs/adrs/0004-state-machine-parser-with-no-ast.md deleted file mode 100644 index 4e851d6..0000000 --- a/docs/adrs/0004-state-machine-parser-with-no-ast.md +++ /dev/null @@ -1,78 +0,0 @@ ---- -number: 4 -title: State Machine Parser with No AST -date: 2026-03-13 -status: accepted ---- - -# 4. State Machine Parser with No AST - -Date: 2026-03-13 - -## Status - -Accepted - -## Context - -The slash-cmd format is line-oriented with three distinct argument modes (single-line, continuation, -fenced). A traditional recursive-descent parser or AST-building approach would add unnecessary -complexity for a format that can be fully parsed in a single forward pass over lines. The parser -needs to track which mode it is in, accumulate payloads accordingly, and emit finalized commands and -text blocks without constructing an intermediate tree representation. - -## Decision - -The parser uses a line-based state machine with three primary states and no intermediate AST. - -### States - -- `idle`: not currently accumulating a command. The parser scans each line for command detection - (ADR-0001). -- `accumulating`: collecting continuation-mode arguments for a command (ADR-0002). Each line is - tested for continuation markers, empty-line terminators, fence openers, or literal content. -- `inFence`: collecting raw lines inside a fenced block for a command (ADR-0003). All lines are - literal until a closing fence is detected. - -### Tracked state per command - -While processing the current command, the parser tracks: - -- `name`: the command name from the first line. -- `arguments.header`: the header text after the command name on the first line, before any fence - opener. -- `arguments.mode`: one of `single-line`, `continuation`, or `fence`. -- `arguments.fence_lang`: optional language identifier when in fence mode. -- `arguments.payload`: the assembled argument string, with newlines between logical payload lines. - -### Multi-command scanning - -The parser walks the normalized input (ADR-0010) line by line: - -1. In `idle`, when it encounters a command line, it starts a new command according to argument mode - rules (ADR-0005 for single-line, ADR-0002 for continuation, ADR-0003 for fenced). -2. After a command is finalized (single-line completion, empty-line termination in continuation, or - fence close), the parser returns to `idle` and continues scanning. -3. Non-command lines encountered in `idle` are collected into text blocks. Each text block records - its line range (start_line, end_line inclusive) and content with internal newline separators - reflecting the normalized input. -4. Commands and text blocks are emitted in document order. - -### No AST - -The parser emits a flat list of commands and text blocks directly. There is no intermediate abstract -syntax tree. The `children` field on commands is reserved for future hierarchical structures but is -currently always empty. - -## Consequences - -- The single-pass, line-based approach is simple to implement, test, and reason about. Each state - has a small, well-defined set of transitions. -- Memory usage is proportional to the largest single command payload, not the entire document, since - commands are finalized and emitted incrementally. -- No AST means no tree-manipulation overhead. Consumers work directly with the flat command and text - block lists. -- Adding new states or argument modes in the future requires extending the state machine rather than - restructuring a grammar. -- The three-state design maps directly to the three argument modes, making the correspondence - between spec and implementation transparent. diff --git a/docs/adrs/0005-opaque-argument-payload-with-command-level-parsing.md b/docs/adrs/0005-opaque-argument-payload-with-command-level-parsing.md deleted file mode 100644 index d4f9232..0000000 --- a/docs/adrs/0005-opaque-argument-payload-with-command-level-parsing.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -number: 5 -title: Opaque Argument Payload with Command-Level Parsing -date: 2026-03-13 -status: accepted ---- - -# 5. Opaque Argument Payload with Command-Level Parsing - -Date: 2026-03-13 - -## Status - -Accepted - -## Context - -Different commands have wildly different argument shapes: some take a single keyword, others take -key-value pairs, and others need entire JSON or YAML documents. If the parser tried to understand -argument structure (quoting, escaping, key-value syntax), it would need to know every command's -grammar. This couples the parser to command definitions and makes the format difficult to extend. - -The parser needs to deliver arguments to commands as an opaque payload, letting each command define -its own interpretation. - -## Decision - -The parser treats all argument content as opaque byte sequences. It determines argument boundaries -and transport mode but never interprets argument semantics. - -### Argument structure - -Every command carries an `arguments` object with the following fields: - -- `header`: the text after the command name on the first line, before any fence opener. This is the - inline portion of the arguments. -- `mode`: one of `single-line`, `continuation`, or `fence`, indicating how the payload was - assembled. -- `fence_lang`: the language tag from a fence opener (e.g., `jsonl`), or null if not in fence mode. -- `payload`: the final assembled argument string with newlines between logical payload lines. - -### Three argument modes - -The parser supports exactly three modes for assembling arguments: - -1. Single-line mode (this ADR): the command and all its arguments fit on one line. -2. Continuation mode (ADR-0002): arguments span multiple lines via trailing `/` markers. -3. Fence mode (ADR-0003): arguments are wrapped in backtick fences for verbatim transport. - -### Single-line mode - -A command is single-line if its first line does not end with a continuation marker (`/` per -ADR-0002) and does not contain a fence opener (per ADR-0003). - -In single-line mode: - -- The text after the command name, trimmed of the initial separating space, is the entire argument. -- The command is finalized immediately on that line. -- `arguments.mode` is set to `single-line`. -- `arguments.payload` is exactly that argument string. The parser does not append a trailing - newline. - -Example: `/echo hello world` produces `arguments.payload = "hello world"` with -`arguments.mode = "single-line"`. - -### Command-level parsing - -The parser's job ends at delivering the opaque payload. Each command implementation is responsible -for interpreting its own payload content (splitting on spaces, parsing JSON, etc.). The parser makes -no assumptions about quoting, escaping, or structure within the payload. - -## Consequences - -- The parser is command-agnostic. New commands can be added without modifying parser logic. -- Argument parsing errors are the command's responsibility, not the parser's, which simplifies error - boundaries. -- The three-mode system gives authors a clear choice: single-line for simple arguments, continuation - for moderate multi-line, and fenced blocks for complex or ambiguous payloads. -- The `header` field preserves inline arguments even when a fence is used, allowing commands to - accept both positional arguments and a fenced body (e.g., ``/mcp call_tool write_file ```jsonl``). -- No quoting or escaping conventions are imposed by the parser, avoiding conflicts with - payload-specific syntax like JSON strings or shell expressions. -- Textual reconstruction (format_text) guarantees spec-level semantic roundtrip fidelity, not - byte-for-byte reproduction of the original input. Line ending normalization (ADR-0010), whitespace - handling, and continuation marker stripping are lossy transformations. The roundtrip invariant is: - `parse(to_plaintext(parse(input)))` produces structurally equivalent commands and text blocks - (same names, modes, payloads, and text block content), not identical bytes. diff --git a/docs/adrs/0006-repository-structure-and-project-boundaries.md b/docs/adrs/0006-repository-structure-and-project-boundaries.md deleted file mode 100644 index f5f7bf4..0000000 --- a/docs/adrs/0006-repository-structure-and-project-boundaries.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -number: 6 -title: Repository Structure and Project Boundaries -date: 2026-03-14 -status: accepted ---- - -# 6. Repository Structure and Project Boundaries - -Date: 2026-03-14 - -## Status - -Accepted - -## Context - -The slash project encompasses a Rust parser core, WASM build targets, language-specific SDKs, a CLI -tool (riff), distribution packaging, a documentation website, and a formal spec. - -Deep directory nesting (e.g., `core/parser/`, `sdk/slash-python/`, `cli/riff/`) adds navigation -friction. A flat layout with a strict naming convention communicates category membership and -dependency tiers without requiring a nested filesystem hierarchy. - -Naming: "slash" is the format and project name (80s rock reference). "riff" is the CLI tool (guitar -riff; it reads stdin, writes stdout, stays simple). All SDK packages are prefixed `slash-` for -discoverability in package registries. All CLI packaging projects are prefixed `riff-`. - -Because this is a polyglot monorepo, we need strict architectural constraints to ensure SDKs don't -accidentally import the CLI, and that WASM builds don't depend on downstream consumers. The -mechanisms for enforcing these constraints (e.g., build system tooling) will be handled in a -separate tooling ADR. - -## Decision - -The repository uses a flat, single-level project layout. Logical grouping and dependency enforcement -are handled via project metadata and tags, not directory depth. - -### Project Layout - -```text -wit/ core WIT interface definitions (single source of truth) -parser/ core Rust library, internal to repo, not published -wasm-javascript/ wasm wasm-bindgen module for JS/TS runtimes -wasm-wasi/ wasm WASI component for polyglot SDK consumption -slash-web/ sdk Browser runtime SDK (depends on wasm-javascript) -slash-javascript/ sdk ESM server-side SDK (depends on wasm-javascript) -slash-python/ sdk Python SDK (depends on wasm-wasi) -slash-ruby/ sdk Ruby SDK (depends on wasm-wasi) -slash-php/ sdk PHP SDK (depends on wasm-wasi) -slash-elixir/ sdk Elixir SDK (depends on wasm-wasi) -slash-ocaml/ sdk OCaml SDK (depends on wasm-wasi) -slash-haskell/ sdk Haskell SDK (depends on wasm-wasi) -slash-dart/ sdk Dart SDK (depends on wasm-wasi) -slash-java/ sdk Java SDK (depends on wasm-wasi) -slash-go/ sdk Go SDK (depends on wasm-wasi) -slash-zig/ sdk Zig SDK (native FFI or wasm-wasi) -slash-rust/ sdk Rust SDK, thin published crate wrapping parser -riff-cli/ cli CLI binary (depends on slash-rust) -riff-deb/ pkg Debian package -riff-rpm/ pkg RPM package -riff-oci/ pkg OCI container image -riff-proto/ pkg Proto toolchain plugin -website/ docs Static site for documentation and promotion -docs/ docs Formal specifications -docs/adrs/ docs Architecture Decision Records -``` - -### Module Boundaries - -Projects are grouped into six distinct tags/tiers to define architectural boundaries. Build tooling -must enforce that dependencies only flow downward through these layers: - -- `core`: the internal parser library and WIT definitions. Depends on nothing. -- `wasm`: WASM build targets that compile `parser`. Can only depend on `core`. -- `sdk`: language-specific packages wrapping a WASM module (or native FFI). Can only depend on - `wasm` and `core`. -- `cli`: the riff binary. Can only depend on `sdk` and `core`. -- `pkg`: distribution packaging for riff. Can only depend on `cli`. -- `docs`: website, specifications, and ADRs. Can only depend on `docs`. - -### Dependency Flow and The WIT Contract - -```text -wit -> parser -parser -> wasm-javascript -> slash-javascript, slash-web -parser -> wasm-wasi -> slash-python, slash-ruby, slash-php, ... -parser -> slash-rust -> riff-cli -> riff-deb, riff-rpm, riff-oci, riff-proto -docs -> website -``` - -The `wit/` directory contains the WebAssembly Interface Types definitions, acting as the single -source of truth for the parser's API surface. Language SDKs generate types and thin wrappers based -on this contract. - -### Internal vs Published Code - -`parser/` is an internal Rust crate. It is consumed only by `wasm-javascript`, `wasm-wasi`, and -`slash-rust` within this repo. External Rust consumers use the `slash-rust` crate, which provides -the stable public API. - -To keep the repository root clean, all fuzzing, profiling, and testing infrastructure for the core -engine must live entirely within the `parser/` directory, avoiding root-level artifact clutter. - -## Consequences - -- Every project is one `cd` from root. No navigating through intermediate grouping directories. -- Prefix conventions (`slash-*`, `riff-*`, `wasm-*`) make filesystem navigation and glob-based - targeting self-documenting. -- The `parser` crate stays internal, giving freedom to change its API without semver concerns. Only - `slash-rust` carries the public contract. -- Adding a new language SDK requires only a single new root directory; no restructuring is needed. -- Clear structural boundaries physically prevent architectural regressions (e.g., an SDK cannot - accidentally depend on the CLI). -- The specification and ADRs live in `docs/` alongside the code. The website project pulls from - `docs/` at build time, ensuring documentation is version-controlled with the codebase. diff --git a/docs/adrs/0007-build-tooling-and-toolchain-management.md b/docs/adrs/0007-build-tooling-and-toolchain-management.md deleted file mode 100644 index a9d58e6..0000000 --- a/docs/adrs/0007-build-tooling-and-toolchain-management.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -number: 7 -title: Build Tooling and Toolchain Management -date: 2026-03-14 -status: accepted ---- - -# 7. Build Tooling and Toolchain Management - -Date: 2026-03-14 - -## Status - -Accepted - -## Context - -The slash project is a polyglot monorepo containing Rust (core parser), WebAssembly components, and -multiple language-specific SDKs. Managing this requires a build system that can orchestrate a -directed acyclic graph (DAG) of tasks across different language ecosystems. We also need tooling to -physically enforce the architectural boundaries established in ADR 6, both between projects (macro) -and within projects (micro). - -Furthermore, we must prevent "works on my machine" issues by guaranteeing that all contributors and -CI runners use the exact same toolchain versions, while preventing our monorepo orchestrator from -getting into a tug-of-war with workspace-aware tools like Cargo and pnpm. - -## Decision - -We will use Moon as our primary task orchestrator and monorepo manager, alongside its companion tool -`proto` for toolchain management. Moon will wrap, rather than replace, the native tools for each -language ecosystem. - -### Boundary Enforcement (Macro vs Micro) - -Tooling will explicitly enforce the architectural constraints defined in ADR 6 at two levels: - -1. **Macro-Boundaries (Project Level):** We use Moon's `tagRelationships` in `.moon/workspace.yml`. - Projects are tagged (`core`, `wasm`, `sdk`, `cli`, `pkg`, `docs`). If an `sdk` project attempts - to depend on a `cli` project, Moon will fail the build graph generation. This physically prevents - architectural regressions across the monorepo. - -2. **Micro-Boundaries (File Level):** Within individual projects (like the `parser/` crate), we use - ecosystem-specific tools configured as Moon lint tasks to enforce Clean Architecture: - - **Rust:** We use `clippy.toml` restrictions (`disallowed-methods`, `disallowed-types`) to - prevent Infrastructure IO and WASM types from leaking into the `src/domain/` and - `src/application/` modules. We may adopt `tach` in the future for stricter folder-to-folder DAG - enforcement. - - **TypeScript:** We use `eslint-plugin-boundaries` to prevent illegal cross-folder imports. - -### Toolchain Pinning (Proto) - -All toolchain versions are strictly pinned at the repository root: - -- `.prototools` acts as the single source of truth for exact version numbers (`rust = "1.85.0"`). -- `.moon/toolchains.yml` handles configuration behavior (e.g., auto-installing the `wasm32-wasip1` - target) but explicitly omits version numbers to prevent configuration drift. - -### Virtual Workspaces and Task Scoping - -To support shared lockfiles and efficient caching, we use root-level virtual workspaces (a root -`Cargo.toml` and `pnpm-workspace.yaml`). - -**The Task Scoping Rule:** To prevent Cargo and pnpm from attempting to build the entire monorepo -when invoked inside a specific project, all inherited Moon tasks must use scope flags injected with -Moon's `$project` variable (e.g., `cargo build -p $project` instead of bare `cargo build`). - -### Editor Configuration & Validation - -All Moon configuration files will utilize remote JSON `$schema` headers. To prevent the VS Code YAML -extension from reporting false-positive validation errors, contributors must use the -`moon: Append YAML schemas configuration to settings` command provided by the official Moon VS Code -extension. - -## Consequences - -- Complete reproducibility: anyone cloning the repo will automatically download the correct - compilers and toolchains via `proto`. -- Clean Architecture is structurally enforced: Moon tags prevent illegal cross-project imports, and - Clippy/ESLint prevent illegal intra-project imports. -- By using `$project` variables with scope flags (`-p`), Moon and Cargo cooperate peacefully rather - than fighting over workspace control. -- The use of `$schema` headers combined with the VS Code extension provides immediate IDE validation - and prevents silent configuration errors. diff --git a/docs/adrs/0008-sequential-zero-based-command-ids.md b/docs/adrs/0008-sequential-zero-based-command-ids.md deleted file mode 100644 index e15f761..0000000 --- a/docs/adrs/0008-sequential-zero-based-command-ids.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -number: 8 -title: Sequential Zero-Based Command IDs -date: 2026-03-13 -status: accepted ---- - -# 8. Sequential Zero-Based Command IDs - -Date: 2026-03-13 - -## Status - -Accepted - -## Context - -The parser output contains multiple commands and text blocks that consumers need to reference, -correlate, and process. Each element needs a stable, predictable identifier that conveys its type -and position within the document. UUIDs or random IDs would make output non-deterministic and harder -to test. Line numbers alone are fragile because they change when content is inserted or removed -above. - -## Decision - -Every command and text block receives a sequential, zero-based identifier with a type prefix. - -### ID format - -- Commands: `cmd-0`, `cmd-1`, `cmd-2`, etc. -- Text blocks: `text-0`, `text-1`, `text-2`, etc. - -The prefix indicates the element type. The numeric suffix is a zero-based counter that increments -independently for each type, in document order. - -### Assignment rules - -1. IDs are assigned in the order elements appear in the normalized input. -2. The command counter and text block counter are independent sequences, both starting at 0. -3. IDs are deterministic: the same input always produces the same IDs. - -### Range tracking - -In addition to the ID, every command and text block carries a `range` object with: - -- `start_line`: the first line of the element (inclusive). -- `end_line`: the last line of the element (inclusive). - -Line numbers provide a secondary reference for diagnostics and error reporting, complementing the -stable ID. - -## Consequences - -- Output is fully deterministic, making snapshot testing and diffing straightforward. -- The type prefix eliminates ambiguity when referencing elements (no confusion between command 1 and - text block 1). -- Zero-based indexing aligns with array indexing in most programming languages, simplifying consumer - code. -- Independent counters mean inserting a new text block does not change command IDs, and vice versa. -- The `range` field preserves source location for error reporting without overloading the ID with - positional information. diff --git a/docs/adrs/0009-jsonl-default-output-with-pretty-json-array-option.md b/docs/adrs/0009-jsonl-default-output-with-pretty-json-array-option.md deleted file mode 100644 index 5ef2fa7..0000000 --- a/docs/adrs/0009-jsonl-default-output-with-pretty-json-array-option.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -number: 9 -title: JSONL Default Output with Pretty JSON Array Option -date: 2026-03-13 -status: accepted ---- - -# 9. JSONL Default Output with Pretty JSON Array Option - -Date: 2026-03-13 - -## Status - -Accepted - -## Context - -The parser needs a serialization format for its output that is both machine-readable for downstream -tooling and human-inspectable for debugging. The output must represent the full result of a parse -run, including metadata, commands with their argument payloads, and interleaved text blocks. Two -common JSON-based approaches exist: a single JSON document (easy to pretty-print, harder to stream) -and JSONL (one object per line, streamable, harder to read). - -## Decision - -The parser supports two output formats, selectable via CLI flag (see ADR-0007): - -- JSONL (default): one JSON object per line, suitable for streaming and piping. -- Pretty JSON array: a single formatted JSON document, suitable for human inspection. - -### Output envelope - -A parser run produces a JSON object with the following top-level structure: - -```json -{ - "version": "0.1.0", - "context": { - "source": "string", - "timestamp": "2026-03-13T10:40:00Z", - "user": "string", - "session_id": "string", - "extra": {} - }, - "commands": [], - "text_blocks": [] -} -``` - -- `version`: semantic version of the output schema. -- `context`: metadata about the parse run. Fields `source`, `timestamp`, `user`, and `session_id` - are populated from CLI flags (ADR-0007) or defaults. `extra` is a free-form object for additional - metadata. -- `commands`: ordered array of Command objects. -- `text_blocks`: ordered array of TextBlock objects. - -### Command schema - -Each element in `commands` is: - -````json -{ - "id": "cmd-0", - "name": "mcp", - "raw": "/mcp call_tool write_file ```jsonl\n...\n```", - "range": { "start_line": 10, "end_line": 20 }, - "arguments": { - "header": "call_tool write_file", - "mode": "fence", - "fence_lang": "jsonl", - "payload": "{\n \"path\": \"...\"\n}" - }, - "children": [] -} -```` - -- `id`: unique identifier per ADR-0008 (`cmd-0`, `cmd-1`, etc.). -- `name`: command name without the leading `/`. -- `raw`: exact source slice for the command (header + all argument lines). -- `range`: inclusive line range the command covers (`start_line`, `end_line`). -- `arguments`: argument payload object per ADR-0005 with `header`, `mode`, `fence_lang`, and - `payload` fields. -- `children`: reserved for future hierarchical structures, currently always empty. - -### TextBlock schema - -Each element in `text_blocks` is: - -```json -{ "id": "text-0", "range": { "start_line": 0, "end_line": 9 }, "content": "arbitrary text\n..." } -``` - -- `id`: unique identifier per ADR-0008 (`text-0`, `text-1`, etc.). -- `range`: inclusive line range. -- `content`: exact text for the block with internal newline separators from the normalized input. - -## Consequences - -- JSONL as default enables streaming pipelines where consumers process commands as they are emitted - without waiting for the full document. -- Pretty JSON mode provides a single-document view for debugging and manual inspection. -- The envelope schema captures both parse results and run metadata, making output self-describing - and reproducible. -- The `raw` field on commands preserves the original source text for round-trip inspection or error - reporting. -- The `children` field is a forward-compatible extension point that does not affect current - consumers. -- Text blocks between commands are explicitly captured rather than discarded, ensuring no input - content is silently lost. diff --git a/docs/adrs/0010-input-model-and-line-ending-normalization.md b/docs/adrs/0010-input-model-and-line-ending-normalization.md deleted file mode 100644 index c3d0bd9..0000000 --- a/docs/adrs/0010-input-model-and-line-ending-normalization.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -number: 10 -title: Input Model and Line Ending Normalization -date: 2026-03-14 -status: accepted ---- - -# 10. Input Model and Line Ending Normalization - -Date: 2026-03-14 - -## Status - -Accepted - -## Context - -The parser consumes text input from various sources (files, stdin, editor buffers) that may use -different line ending conventions: CRLF (Windows), LF (Unix), or bare CR (legacy Mac). Without a -normalization step, every downstream rule (command detection, continuation markers, fence -boundaries) would need to handle all three variants independently, increasing complexity and bug -surface. - -A clear input model also needs to distinguish between structural line terminators (which delimit -parser lines) and literal backslash-n sequences that appear inside content such as JSON strings. - -## Decision - -The parser applies line ending normalization as a preprocessing step before any parsing logic runs. - -The normalization rules are: - -1. Replace all CRLF (\r\n) sequences with LF (\n). -2. Replace all remaining bare CR (\r) characters with LF (\n). - -After normalization: - -- All line terminators are LF regardless of the original encoding. -- The parser iterates over the normalized input using LF as the line separator. -- Each LF terminates a line; the LF itself is not part of the line content. -- A trailing LF at the end of input produces a trailing empty line. -- Literal escape sequences like `\\n` inside content (e.g., JSON strings `"blah\\nblah"`) are - ordinary characters, not line terminators. They are preserved verbatim in payloads. - -The parser then operates on a sequence of logical lines (possibly including blank lines) derived -solely from the normalized line terminators. - -## Consequences - -- All downstream parsing rules (command detection per ADR-0001, continuation per ADR-0002, fencing - per ADR-0003) operate on a uniform LF-only input model. -- Input from any OS line ending convention is handled identically with no special-casing in the - state machine (ADR-0004). -- The normalization is a single pass over the input, adding negligible overhead. -- Content-embedded escape sequences are never confused with structural line terminators. -- The preprocessing step is the first thing that runs, establishing a clean invariant that all - subsequent parsing logic can rely on. diff --git a/docs/slash-cmd parseing sematic.md b/docs/slash-cmd parseing sematic.md deleted file mode 100644 index 9650ddf..0000000 --- a/docs/slash-cmd parseing sematic.md +++ /dev/null @@ -1,314 +0,0 @@ -## 1. Input model and line normalization - -- The parser consumes a single text input as a byte/char stream. -- It treats certain characters as **line terminators** in the input stream, distinct from literal - `\n` characters that might appear inside string data. - -### 1.1 Line ending normalization - -Before any parsing: - -- Replace all `"\r\n"` with `"\n"`. -- Replace all remaining `"\r"` with `"\n"`. - -After normalization: - -- `CRLF`, `LF`, and bare `CR` are all treated equivalently as a single `LF`. -- The parser then **iterates over the input using these `\n` characters as separators** to obtain - lines. Conceptually: - - - Each `\n` in the normalized input terminates a line. - - The `\n` itself is not part of the line’s content. - - A final `\n` at the end of input produces a trailing empty line. - -- Literal `\n` sequences that appear inside line content (for example in JSON strings like - `"blah\\nblah"`) are **not** treated as line terminators. They are ordinary characters and are - preserved verbatim in the payload. - -Result: the parser operates on a sequence of logical lines (possibly including blank lines) derived -solely from the normalized line terminators. - -## 2. Command detection - -A **command line** is any line whose first non-whitespace character is `/`. - -Command line structure: - -```text -/[] -``` - -- ``: - - Regex: `[a-z][a-z0-9-]*`. - - Starts immediately after the leading `/`. - - Ends at the first whitespace or end-of-line. -- `` (optional): - - Everything after the first whitespace following ``. - - May contain: - - Inline arguments. - - An inline fence opener (```lang). - -Any line that does **not** begin (after leading whitespace) with `/` is a **non-command line**. - -## 3. Parser state machine - -The parser is line-based and uses three primary states: - -- `idle` – not currently accumulating a command. -- `accumulating` – collecting continuation-mode arguments for a command. -- `inFence` – collecting raw lines inside a fenced block for a command. - -For the **current command**, the parser tracks: - -- `name`: the command name from the first line. -- `arguments.header`: the header text after the command name on the first line, **before** any fence - opener. -- `arguments.mode`: one of `"single-line"`, `"continuation"`, or `"fence"`. -- `arguments.fence_lang`: optional language identifier when in fence mode. -- `arguments.payload`: the assembled argument string, with `\n` between logical payload lines. - -## 4. Argument modes - -### 4.1 Single-line mode - -If a command line: - -- Does **not** end with a continuation marker (see 4.2), and -- Does **not** contain a fence opener, - -then: - -- The text after `` (trimmed once for the initial space) is the entire argument. -- The command is finalized on that line. - -Properties: - -- `arguments.mode = "single-line"`. -- `arguments.payload` is exactly that argument string (the parser does not append a newline simply - for being the last argument). - -### 4.2 Continuation mode (explicit `" /"` marker) - -Lines that end with " /" are continuation markers and contribute payload lines as described. - -Lines that are completely empty end the command. - -Lines whose first non-whitespace is / but do not form a valid command name are treated as literal -payload lines. - -Lines that form a valid new command name are out of scope for this command (you can either treat -them as payload or define that they end the current command; your current implementation treats the -invalid / as payload). - -Continuation is **opt-in** and **strict**. - -#### 4.2.1 Continuation marker definition - -- A **continuation marker** is defined as a line (after normalization) whose content ends with a - **space + slash** (`" /"`) immediately before the line terminator, with nothing after that slash. -- Optional leading whitespace is allowed. For example, `"/echo /"` or `" /"` (space+slash with - indentation) both qualify as continuation markers. - -Formally, a line is a continuation marker if it matches: - -- `^[ \t]*.*[ ]/[ ]*$` and the last two non-newline characters are `" /"`. - -A special case of this is a line that, aside from leading whitespace, is exactly `" /"`; this -represents a **blank payload line** in continuation mode. - -#### 4.2.2 First command line with continuation - -For a command’s first line: - -- If it ends with `" /"` (continuation marker): - - The parser **strips** the final `" /"` from that line’s content. - - The remaining content (which might be empty after the header) is appended to - `arguments.payload`, followed by `"\n"` if you treat it as part of the payload body. - - `arguments.mode` is set to `"continuation"`. - - The parser moves to `accumulating`. - -- If it does **not** end with `" /"` and has no fence opener: - - The parser treats the command as a single-line command (4.1). - -#### 4.2.3 Lines in `accumulating` state - -In `accumulating`, for each subsequent line: - -Let `line` be the current line content (without the trailing `\n`): - -- **Continuation marker line** (matches the pattern, ends in `" /"` with no content after it): - - This represents a **blank payload line**. - - The parser appends `"\n"` to `arguments.payload`. - - The parser remains in `accumulating`. - -- **Empty line** (`line == ""` after normalization): - - This is a **true blank line** with no continuation marker. - - The parser **finalizes the command** and transitions to `idle`. - - This blank line is **not** appended to `arguments.payload`. - - Subsequent lines are parsed independently (either as text or new commands). - -- **Any other non-empty line** (not a marker): - - The parser appends `line + "\n"` to `arguments.payload`. - - The parser remains in `accumulating`. - -#### 4.2.4 Bare slash is not continuation - -- A bare `/` at the end of a line (with no preceding space as `" /"`) is **not** a continuation - marker. - -Behavior: - -- If the line’s first non-whitespace character is `/`, it is processed via command detection (either - a new command or invalid command name). -- If the `/` appears elsewhere in the line, it is just part of the literal content. -- The parser does not introduce any special handling for this edge case; users are encouraged to use - fenced blocks when they need more complex or ambiguous multi-line payloads. - -**Implication:** examples like: - -```text -/echo / -ooga booga -/ -testing 123 -``` - -do **not** cause `testing 123` to be included in `/echo`’s payload, because the line with just `/` -is not a `" /"` marker and either starts a new command or is treated as content depending on -position. - -### 4.3 Fenced block mode - -Fenced blocks allow attaching a raw, multi-line payload to a command, similar to markdown code -fences. - -#### 4.3.1 Fence openers - -Two ways to enter fence mode: - -1. **Inline fence on the command line** - - ````text - / ```[lang] - ```` - - - In ``, the first occurrence of three or more consecutive backticks (```…) is - treated as the fence opener. - - Text before the opener is kept in `arguments.header`. - - The parser records: - - Fence marker length (number of backticks). - - Optional language identifier (e.g., `jsonl`). - - The parser enters `inFence` state. - -2. **Fence on the next line after continuation** - - ````text - / / - ```[lang] - - ```` - ``` - - First line ends with `" /"` → command enters `continuation` mode, but the next line is recognized as a fence opener. - - The parser transitions to `inFence` for this command. - - Subsequent lines until the fence close are payload lines for this command. - ``` - -#### 4.3.2 Fence mode semantics - -While in `inFence`: - -- All lines (including blank lines) are appended to `arguments.payload` with `"\n"` separators. -- Continuation markers (`" /"`) inside a fence are ignored as syntax and treated as literal content. -- The parser looks for a **closing fence**: - - - After trimming leading/trailing whitespace, a line is a closing fence if: - - It consists solely of backticks. - - The number of backticks is **greater than or equal** to the opener’s count. - -- The closing fence line itself is **not** appended to `arguments.payload`. -- Once the closing fence is found: - - `arguments.mode = "fence"`. - - `arguments.fence_lang` is the captured language (`lang`) or `null`. - - The command is finalized, and the parser transitions back to `idle`. - -## 5. Multiple commands and text blocks - -- The parser walks the normalized input line by line. -- In `idle`, when it encounters a command line, it starts a new command according to the rules - above. -- After a command is finalized (single-line, continuation termination, or fence close), the parser - returns to `idle` and continues scanning for the next command. -- Non-command text segments between commands can optionally be represented as `text_blocks`: - - - Each `text_block` has: - - `range.start_line`, `range.end_line` (inclusive, zero-based or one-based by convention). - - `content`: the exact text for that block, with internal `\n` separators reflecting the - normalized input. - -## 6. JSON output schematics (summary) - -A parser run produces a JSON object: - -````json -{ - "version": "0.1.0", - "context": { - "source": "string", - "timestamp": "2026-03-13T10:40:00Z", - "user": "string", - "session_id": "string", - "extra": {} - }, - "commands": [ /* Command[] */ ], - "text_blocks": [ /* TextBlock[] */ ] -} -```[] - -### 6.1 Command - -Each `commands[i]` is: - -```json -{ - "id": "cmd-1", - "name": "mcp", - "raw": "/mcp call_tool write_file ```jsonl\n...\n```", - "range": { - "start_line": 10, - "end_line": 20 - }, - "arguments": { - "header": "call_tool write_file", - "mode": "fence", - "fence_lang": "jsonl", - "payload": "{\n \"path\": \"...\"\n}" - }, - "children": [] -} -```[] - -- `id`: unique identifier for this command instance. -- `name`: command name (without the leading `/`). -- `raw`: exact source slice for this command (header + argument lines). -- `range`: inclusive line range that the command covers. -- `arguments`: - - `header`: header arguments from the command line before any fence opener. - - `mode`: `"single-line" | "continuation" | "fence"`. - - `fence_lang`: language tag or `null`. - - `payload`: final assembled argument string with `\n` between logical payload lines. -- `children`: reserved for future hierarchical structures (may be empty). - -### 6.2 TextBlock - -Each `text_blocks[i]` is: - -```json -{ - "id": "text-1", - "range": { - "start_line": 0, - "end_line": 9 - }, - "content": "arbitrary text\n..." -} -```` diff --git a/docs/slash-parser CLI Specification.md b/docs/slash-parser CLI Specification.md index f5fd861..4ff3199 100644 --- a/docs/slash-parser CLI Specification.md +++ b/docs/slash-parser CLI Specification.md @@ -1,6 +1,4 @@ - - -## `riff` CLI Specification +## `riff` CLI Specification - WIP ### 1. Overview @@ -229,17 +227,4 @@ riff -p -c '{"user":"tom","run_id":"abc-123"}' ./prompt.md # Context from multiple sources merged together riff -c ./defaults.json -c ./overrides.env -c debug=true ./prompt.md -``` - -parser/ core Rust library, internal to repo, not published wasm-js/ wasm wasm-bindgen module for -JS/TS runtimes wasm-wasi/ wasm WASI module for polyglot SDK consumption slash-js-web/ sdk Browser -runtime SDK (depends on wasm-js) slash-js-bundle/ sdk ESM server-side SDK (depends on wasm-js) -slash-py/ sdk Python SDK (depends on wasm-wasi) slash-ruby/ sdk Ruby SDK (depends on wasm-wasi) -slash-php/ sdk PHP SDK (depends on wasm-wasi) slash-elixir/ sdk Elixir SDK (depends on wasm-wasi) -slash-ocaml/ sdk OCaml SDK (depends on wasm-wasi) slash-haskell/ sdk Haskell SDK (depends on -wasm-wasi) slash-dart/ sdk Dart SDK (depends on wasm-wasi) slash-java/ sdk Java SDK (depends on -wasm-wasi) slash-go/ sdk Go SDK (depends on wasm-wasi) slash-zig/ sdk Zig SDK (native FFI or -wasm-wasi) slash-rs/ sdk Rust SDK, thin published crate wrapping parser riff/ cli CLI binary -(depends on slash-rs) riff-deb/ pkg Debian package riff-rpm/ pkg RPM package riff-oci/ pkg OCI -container image riff-proto/ pkg Proto toolchain plugin website/ docs Static site for documentation -and promotion docs/ docs ADRs and formal spec (bundled into website) +``` \ No newline at end of file diff --git a/docs/slash-parser-spec-v0.3.0.md b/docs/slash-parser-spec-v0.3.0.md new file mode 100644 index 0000000..413c3be --- /dev/null +++ b/docs/slash-parser-spec-v0.3.0.md @@ -0,0 +1,1096 @@ +# Slash Command Parser Specification + +Version: 0.3.0 Date: 2026-03-18 + +## 1. Overview + +The Slash Command Parser consumes a single UTF-8 text input and produces a structured JSON result +containing all detected slash commands, interleaved text blocks, optional caller-supplied context, +and any parser warnings. + +The parser is deterministic and pure: given the same input and context, it always produces the same +output. It performs no I/O, maintains no global mutable state, and accepts all configuration through +its input parameters. + +### 1.1 Design Principles + +1. **Command-agnostic.** The parser never interprets argument content. It determines argument + boundaries and transport mode but treats all argument strings as opaque byte sequences. + Tokenization, quoting, key-value parsing, and any other semantic interpretation of arguments are + the sole responsibility of the command implementation. + +2. **Flat output, no AST.** The parser emits a flat list of commands and text blocks in document + order. There is no intermediate abstract syntax tree. + +3. **Incremental emission.** A conforming implementation must be capable of finalizing and emitting + each command or text block as soon as its last line has been consumed, without buffering the + entire document. Memory usage should be bounded by the size of the largest single command payload + or text block, not the total input size. + +4. **Single forward pass.** The parser processes the input in a single left-to-right pass (after + normalization and line joining). It never backtracks. + +## 2. Input Model + +### 2.1 Line Ending Normalization + +Before any other processing: + +1. Replace all `\r\n` (CRLF) sequences with `\n` (LF). +2. Replace all remaining `\r` (bare CR) characters with `\n`. + +After normalization, all line terminators are LF. The parser splits the normalized input on `\n` to +produce a sequence of physical lines. Each `\n` terminates a line and is not part of the line +content. A trailing `\n` at the end of input produces a trailing empty line. + +Literal escape sequences inside content (e.g., `\n` in a JSON string `"blah\nblah"`) are ordinary +characters, not line terminators. They are preserved verbatim in the output. + +### 2.2 Line Joining (Backslash Continuation) + +After line-ending normalization and before command parsing, the parser performs a line-joining +pre-pass. This mechanism is identical in behavior to POSIX shell backslash-newline removal. + +For each physical line, if the line ends with a backslash (`\`) character: + +1. Remove the trailing backslash. +2. Remove the line terminator (the split boundary). +3. Concatenate the remainder with the next physical line, separated by a single space. +4. Repeat: if the joined result still ends with `\`, continue joining with subsequent lines. + +Lines that do not end with `\` are left unchanged. + +The join marker is any backslash character immediately before the physical line terminator, +regardless of what precedes it. There is no requirement for a space or any other character before +the backslash. This includes lines that serve other syntactic roles, such as a closing fence line +followed by a trailing backslash (e.g., `` ``` \ ``). In all cases, a trailing `\` triggers line +joining. This matches POSIX shell behavior. + +After this pass, the parser operates on a sequence of logical lines. Each logical line maps back to +one or more physical lines from the original input. + +#### 2.2.1 Physical Line Tracking + +The parser must track the mapping from each logical line back to its original physical line range. +This mapping is used to populate `range.start_line` and `range.end_line` in the output, which always +refer to zero-based physical line numbers from the normalized input (before joining). + +#### 2.2.2 Trailing Backslash at EOF + +If the final physical line ends with a backslash and there is no subsequent line to join with, the +trailing backslash is removed and the line stands alone. This mirrors POSIX shell behavior where a +backslash-newline pair removes the newline; at EOF, the backslash simply disappears. + +Example: + +```text +/echo hello \ +``` + +Logical line: `/echo hello` + +The command parses as a single-line command with `arguments.payload` of `"hello "`. + +#### 2.2.3 Joining Examples + +Physical input (three physical lines): + +```text +/mcp call_tool read_file \ + --path src/index.ts \ + --format json +``` + +Logical line after joining: + +```text +/mcp call_tool read_file --path src/index.ts --format json +``` + +This logical line maps to physical lines 0 through 2. + +A line ending with a literal backslash that is not intended as a join marker should use a fenced +block instead (see Section 5.2). + +### 2.3 Fence Immunity + +Line joining does not apply inside fenced blocks. Once the parser enters fence mode (see Section +5.2), all physical lines are consumed verbatim until the closing fence or EOF. A trailing `\` inside +a fence is literal content, not a join marker. + +Because line joining is conceptually a pre-pass and fences are detected during command parsing, the +implementation must either (a) perform joining lazily during parsing, skipping lines inside fences, +or (b) use a two-pass approach where fence boundaries are identified first. Either strategy is +acceptable as long as fence content is never subject to line joining. + +## 3. Command Detection + +A command line is any logical line (after normalization and joining) whose first non-whitespace +character is `/` (U+002F) and whose subsequent characters form a valid command name. + +### 3.1 Command Name + +The command name: + +- Starts immediately after the leading `/` with no intervening space. +- Ends at the first whitespace character or end-of-line. +- Must match the pattern `[a-z][a-z0-9-]*`: + - Begins with a lowercase ASCII letter (a-z). + - Followed by zero or more lowercase ASCII letters, ASCII digits, or hyphens. + +### 3.2 Invalid Slash Lines + +If a logical line's first non-whitespace character is `/` but the text after `/` does not match the +command name pattern (for example, a bare `/`, `/123`, `/Hello`, or `/ space`), the line is not a +command. In `idle` state, such lines are treated as ordinary text and may become part of a text +block (Section 6). In `in_fence` state, all lines (including invalid slash lines) are literal +payload content. + +### 3.3 Arguments + +Everything after the first whitespace following the command name is the arguments portion. The +whitespace between the command name and the arguments is consumed as a separator and is not included +in the arguments string. + +The arguments portion may be: + +- Empty (command with no arguments). +- Inline text (single-line mode, Section 5.1). +- A fence opener (fence mode, Section 5.2). +- Inline text followed by a fence opener (the text before the fence becomes the `header`, the fenced + content becomes the `payload`). + +### 3.4 The `header` Field + +The `header` field contains the inline, non-fenced argument portion of the command line. It serves +as the dispatch or routing segment of the command and is present in both argument modes: + +- In single-line mode, `header` and `payload` contain the same string (the full arguments text). +- In fence mode, `header` contains the arguments text that appears before the fence opener on the + command line. This is typically used for subcommand names and flags, while the fenced `payload` + carries bulk data such as JSON or code. + +## 4. Parser States + +The parser uses two states: + +- `idle`: not inside a fenced block. The parser scans logical lines for commands and collects + non-command lines into text blocks. +- `in_fence`: collecting raw physical lines inside a fenced block for the current command. + +There is no "accumulating" or "continuation" state. Multi-physical-line commands that are not fenced +are handled entirely by line joining (Section 2.2) before the state machine runs. + +### 4.1 State Transitions + +| Current State | Condition | Action | Next State | +| ------------- | ---------------------------------------------------- | --------------------------------------------- | ---------- | +| `idle` | Logical line is a valid command with no fence opener | Finalize as single-line command | `idle` | +| `idle` | Logical line is a valid command with a fence opener | Begin fence; record header and fence metadata | `in_fence` | +| `idle` | Logical line is not a command | Append to current text block | `idle` | +| `in_fence` | Physical line is a closing fence | Finalize fenced command | `idle` | +| `in_fence` | Physical line is not a closing fence | Append to payload | `in_fence` | +| `in_fence` | EOF reached without closing fence | Finalize command with warning | `idle` | + +## 5. Argument Modes + +### 5.1 Single-Line Mode + +If a command's arguments portion (after joining) does not contain a fence opener: + +- The full arguments text is stored in both `header` and `payload`. +- `mode` is `"single-line"`. +- `fence_lang` is `null`. +- The command is finalized on that logical line. + +Example: + +```text +/deploy production --region us-west-2 +``` + +Result: + +- `name`: `"deploy"` +- `arguments.header`: `"production --region us-west-2"` +- `arguments.mode`: `"single-line"` +- `arguments.fence_lang`: `null` +- `arguments.payload`: `"production --region us-west-2"` + +### 5.2 Fence Mode + +Fenced blocks allow attaching a raw, multi-line payload to a command using markdown-style code fence +syntax. Fenced payloads are completely verbatim: no parsing rules (including line joining) apply to +content lines, eliminating escaping concerns. + +Only backtick (`` ` ``) fences are recognized. Tilde (`~`) fences are not supported. + +#### 5.2.1 Fence Opener + +In the arguments portion of a command line, the first occurrence of three or more consecutive +backtick characters is treated as a fence opener. + +- Text before the backtick run (trimmed of trailing whitespace) becomes `arguments.header`. +- The backtick run length is recorded as the fence marker length. +- Text after the backtick run (trimmed of leading whitespace), if non-empty and consisting of a + single token (no internal whitespace), is the optional language identifier stored in + `arguments.fence_lang`. +- The parser transitions to `in_fence`. + +The variable-length backtick fence (three or more) means content containing triple backticks can be +fenced with four or more backticks, avoiding collision. + +#### 5.2.2 Fence Body + +While in `in_fence`, the parser reads physical lines from the normalized input (not joined logical +lines). All lines are appended to the payload verbatim, preserving their original content including +any trailing backslashes. + +Lines are joined in the payload with `\n` separators. The payload does not include a trailing `\n` +after the last content line. + +#### 5.2.3 Fence Lifetime + +A fenced argument block extends from the fence opener line through either: + +- The first closing fence line, or +- End of input (EOF), in which case the fence is considered unclosed and a warning is emitted (see + Section 5.2.5). + +There is no other mechanism that terminates a fence. Inside a fence, command triggers, invalid slash +lines, blank lines, and all other content are literal payload. + +#### 5.2.4 Fence Closer + +A physical line is a closing fence if, after trimming leading and trailing whitespace: + +- It consists solely of backtick characters. +- The number of backticks is greater than or equal to the opener's backtick count. + +The closing fence line is not included in the payload. Once found, the parser finalizes the command +and returns to `idle`. + +Note that line joining still applies to lines outside fences. If a closing fence line ends with a +trailing backslash (e.g., `` ``` \ ``), the fence closes normally, and then the backslash triggers a +line join between the closing fence and the next physical line. The joined result is outside the +fenced command and is parsed normally (see Appendix A.8 for an example). + +#### 5.2.5 Unclosed Fence + +If the input ends before a closing fence is found: + +- The parser finalizes the command with whatever payload has been accumulated through EOF. +- A warning object with `"type": "unclosed_fence"` and `"start_line"` set to the fence opener's + physical line number is added to the `warnings` array in the result envelope. + +The parser emits the command with its partial payload. Consumers that require strictly well-formed +fenced payloads SHOULD treat `unclosed_fence` warnings as hard errors and reject the affected +command. + +#### 5.2.6 Backslash Joining Around Fences + +Backslash line joining applies to physical lines outside of fences. This means joining can merge +lines before the fence opener and lines after the fence closer, but never lines inside the fence +body (see Section 2.3). + +Joining into a fence opener. When backslash joining merges a command line with a line containing a +fence opener, the fence is detected in the resulting logical line. This is the natural way to place +a fence opener on a separate physical line from the command name. + +Example (four physical lines): + +/mcp call_tool write_file \ + +```json +{ "path": "foo" } +``` + +After joining, physical lines 0 and 1 become one logical line: + +/mcp call_tool write_file ```json + +The backtick run is detected as a fence opener. arguments.header is "call_tool write_file", +arguments.fence_lang is "json", and the JSON body is collected as the fenced payload. + +Joining after a fence closer. Once the closing fence is found and the parser returns to idle, any +subsequent physical lines are again subject to normal line joining. If lines following the closing +fence end with trailing backslashes, they join with each other as usual. + +Example (six physical lines): + +/mcp call_tool write_file -c\ + +```json +{ "path": "foo" } +``` + +\ +production + + Lines 0 and 1 join into: /mcp call_tool write_file -c ```json + + Lines 2 and 3 are inside and closing the fence (no joining applies). + + Lines 4 and 5 join into: production (the backslash on line 4 is removed and line 5 is + appended). + +The fenced command has header of "call_tool write_file -c", fence_lang of "json", and payload of "{ +\"path\": \"foo\" }". The joined production line is outside the command and becomes a text block. +See Appendix A.9 for the full output. + +## 6. Text Blocks + +Non-command logical lines encountered in `idle` state are collected into text blocks. + +- Consecutive non-command lines form a single text block. +- Blank lines that are part of a text region are included in the text block content. +- Text block content preserves the original lines joined with `\n` separators. +- Text blocks use physical line numbers for their `range`. +- A new text block begins after a command is finalized, if non-command lines follow. + +Text blocks exist so that: + +- The roundtrip invariant (Section 9.4) can be satisfied: a formatter needs to know where + non-command regions are to reconstruct the input. +- Tooling can distinguish "what will run" from "what is commentary" when rendering or analyzing + documents that mix commands with prose. + +Consumers that do not need text blocks may ignore the `text_blocks` array. + +## 7. Multiple Commands and Ordering + +The parser walks the input line by line: + +1. In `idle`, when it encounters a command line, it starts a new command according to the argument + mode rules (Section 5). +2. After a command is finalized, the parser returns to `idle` and continues scanning. +3. Non-command lines in `idle` are collected into text blocks (Section 6). + +Commands are assigned sequential zero-based IDs in encounter order: `cmd-0`, `cmd-1`, `cmd-2`, and +so on. Text blocks are assigned IDs independently in the same manner: `text-0`, `text-1`, `text-2`, +and so on. + +Commands and text blocks appear in the output arrays in document order. + +## 8. Output Format + +### 8.1 Envelope + +A parser run produces a single JSON object (the "envelope") conforming to the `SlashParserResult` +schema defined in Section 8.5. The envelope contains: + +- `version`: the spec version (`"0.3.0"`). +- `context`: caller-supplied metadata passed through to the output unchanged. +- `commands`: an ordered array of all detected commands. +- `text_blocks`: an ordered array of all non-command text regions. +- `warnings`: an array of parser warnings (e.g., unclosed fences). Empty if no warnings. + +The parser is a total function: it always produces a valid envelope for any input. There is no input +that causes the parser to fail or return an error instead of an envelope. An empty input produces an +envelope with empty `commands`, `text_blocks`, and `warnings` arrays. Malformed constructs (such as +unclosed fences) produce commands with partial data and corresponding entries in the `warnings` +array, never a parse failure. + +### 8.2 The `raw` Field + +The `raw` field on each command contains the exact source text for that command as it appeared in +the normalized input (after line-ending normalization but before line joining). For single-line +commands that span multiple physical lines via backslash joining, `raw` contains all the physical +lines including the backslash characters and the `\n` separators between them. For fenced commands, +`raw` includes the command line, all fence body lines, and the closing fence line (if present). + +### 8.3 JSONL Streaming Mode + +When a parser implementation supports JSON Lines (JSONL) output, each line of the JSONL stream is +exactly one complete `SlashParserResult` envelope, encoded as a single JSON object on one line. + +JSONL mode does not emit one command per line. Commands remain elements of the `commands` array +inside each envelope. In a pipeline processing multiple input files, each input produces one JSONL +line. + +This design was chosen because: + +- Each JSONL line is a self-contained parse result with commands, text blocks, warnings, and + context, making each line independently meaningful. +- Command IDs (`cmd-0`, `cmd-1`) are scoped per envelope, so per-envelope lines maintain stable ID + semantics. +- Consumers can treat each line as a complete unit of work corresponding to one input document. + +Per-command JSONL (one command per line) is out of scope for this specification. Implementations may +offer it as a non-standard extension. + +### 8.4 Envelope Example + +```json +{ + "version": "0.3.0", + "context": { "source": "deploy.md", "user": "tom" }, + "commands": [ + { + "id": "cmd-0", + "name": "deploy", + "raw": "/deploy production \\\n --region us-west-2 \\\n --canary", + "range": { "start_line": 0, "end_line": 2 }, + "arguments": { + "header": "production --region us-west-2 --canary", + "mode": "single-line", + "fence_lang": null, + "payload": "production --region us-west-2 --canary" + } + } + ], + "text_blocks": [ + { "id": "text-0", "range": { "start_line": 3, "end_line": 3 }, "content": "Deploy initiated." } + ], + "warnings": [] +} +``` + +### 8.5 JSON Schema + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://slash-parser.dev/schema/v0.3.0/parse-result.json", + "title": "SlashParserResult", + "description": "Output envelope for the Slash Command Parser v0.3.0.", + "type": "object", + "required": ["version", "context", "commands", "text_blocks", "warnings"], + "additionalProperties": false, + "properties": { + "version": { + "type": "string", + "const": "0.3.0", + "description": "Specification version that produced this result." + }, + "context": { + "type": "object", + "description": "Caller-supplied metadata passed through to the output unchanged.", + "additionalProperties": true, + "properties": { + "source": { + "type": ["string", "null"], + "description": "Identifier for the input source (e.g., filename, URI)." + }, + "timestamp": { + "type": ["string", "null"], + "description": "ISO 8601 timestamp, if provided by the caller." + }, + "user": { + "type": ["string", "null"], + "description": "User identifier, if provided by the caller." + }, + "session_id": { + "type": ["string", "null"], + "description": "Session identifier, if provided by the caller." + }, + "extra": { + "type": ["object", "null"], + "description": "Arbitrary additional metadata.", + "additionalProperties": true + } + } + }, + "commands": { + "type": "array", + "description": "Ordered array of all detected commands, in document order.", + "items": { "$ref": "#/$defs/Command" } + }, + "text_blocks": { + "type": "array", + "description": "Ordered array of non-command text regions, in document order.", + "items": { "$ref": "#/$defs/TextBlock" } + }, + "warnings": { + "type": "array", + "description": "Parser warnings (e.g., unclosed fences). Empty array if no warnings.", + "items": { "$ref": "#/$defs/Warning" } + } + }, + "$defs": { + "LineRange": { + "type": "object", + "description": "Zero-based physical line range (inclusive on both ends).", + "required": ["start_line", "end_line"], + "additionalProperties": false, + "properties": { + "start_line": { + "type": "integer", + "minimum": 0, + "description": "First physical line (zero-based) covered by this element." + }, + "end_line": { + "type": "integer", + "minimum": 0, + "description": "Last physical line (zero-based) covered by this element." + } + } + }, + "CommandArguments": { + "type": "object", + "description": "Parsed argument structure for a command. Content is opaque to the parser.", + "required": ["header", "mode", "fence_lang", "payload"], + "additionalProperties": false, + "properties": { + "header": { + "type": "string", + "description": "Inline argument text from the command line, before any fence opener. Serves as the dispatch/routing portion in both single-line and fence modes." + }, + "mode": { + "type": "string", + "enum": ["single-line", "fence"], + "description": "How the arguments were assembled." + }, + "fence_lang": { + "type": ["string", "null"], + "description": "Language identifier from the fence opener, or null if not in fence mode or no language was specified." + }, + "payload": { + "type": "string", + "description": "Assembled argument content. In single-line mode, identical to header. In fence mode, the verbatim content between the fence delimiters." + } + } + }, + "Command": { + "type": "object", + "description": "A single parsed slash command.", + "required": ["id", "name", "raw", "range", "arguments"], + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "pattern": "^cmd-[0-9]+$", + "description": "Sequential zero-based identifier (cmd-0, cmd-1, ...)." + }, + "name": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]*$", + "description": "Command name without the leading slash." + }, + "raw": { + "type": "string", + "description": "Exact source text from the normalized input (before line joining), including backslashes and physical newlines for joined commands, and fence delimiters for fenced commands." + }, + "range": { + "$ref": "#/$defs/LineRange", + "description": "Physical line range in the normalized input covered by this command." + }, + "arguments": { "$ref": "#/$defs/CommandArguments" } + } + }, + "TextBlock": { + "type": "object", + "description": "A contiguous region of non-command text.", + "required": ["id", "range", "content"], + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "pattern": "^text-[0-9]+$", + "description": "Sequential zero-based identifier (text-0, text-1, ...)." + }, + "range": { + "$ref": "#/$defs/LineRange", + "description": "Physical line range covered by this text block." + }, + "content": { + "type": "string", + "description": "Original text content with lines joined by newline separators." + } + } + }, + "Warning": { + "type": "object", + "description": "A parser warning indicating a non-fatal issue.", + "required": ["type"], + "additionalProperties": true, + "properties": { + "type": { + "type": "string", + "description": "Warning category identifier (e.g., 'unclosed_fence')." + }, + "start_line": { + "type": "integer", + "minimum": 0, + "description": "Physical line number where the issue was detected." + }, + "message": { + "type": "string", + "description": "Human-readable description of the warning." + } + } + } + } +} +``` + +### 8.6 Parser Engine Interface + +The parser engine exposes two entry points. Implementations may adapt the signatures to their host +language, but the semantics must be preserved. + +1. **Parse with default context.** Accepts a single UTF-8 string input. Returns a + `SlashParserResult` envelope with an empty `context` object. + +Logical signature: + + parse(input: string) → SlashParserResult + +2. **Parse with caller-supplied context.** Accepts a UTF-8 string input and a context object. The + context is passed through to the envelope unchanged. The parser does not interpret context + fields. + +Logical signature: + + parse(input: string, context: object) → SlashParserResult + +Both entry points are pure functions: no I/O, no global mutable state, no side effects. All +configuration is supplied through the input parameters. + +The return type is always `SlashParserResult` as defined in Section 8.5. Implementations must not +use exceptions, panics, or error return channels for parse outcomes. Warnings about malformed input +(such as unclosed fences) are reported inside the envelope's `warnings` array. + +## 9. Conformance + +### 9.1 No AST Requirement + +A conforming implementation must not expose or require an abstract syntax tree as a public artifact. +The only standardized observable output is the JSON envelope defined in Section 8. Implementations +may use any internal data structures, but the contract with consumers is the envelope schema alone. + +### 9.2 Incremental Emission + +A conforming implementation must support finalizing each command or text block as soon as its last +physical line has been consumed. Memory usage must be bounded by the size of the largest single +command payload or text block, not the total size of the input. + +### 9.3 Opaque Payload + +The parser is command-agnostic. It never interprets argument content, does not tokenize, and does +not apply quoting, escaping, or key-value semantics to the `header` or `payload` fields. All +argument strings are treated as opaque byte sequences. Any higher-level parsing (e.g., splitting on +spaces, parsing JSON or YAML) is the sole responsibility of the command implementation. + +### 9.4 Roundtrip Fidelity Invariant + +Let `P` be a conforming parser and `F` be a formatter that generates a canonical plaintext +representation from `P`'s JSON envelope output. The following invariant must hold: + +For any valid input `I`, parsing, formatting, and parsing again yields a structurally equivalent +result: + +``` +P(F(P(I))) ≡ P(I) +``` + +Structural equivalence means the same sequence of commands and text blocks with identical values for +`name`, `arguments.mode`, `arguments.header`, `arguments.payload`, `arguments.fence_lang`, and text +block `content`. + +The invariant is structural, not byte-for-byte: `raw` values, line numbers, and whitespace details +may differ between the two parse results due to normalization and line joining. These are allowed +lossy transformations. + +### 9.5 Determinism + +A conforming implementation must be deterministic: given identical input bytes and identical +context, the output envelope must be byte-for-byte identical across invocations and across +implementations. + +### 9.6 JSONL Envelope Semantics + +In JSONL streaming mode, each line of the output stream is exactly one `SlashParserResult` envelope +encoded as a single JSON object. One input unit (e.g., one file or one string passed to the parse +function) produces exactly one JSONL line. + +The `commands` and `text_blocks` arrays inside each envelope are ordered by document position. +Command IDs (`cmd-0`, `cmd-1`, ...) and text block IDs (`text-0`, `text-1`, ...) are scoped to their +envelope and reset for each input unit. + +Implementations that process multiple inputs in a pipeline emit one JSONL line per input, in the +order the inputs were supplied. The parser itself is not aware of the pipeline; the JSONL framing is +the responsibility of the host runtime or CLI layer. + +## Appendix A: Parsing Examples + +### A.1 Single-Line Command + +Input: + +```text +/echo hello world +``` + +Output (commands array element): + +```json +{ + "id": "cmd-0", + "name": "echo", + "raw": "/echo hello world", + "range": { "start_line": 0, "end_line": 0 }, + "arguments": { + "header": "hello world", + "mode": "single-line", + "fence_lang": null, + "payload": "hello world" + } +} +``` + +### A.2 Joined Multi-Line Command + +Input (three physical lines): + +```text +/deploy production \ + --region us-west-2 \ + --canary +``` + +After joining: `/deploy production --region us-west-2 --canary` + +Output: + +```json +{ + "id": "cmd-0", + "name": "deploy", + "raw": "/deploy production \\\n --region us-west-2 \\\n --canary", + "range": { "start_line": 0, "end_line": 2 }, + "arguments": { + "header": "production --region us-west-2 --canary", + "mode": "single-line", + "fence_lang": null, + "payload": "production --region us-west-2 --canary" + } +} +``` + +### A.3 Fenced Command with Header + +Input: + +````text +/mcp call_tool write_file ```json +{ "path": "/src/index.ts" } +``` +```` + +Output: + +````json +{ + "id": "cmd-0", + "name": "mcp", + "raw": "/mcp call_tool write_file ```json\n{ \"path\": \"/src/index.ts\" }\n```", + "range": { "start_line": 0, "end_line": 2 }, + "arguments": { + "header": "call_tool write_file", + "mode": "fence", + "fence_lang": "json", + "payload": "{ \"path\": \"/src/index.ts\" }" + } +} +```` + +### A.4 Backslash Join into Fence + +Input (four physical lines): + +````text +/mcp call_tool write_file \ +```json +{ "path": "foo" } +``` +```` + +After joining, physical lines 0 and 1 become: ``/mcp call_tool write_file ```json`` + +Output: + +````json +{ + "id": "cmd-0", + "name": "mcp", + "raw": "/mcp call_tool write_file \\\n```json\n{ \"path\": \"foo\" }\n```", + "range": { "start_line": 0, "end_line": 3 }, + "arguments": { + "header": "call_tool write_file", + "mode": "fence", + "fence_lang": "json", + "payload": "{ \"path\": \"foo\" }" + } +} +```` + +### A.5 Text Blocks and Multiple Commands + +Input: + +```text +Welcome to the deployment system. + +/deploy staging +/notify team --channel ops +Deployment complete. +``` + +Output: + +```json +{ + "version": "0.3.0", + "context": {}, + "commands": [ + { + "id": "cmd-0", + "name": "deploy", + "raw": "/deploy staging", + "range": { "start_line": 2, "end_line": 2 }, + "arguments": { + "header": "staging", + "mode": "single-line", + "fence_lang": null, + "payload": "staging" + } + }, + { + "id": "cmd-1", + "name": "notify", + "raw": "/notify team --channel ops", + "range": { "start_line": 3, "end_line": 3 }, + "arguments": { + "header": "team --channel ops", + "mode": "single-line", + "fence_lang": null, + "payload": "team --channel ops" + } + } + ], + "text_blocks": [ + { + "id": "text-0", + "range": { "start_line": 0, "end_line": 1 }, + "content": "Welcome to the deployment system.\n" + }, + { + "id": "text-1", + "range": { "start_line": 4, "end_line": 4 }, + "content": "Deployment complete." + } + ], + "warnings": [] +} +``` + +### A.6 Invalid Slash Lines + +Input: + +```text +/123 not a command +/ bare slash +/Hello capitalized +/deploy staging +``` + +Output: the first three lines form `text-0`; `/deploy staging` is `cmd-0`. + +### A.7 Unclosed Fence + +Input: + +````text +/mcp call_tool ```json +{ "incomplete": true } +```` + +Output: the command is finalized with the accumulated payload, and a warning is emitted: + +````json +{ + "version": "0.3.0", + "context": {}, + "commands": [ + { + "id": "cmd-0", + "name": "mcp", + "raw": "/mcp call_tool ```json\n{ \"incomplete\": true }", + "range": { "start_line": 0, "end_line": 1 }, + "arguments": { + "header": "call_tool", + "mode": "fence", + "fence_lang": "json", + "payload": "{ \"incomplete\": true }" + } + } + ], + "text_blocks": [], + "warnings": [ + { + "type": "unclosed_fence", + "start_line": 0, + "message": "Fenced block opened at line 0 was never closed." + } + ] +} +```` + +### A.8 Closing Fence with Trailing Backslash + +A closing fence line that ends with a trailing backslash closes the fence normally, then the +backslash triggers a line join with the next physical line. The joined content is outside the fenced +command. + +Input (six physical lines): + +````text +/mcp call_tool write_file -c \ +```json +{ "path": "foo" } +``` \ +\ +production +```` + +Physical lines: + +- 0: `/mcp call_tool write_file -c \` +- 1: `` ```json `` +- 2: `{ "path": "foo" }` +- 3: `` ``` \ `` +- 4: `\` +- 5: `production` + +Line joining (outside fences): + +- Lines 0 and 1 join: ``/mcp call_tool write_file -c ```json`` +- Lines 2 is inside the fence (verbatim, no joining). +- Line 3 is the closing fence (backticks only after trim? No: it contains `` ``` \ ``, which after + trimming whitespace is `` ```\ `` or `` ``` \ ``. The backslash means this is not solely + backticks, so it is not a valid closing fence.) + +Because line 3 is not a valid closing fence, the fence never closes. All remaining lines (2 through +5) become part of the fenced payload, and the parser emits an `unclosed_fence` warning. + +Output: + +````json +{ + "version": "0.3.0", + "context": {}, + "commands": [ + { + "id": "cmd-0", + "name": "mcp", + "raw": "/mcp call_tool write_file -c \\\n```json\n{ \"path\": \"foo\" }\n``` \\\n\\\nproduction", + "range": { "start_line": 0, "end_line": 5 }, + "arguments": { + "header": "call_tool write_file -c", + "mode": "fence", + "fence_lang": "json", + "payload": "{ \"path\": \"foo\" }\n``` \\\n\\\nproduction" + } + } + ], + "text_blocks": [], + "warnings": [ + { + "type": "unclosed_fence", + "start_line": 1, + "message": "Fenced block opened at line 1 was never closed." + } + ] +} +```` + +To avoid this, write the closing fence on its own line without a trailing backslash: + +````text +/mcp call_tool write_file -c \ +```json +{ "path": "foo" } +``` +```` + +### A.9 Proper Fence Close Followed by Additional Content + +Input (six physical lines): + +````text +/mcp call_tool write_file -c\ +```json +{ "path": "foo" } +``` +\ +production +```` + +Physical lines: + +- 0: `/mcp call_tool write_file -c\` +- 1: `` ```json `` +- 2: `{ "path": "foo" }` +- 3: `` ``` `` +- 4: `\` +- 5: `production` + +Line joining (outside fences): + +- Lines 0 and 1 join: ``/mcp call_tool write_file -c ```json`` +- Line 2 is inside the fence (verbatim). +- Line 3 is a valid closing fence (solely backticks after trim). Fence closes. Parser returns to + `idle`. +- Lines 4 and 5 join: `production` + +Output: + +````json +{ + "version": "0.3.0", + "context": {}, + "commands": [ + { + "id": "cmd-0", + "name": "mcp", + "raw": "/mcp call_tool write_file -c\\\n```json\n{ \"path\": \"foo\" }\n```", + "range": { "start_line": 0, "end_line": 3 }, + "arguments": { + "header": "call_tool write_file -c", + "mode": "fence", + "fence_lang": "json", + "payload": "{ \"path\": \"foo\" }" + } + } + ], + "text_blocks": [ + { "id": "text-0", "range": { "start_line": 4, "end_line": 5 }, "content": " production" } + ], + "warnings": [] +} +```` + +## Appendix B: Change Log + +Changes from version 0.2.0: + +- Replaced continuation markers (`" /"`) with POSIX-style backslash line joining, reducing the state + machine from three states to two. +- Added explicit treatment of invalid slash lines in idle state (Section 3.2). +- Defined fence lifetime as "until closing fence or EOF" (Section 5.2.3). +- Formalized trailing backslash at EOF behavior (Section 2.2.2). +- Added backslash-join-then-fence example (Section 5.2.6). +- Added closing-fence-with-backslash examples (Appendix A.8, A.9). +- Stated opaque payload principle as a conformance requirement (Section 9.3). +- Stated roundtrip fidelity invariant as a testable property (Section 9.4). +- Stated incremental emission as a conformance constraint (Section 9.2). +- Committed to no-AST as a conformance requirement (Section 9.1). +- Added determinism requirement (Section 9.5). +- Clarified JSONL streaming semantics with rationale (Section 8.3). +- Documented `header` as the dispatch portion available in both modes (Section 3.4). +- Introduced `warnings` array for non-fatal issues (Section 5.2.5). +- Removed unused `children` field from Command schema. +- Only backtick fences are supported; tilde fences are explicitly excluded (Section 5.2). +- Version bumped to 0.3.0. diff --git a/docs/slash-parser.md b/docs/slash-parser.md deleted file mode 100644 index 833dec6..0000000 --- a/docs/slash-parser.md +++ /dev/null @@ -1,333 +0,0 @@ -# small Rust+WASM “slash command parser” component with a JSON-in / JSON-out API, - -## style and patterns - -code snippets and patterns in this doc other than the parser semantics are only examples. The -language, build, and testing patterns override and should be followed. - -- toms-clean-code.md -- toms-clean-arch.md -- lang-rust.md -- testing.md -- testing-rust.md -- universal-rust-wasm-lib.md - -## 1. Component overview - -- Name: `slash_parser` -- Implementation language: Rust -- Compilation target: WebAssembly -- Binding strategy: `wasm-bindgen` for JS/TS environments.[^1][^2] -- Interface style: - - Input: UTF‑8 text (string). - - Output: UTF‑8 JSON string conforming to a provided JSON Schema. - -Responsibilities: - -1. Parse an input text buffer using the **slash-command semantics** described below. -2. Emit a JSON document describing: - - All detected commands and their arguments, - - Optional non-command text blocks, - - A context object. - -The WASM module must be deterministic and pure (no I/O, no global mutable state aside from internal -caches). - ---- - -## 2. Parsing semantics (Rust core) - -The Rust core must implement the exact language you’ve defined: - -### 2.1 Input - -- Input is a `&str` (UTF‑8 text). -- The parser processes it line-by-line, splitting on `\n`. -- `\r\n` must be normalized to `\n` before parsing. - -### 2.2 Command lines - -A **command line** is any line whose first non-whitespace character is `/`. - -Structure: - -- Leading whitespace (spaces/tabs) before `/` is ignored for detection but not included in `raw`. -- After the first `/`, parse `` using regex `[a-z][a-z0-9-]*`. -- The rest of the line, after one or more whitespace characters, is `arguments.header` prefix. - -Non-command lines: - -- Lines that do not start with `/` (ignoring leading whitespace) are considered **non-command text** - when the parser is in `idle` state. -- If a command is currently being accumulated, non-command lines are appended to that command’s - payload (depending on the state). - -### 2.3 States - -The parser is a state machine: - -- `Idle` – not inside any command. -- `Accumulating` – building `arguments.payload` for a command outside a fence. -- `InFence` – inside a fenced block attached to a command. - -State transitions are driven by: - -- New command detection, -- Continuation marker `" /"`, -- Fence openers / closers. - -### 2.4 Continuation with `" /"` - -Continuation is explicit and conservative: - -- A line continues the current command if and only if it ends with **space + slash** (`" /"`) - immediately before the newline. - -Semantics: - -- In `Accumulating` (or on the first header line of a command): - - If the line ends with `" /"`: - - Remove the trailing `" /"` from the line content. - - Append the remaining content plus `\n` to `arguments.payload`. - - Stay in `Accumulating`. - - If the line does not end with `" /"` and does not start a fence: - - Append the full line content plus `\n`. - - Finalize the command and return to `Idle`. - -Note: `" /"` must be checked literally; a line ending in `/` with no preceding space is **not** a -continuation marker. This avoids collisions with paths like `/var/log/`. - -### 2.5 Fenced block arguments - -Use markdown-style fenced code blocks:[^3][^4] - -#### Fence opener detection - -Fence openers can appear: - -1. **Inline on the command line**: - - Inside ``, find the first occurrence of three or more consecutive backticks: - ```… - ``` - - Everything before the backticks remains in `arguments.header`. - - The backticks and optional language identifier (e.g., `jsonl`) mark the start of fence mode. -2. **On the next line after continuation**: - - The command line ends with `" /"` and the parser enters `Accumulating`. - - The next line starts with the fence opener ```[lang]. - - The parser transitions to `InFence`. - -Fence metadata: - -- Record: - - Fence marker: number of backticks (typically 3). - - Optional `lang` following the backticks (up to first whitespace). - -#### Fence mode - -In `InFence`: - -- Every line is appended verbatim to `arguments.payload` with a trailing `\n`. -- `" /"` has no special meaning inside the fence. -- The parser looks for a closing fence: a line whose first non-whitespace characters are the same - number of backticks and nothing else but whitespace. - -On closing fence: - -- The closing fence line is not included in the payload. -- Command is finalized. -- `arguments.mode = "fence"`. -- `arguments.fence_lang` is set to the captured language or `null`. -- Parser returns to `Idle`. - -### 2.6 Multiple commands - -- In `Idle`, when a command line appears, start a new command. -- When a command is finalized, append it to the `commands` array and go back to `Idle`. -- Non-command text between commands may be recorded as `text_blocks`. - -You can treat lines that fall entirely outside commands as text blocks grouped by contiguous ranges. - ---- - -## 3. JSON output format - -The Rust core will produce a JSON value of type `SlashParseResult`: - -```ts -interface SlashParseResult { - version: string // e.g., "0.1.0" - context: { - source?: string - timestamp?: string // ISO 8601, if provided by caller - user?: string - session_id?: string - extra?: Record - [key: string]: unknown - } - commands: Command[] - text_blocks?: TextBlock[] -} - -interface CommandRange { - start_line: number // 0-based or 1-based (document in spec) - end_line: number -} - -type ArgumentMode = 'single-line' | 'continuation' | 'fence' - -interface CommandArguments { - header?: string - mode: ArgumentMode - fence_lang?: string | null - payload: string -} - -interface Command { - id: string - name: string - raw?: string - range: CommandRange - arguments: CommandArguments - children?: Command[] // reserved -} - -interface TextBlock { - id: string - range: CommandRange - content: string -} -``` - -The JSON Schema in your previous step applies; the Rust types should be defined to match that -schema, using `serde` for serialization.[^5][^6][^7] - ---- - -## 4. WASM API design - -### 4.1 Rust public API (pre-wasm) - -Define a core Rust function: - -```rust -pub struct ParserContext { - pub source: Option, - pub timestamp: Option, - pub user: Option, - pub session_id: Option, - pub extra: Option, -} - -pub fn parse_slash_commands( - input: &str, - context: ParserContext, -) -> Result; -``` - -- `ParseError` should include at least a message and optionally a line/column. - -### 4.2 wasm-bindgen exports - -Use `wasm-bindgen` to expose a JS-friendly API:[^8][^2][^1] - -- Module name: `slash_parser`. -- Exported functions: - -```rust -use serde::{Deserialize, Serialize}; -use wasm_bindgen::prelude::*; - -#[wasm_bindgen] -pub fn parse_text(input: &str) -> Result; - -#[wasm_bindgen] -pub fn parse_text_with_context(input: &str, context: &JsValue) -> Result; -``` - -Semantics: - -- `parse_text`: - - Uses a default `ParserContext { source: None, ... }`. - - Returns a `JsValue` containing a JSON object conforming to `SlashParseResult`, via - `JsValue::from_serde(&result)`. -- `parse_text_with_context`: - - `context` is optional; if provided, it is interpreted as a JS object containing any of: - - `source`, `timestamp`, `user`, `session_id`, `extra`. - - Rust side should `from_serde` the context into `ParserContext` (or build it manually from - `JsValue`).[^9] - - Returns the same JSON structure as `parse_text`. - -On error: - -- Return a `JsValue` representing a JS `Error` or a structured - `{ error: string, line?: number, column?: number }`. - -### 4.3 JS / TS usage expectations - -Consumers (e.g. a Node-based or browser-based tool) will: - -```ts -import init, { parse_text, parse_text_with_context } from 'slash_parser' - -await init() // or equivalent wasm-bindgen init - -const result = parse_text_with_context(sourceText, { source: 'file://path/to/doc.md', user: 'tom' }) - -// `result` is a JS object matching SlashParseResult -console.log(result.commands) -``` - -The bindings should be compatible with bundlers (Vite, Webpack) and Node 18+. - ---- - -## 5. Non-goals / constraints - -To keep Perplexity Computer’s scope tight: - -- No I/O: the parser does not read files or network; it only parses the provided string. -- No global config: all configuration is via parameters or context. -- No incremental parsing: first version can parse whole strings only. -- No reliance on external crates beyond: - - `serde`, `serde_json`, - - `wasm-bindgen` (and its minimal dependencies). - -[^1]: https://rustwasm.github.io/docs/wasm-bindgen/ - -[^2]: https://rustwasm.github.io/docs/wasm-bindgen/print.html - -[^3]: https://www.markdownguide.org/extended-syntax/ - -[^4]: https://python-markdown.github.io/extensions/fenced_code_blocks/ - -[^5]: https://json-schema.org/learn/json-schema-examples - -[^6]: https://json-schema.org/understanding-json-schema/basics - -[^7]: https://blog.promptlayer.com/how-json-schema-works-for-structured-outputs-and-tool-integration/ - -[^8]: https://rustwasm.github.io/docs/wasm-bindgen/contributing/design/index.html - -[^9]: https://users.rust-lang.org/t/how-do-i-sent-a-js-object-to-rust-through-wasm/80007 - -[^10]: https://www.reddit.com/r/rust/comments/15sjuyo/interfacing_complex_types_in_wasm/ - -[^11]: https://github.com/rustwasm/wasm-bindgen/discussions/2883 - -[^12]: https://stackoverflow.com/questions/65242336/js-binding-for-large-rust-object-using-wasm-bindgen - -[^13]: https://nickb.dev/blog/recommendations-when-publishing-a-wasm-library/ - -[^14]: https://www.speakeasy.com/blog/building-speakeasy-openapi-go-library - -[^15]: https://github.com/douglance/valrs - -[^16]: https://hacks.mozilla.org/2019/08/webassembly-interface-types/ - -[^17]: https://github.com/WebAssembly/design/blob/main/Rationale.md - -[^18]: https://docs.rs/valrs-wasm - -[^19]: https://gov.near.org/t/proposal-use-webassembly-interface-types-to-describe-all-standards-interfaces-and-application-contract-interface-aci/23256 - -[^20]: https://developer.mozilla.org/en-US/docs/WebAssembly/Guides/Concepts From 597eb02829e3a2c9d0122ca8d2068d3f3df39d9f Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Thu, 19 Mar 2026 05:16:49 -0600 Subject: [PATCH 4/5] feat(parser): update domain layer to parser spec 0.3.0 --- parser/src/domain/errors.rs | 7 ++++--- parser/src/domain/mod.rs | 6 ++---- parser/src/domain/types.rs | 28 ++++++---------------------- 3 files changed, 12 insertions(+), 29 deletions(-) diff --git a/parser/src/domain/errors.rs b/parser/src/domain/errors.rs index 23b1e8a..8654a6b 100644 --- a/parser/src/domain/errors.rs +++ b/parser/src/domain/errors.rs @@ -3,7 +3,8 @@ /// Warnings are collected in `ParseResult.warnings` rather than /// causing the parse to fail. The parser is intentionally permissive. #[derive(Debug, Clone, PartialEq, Eq)] -pub enum ParseWarning { - UnclosedFence { start_line: usize }, - UnclosedContinuation { start_line: usize }, +pub struct Warning { + pub wtype: String, + pub start_line: Option, + pub message: Option, } diff --git a/parser/src/domain/mod.rs b/parser/src/domain/mod.rs index ef66902..0453bad 100644 --- a/parser/src/domain/mod.rs +++ b/parser/src/domain/mod.rs @@ -1,9 +1,7 @@ mod errors; mod types; -pub use errors::ParseWarning; +pub use errors::Warning; pub use types::{ - ArgumentMode, Command, CommandArguments, LineRange, ParseResult, ParserContext, TextBlock, + ArgumentMode, Command, CommandArguments, LineRange, ParseResult, TextBlock, SPEC_VERSION, }; - -pub use types::SPEC_VERSION; diff --git a/parser/src/domain/types.rs b/parser/src/domain/types.rs index d07d448..3868589 100644 --- a/parser/src/domain/types.rs +++ b/parser/src/domain/types.rs @@ -1,6 +1,4 @@ -use std::collections::HashMap; - -use super::errors::ParseWarning; +use super::errors::Warning; /// Inclusive line range (zero-based). #[derive(Debug, Clone, PartialEq, Eq)] @@ -13,7 +11,6 @@ pub struct LineRange { #[derive(Debug, Clone, PartialEq, Eq)] pub enum ArgumentMode { SingleLine, - Continuation, Fence, } @@ -29,6 +26,7 @@ pub struct CommandArguments { /// A single parsed slash command. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Command { + pub id: String, pub name: String, pub raw: String, pub range: LineRange, @@ -38,32 +36,18 @@ pub struct Command { /// A contiguous block of non-command text. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TextBlock { + pub id: String, pub range: LineRange, pub content: String, } -/// Metadata provided by the caller, merged into the output envelope. -/// -/// `extra` holds additional key-value pairs beyond the known fields. -/// Values are strings; richer types are a serialization concern -/// handled at the application boundary. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct ParserContext { - pub source: Option, - pub timestamp: Option, - pub user: Option, - pub session_id: Option, - pub extra: HashMap, -} - /// Top-level parse result. #[derive(Debug, Clone, PartialEq)] pub struct ParseResult { pub version: String, - pub context: ParserContext, pub commands: Vec, - pub text_blocks: Vec, - pub warnings: Vec, + pub textblocks: Vec, + pub warnings: Vec, } -pub const SPEC_VERSION: &str = "v1"; +pub const SPEC_VERSION: &str = "0.3.0"; From cc4e81afd488fb9509c135d310903e03b425d568 Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Thu, 19 Mar 2026 05:21:20 -0600 Subject: [PATCH 5/5] feat(parser): partial update app layer to parser spec 0.3.0 --- .../application/command_accumulate.txt | 7 + parser/src/application/command_accumulate.rs | 408 +++++++---- parser/src/application/command_finalize.rs | 305 ++++++--- parser/src/application/document_parse.rs | 644 +++++++++--------- parser/src/application/line_classify.rs | 408 ++++++++--- parser/src/application/line_join.rs | 354 ++++++++++ parser/src/application/mod.rs | 8 +- parser/src/application/normalize.rs | 141 ++++ parser/src/application/tests/mod.rs | 416 ++++++++++- parser/src/application/text_collect.rs | 240 ++++--- parser/src/lib.rs | 6 +- 11 files changed, 2210 insertions(+), 727 deletions(-) create mode 100644 parser/proptest-regressions/application/command_accumulate.txt create mode 100644 parser/src/application/line_join.rs create mode 100644 parser/src/application/normalize.rs diff --git a/parser/proptest-regressions/application/command_accumulate.txt b/parser/proptest-regressions/application/command_accumulate.txt new file mode 100644 index 0000000..ccfb93a --- /dev/null +++ b/parser/proptest-regressions/application/command_accumulate.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 1657315ea6a9fa6475f78c9454ab64e78fab99d9eef8792ed95cd52b7ecd3bdc # shrinks to name = "a", body_lines = [" "] diff --git a/parser/src/application/command_accumulate.rs b/parser/src/application/command_accumulate.rs index d89f76f..0fe9c68 100644 --- a/parser/src/application/command_accumulate.rs +++ b/parser/src/application/command_accumulate.rs @@ -1,7 +1,7 @@ -use super::line_classify::{CommandHeader, LineKind, classify_line}; +use super::line_classify::CommandHeader; use crate::domain::ArgumentMode; -/// Result of offering a line to an in-progress command. +/// Result of offering a physical line to an in-progress command. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AcceptResult { Consumed, @@ -9,46 +9,66 @@ pub enum AcceptResult { Rejected, } -/// In-progress command being assembled from consecutive lines. +/// In-progress command being assembled from physical lines. +/// +/// v0.3.0: Continuation mode is no longer handled here. Multi-physical-line commands +/// that are not fenced are resolved entirely by line joining (§2.2) before this module runs. #[derive(Debug, Clone)] pub struct PendingCommand { + pub id: usize, pub name: String, pub raw_header: String, pub header_text: String, pub mode: ArgumentMode, pub fence_lang: Option, + pub fence_backtick_count: usize, pub start_line: usize, pub end_line: usize, pub payload_lines: Vec, + pub raw_lines: Vec, pub is_open: bool, } -pub fn start_command(header: CommandHeader, line_index: usize) -> PendingCommand { - let initial_lines = if header.header_text.is_empty() { - vec![] - } else { - vec![header.header_text.clone()] - }; - +/// Begin accumulating a new command from its parsed header. +/// +/// §7: id is the caller-supplied sequential zero-based counter used to produce cmd-0, cmd-1, ... +/// §8.2: raw_lines is seeded with the header's raw text; physical lines are appended by accept_line. +pub fn start_command(header: CommandHeader, line_index: usize, id: usize) -> PendingCommand { + // §5.1: single-line commands are fully resolved on their logical line; is_open = false. + // §5.2.1: fence commands open immediately and stay open until a closer is found. let (payload_lines, is_open) = match &header.mode { - ArgumentMode::SingleLine => (initial_lines, false), - ArgumentMode::Continuation => (initial_lines, true), + ArgumentMode::SingleLine => { + let lines = if header.header_text.is_empty() { + vec![] + } else { + vec![header.header_text.clone()] + }; + (lines, false) + } ArgumentMode::Fence => (vec![], true), }; PendingCommand { + id, name: header.name, - raw_header: header.raw, + raw_header: header.raw.clone(), header_text: header.header_text, mode: header.mode, fence_lang: header.fence_lang, + fence_backtick_count: header.fence_backtick_count, start_line: line_index, end_line: line_index, payload_lines, + raw_lines: vec![header.raw], is_open, } } +/// Offer one physical line to an open command. +/// +/// §5.2.2: fence body lines are appended to payload_lines verbatim. +/// §5.2.4: a physical line that satisfies is_fence_closer closes the fence. +/// §8.2: every offered line, including the closer, is appended to raw_lines. pub fn accept_line( cmd: PendingCommand, line_index: usize, @@ -59,37 +79,9 @@ pub fn accept_line( } match cmd.mode { - ArgumentMode::SingleLine => (cmd, AcceptResult::Rejected), - ArgumentMode::Continuation => accept_continuation(cmd, line_index, line), ArgumentMode::Fence => accept_fence(cmd, line_index, line), - } -} - -fn accept_continuation( - mut cmd: PendingCommand, - line_index: usize, - line: &str, -) -> (PendingCommand, AcceptResult) { - if line.trim().is_empty() { - cmd.is_open = false; - return (cmd, AcceptResult::Rejected); - } - - if matches!(classify_line(line), LineKind::Command(_)) { - cmd.is_open = false; - return (cmd, AcceptResult::Rejected); - } - - if line.trim_end().ends_with('\\') { - let content = line.trim_end().trim_end_matches('\\').trim_end(); - cmd.payload_lines.push(content.to_string()); - cmd.end_line = line_index; - (cmd, AcceptResult::Consumed) - } else { - cmd.payload_lines.push(line.to_string()); - cmd.end_line = line_index; - cmd.is_open = false; - (cmd, AcceptResult::Completed) + // §4: only idle and in-fence states exist in v0.3.0; all other modes are closed. + _ => (cmd, AcceptResult::Rejected), } } @@ -98,152 +90,328 @@ fn accept_fence( line_index: usize, line: &str, ) -> (PendingCommand, AcceptResult) { - if line.trim_start().starts_with("```") { - cmd.end_line = line_index; + // §8.2: raw accumulation happens unconditionally before the closer check. + cmd.raw_lines.push(line.to_string()); + cmd.end_line = line_index; + + if is_fence_closer(line, cmd.fence_backtick_count) { + // §5.2.4: the closing fence line is not included in the payload. cmd.is_open = false; (cmd, AcceptResult::Completed) } else { - cmd.end_line = line_index; cmd.payload_lines.push(line.to_string()); (cmd, AcceptResult::Consumed) } } +/// §5.2.4: A physical line is a closing fence if, after trimming leading and trailing +/// whitespace, it consists solely of backtick characters and the count is >= the opener's count. +fn is_fence_closer(line: &str, opener_count: usize) -> bool { + let trimmed = line.trim(); + !trimmed.is_empty() + && trimmed.chars().all(|c| c == '`') + && trimmed.len() >= opener_count +} + #[cfg(test)] mod tests { use super::*; + use crate::application::line_classify::{LineKind, classify_line}; fn header_from(line: &str) -> CommandHeader { match classify_line(line) { LineKind::Command(h) => h, - _ => panic!("expected command header"), + _ => panic!("expected command header from: {line:?}"), } } + // --- start_command --- + #[test] fn single_line_is_immediately_closed() { - let cmd = start_command(header_from("/help"), 0); + // §5.1: a single-line command is finalized on its logical line; no further lines consumed. + let cmd = start_command(header_from("/help"), 0, 0); assert!(!cmd.is_open); } #[test] - fn fence_stays_open_until_closing_ticks() { - let cmd = start_command(header_from("/code ```rust"), 0); + fn fence_opens_with_is_open_true() { + // §5.2.3: fence lifetime begins at the opener and extends until a valid closer or EOF. + let cmd = start_command(header_from("/code ```rust"), 0, 0); assert!(cmd.is_open); - - let (cmd, res) = accept_line(cmd, 1, "fn main() {}"); - assert_eq!(res, AcceptResult::Consumed); - assert!(cmd.is_open); - - let (cmd, res) = accept_line(cmd, 2, "```"); - assert_eq!(res, AcceptResult::Completed); - assert!(!cmd.is_open); - assert_eq!(cmd.payload_lines, vec!["fn main() {}"]); } #[test] - fn continuation_ends_on_line_without_backslash() { - let cmd = start_command(header_from("/cmd first \\"), 0); - assert!(cmd.is_open); + fn id_is_stored_on_pending_command() { + // §7: commands are assigned sequential zero-based IDs in encounter order (cmd-0, cmd-1, ...). + let cmd = start_command(header_from("/deploy production"), 3, 7); + assert_eq!(cmd.id, 7); + } - let (cmd, res) = accept_line(cmd, 1, "second \\"); - assert_eq!(res, AcceptResult::Consumed); + #[test] + fn raw_lines_initialized_with_opener() { + // §8.2: raw includes the command line itself; accumulation starts at start_command. + let cmd = start_command(header_from("/mcp call_tool ```json"), 0, 0); + assert_eq!(cmd.raw_lines, vec!["/mcp call_tool ```json"]); + } - let (cmd, res) = accept_line(cmd, 2, "last line"); - assert_eq!(res, AcceptResult::Completed); - assert!(!cmd.is_open); + #[test] + fn single_line_payload_initialized_from_header_text() { + // §5.1: in single-line mode header and payload contain the same string. + let cmd = start_command(header_from("/deploy production --region us-west-2"), 0, 0); + assert_eq!(cmd.payload_lines, vec!["production --region us-west-2"]); } #[test] - fn continuation_rejects_blank_line() { - let cmd = start_command(header_from("/cmd start \\"), 0); - let (cmd, res) = accept_line(cmd, 1, ""); - assert_eq!(res, AcceptResult::Rejected); - assert!(!cmd.is_open); + fn single_line_no_args_has_empty_payload() { + // §5.1 (implied): a command with no arguments produces an empty payload_lines. + let cmd = start_command(header_from("/ping"), 0, 0); + assert!(cmd.payload_lines.is_empty()); } #[test] - fn continuation_rejects_new_command() { - let cmd = start_command(header_from("/cmd start \\"), 0); - let (cmd, res) = accept_line(cmd, 1, "/other"); - assert_eq!(res, AcceptResult::Rejected); - assert!(!cmd.is_open); + fn fence_payload_lines_initially_empty() { + // §5.2.2: the fence body begins empty; content is appended as physical lines arrive. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + assert!(cmd.payload_lines.is_empty()); } #[test] - fn continuation_two_lines_produces_continuation_mode() { - let header = header_from("/mcp call_tool read_file \\"); - let cmd = start_command(header, 0); + fn fence_backtick_count_carried_from_header() { + // §5.2.1: the opener's backtick run length is recorded for use by the closer check (§5.2.4). + let cmd = start_command(header_from("/cmd ```rust"), 0, 0); + assert_eq!(cmd.fence_backtick_count, 3); + } - assert_eq!(cmd.mode, ArgumentMode::Continuation); + // --- accept_line: fence body --- + + #[test] + fn fence_body_line_is_consumed() { + // §5.2.2: physical lines that are not a closer are appended to the payload verbatim. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, "fn main() {}"); + assert_eq!(res, AcceptResult::Consumed); assert!(cmd.is_open); + assert_eq!(cmd.payload_lines, vec!["fn main() {}"]); } #[test] - fn continuation_header_strips_trailing_marker() { - let header = header_from("/mcp call_tool read_file \\"); - let cmd = start_command(header, 0); + fn fence_body_line_added_to_raw_lines() { + // §8.2: for fenced commands raw includes the command line, all body lines, and the closer. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "first"); + let (cmd, _) = accept_line(cmd, 2, "second"); + assert_eq!(cmd.raw_lines, vec!["/cmd ```", "first", "second"]); + } - assert_eq!(cmd.header_text, "call_tool read_file"); - assert_eq!(cmd.payload_lines, vec!["call_tool read_file"]); + #[test] + fn fence_body_preserves_lines_verbatim() { + // §5.2.2: content is appended verbatim; no parsing rules including line joining apply. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let line = r" leading spaces and trailing backslash\"; + let (cmd, _) = accept_line(cmd, 1, line); + assert_eq!(cmd.payload_lines, vec![line]); + } + + #[test] + fn blank_line_inside_fence_is_payload() { + // §5.2.3: inside a fence, blank lines are literal payload content. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, ""); + assert_eq!(res, AcceptResult::Consumed); + assert_eq!(cmd.payload_lines, vec![""]); } #[test] - fn continuation_payload_preserves_newlines() { - let cmd = start_command(header_from("/mcp call_tool read_file \\"), 0); + fn command_line_inside_fence_is_payload() { + // §5.2.3: inside a fence, command triggers are literal payload; no state change. + let cmd = start_command(header_from("/outer ```"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, "/inner arg"); + assert_eq!(res, AcceptResult::Consumed); + assert!(cmd.is_open); + } + + // --- accept_line: fence closer --- - let line1 = r#"{"path": "src/index.ts"}"#; - let (cmd, res) = accept_line(cmd, 1, line1); + #[test] + fn fence_closer_completes_command() { + // §5.2.4: a valid closer transitions the accumulator to closed and returns Completed. + let cmd = start_command(header_from("/cmd ```rust"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "fn main() {}"); + let (cmd, res) = accept_line(cmd, 2, "```"); assert_eq!(res, AcceptResult::Completed); assert!(!cmd.is_open); + } - assert_eq!( - cmd.payload_lines, - vec!["call_tool read_file".to_string(), line1.to_string()] - ); + #[test] + fn fence_closer_added_to_raw_lines() { + // §8.2: raw includes the closing fence line if present. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "content"); + let (cmd, _) = accept_line(cmd, 2, "```"); + assert_eq!(cmd.raw_lines, vec!["/cmd ```", "content", "```"]); + } - assert_eq!(cmd.start_line, 0); - assert_eq!(cmd.end_line, 1); + #[test] + fn fence_closer_not_in_payload() { + // §5.2.4: the closing fence line is not included in the payload. + let cmd = start_command(header_from("/cmd ```rust"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "fn main() {}"); + let (cmd, _) = accept_line(cmd, 2, "```"); + assert_eq!(cmd.payload_lines, vec!["fn main() {}"]); } #[test] - fn continuation_three_lines() { - let cmd = start_command(header_from("/cmd first \\"), 0); + fn fence_closer_with_more_backticks_than_opener() { + // §5.2.4: closer count >= opener count; four backticks close a triple-backtick fence. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, "````"); + assert_eq!(res, AcceptResult::Completed); + assert!(!cmd.is_open); + } + + #[test] + fn fewer_backticks_than_opener_is_not_a_closer() { + // §5.2.4: closer count must be >= opener count; two backticks cannot close a triple fence. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, "``"); + assert_eq!(res, AcceptResult::Consumed); + assert!(cmd.is_open); + } - let (cmd, res) = accept_line(cmd, 1, "second \\"); + #[test] + fn line_with_backticks_and_trailing_text_is_not_a_closer() { + // §5.2.4: after trimming whitespace the line must consist solely of backtick characters. + // A line like "```rust" or "``` trailing" is body content, not a closer. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, "```rust"); assert_eq!(res, AcceptResult::Consumed); - let (cmd, res) = accept_line(cmd, 2, "third"); + assert!(cmd.is_open); + } + + #[test] + fn fence_closer_with_leading_whitespace_is_valid() { + // §5.2.4: closer is checked after trimming leading and trailing whitespace. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, " ``` "); assert_eq!(res, AcceptResult::Completed); assert!(!cmd.is_open); + } - assert_eq!( - cmd.payload_lines, - vec![ - "first".to_string(), - "second".to_string(), - "third".to_string() - ] - ); + #[test] + fn closed_fence_rejects_subsequent_lines() { + // §5.2.3: once the closing fence is found the command is finalized; no further lines accepted. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "content"); + let (cmd, _) = accept_line(cmd, 2, "```"); + let (cmd, res) = accept_line(cmd, 3, "after"); + assert_eq!(res, AcceptResult::Rejected); + assert_eq!(cmd.payload_lines, vec!["content"]); } + // --- accept_line: single-line --- + #[test] - fn continuation_range_spans_all_lines() { - let cmd = start_command(header_from("/cmd first \\"), 0); + fn single_line_rejects_any_subsequent_line() { + // §5.1: single-line commands are finalized on their logical line; accept_line is a no-op. + let cmd = start_command(header_from("/help"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, "anything"); + assert_eq!(res, AcceptResult::Rejected); + assert!(cmd.payload_lines == vec![""] || cmd.payload_lines.is_empty()); + } - let (cmd, _) = accept_line(cmd, 1, "second \\"); - let (cmd, _) = accept_line(cmd, 2, "third"); + // --- range tracking --- + #[test] + fn end_line_advances_with_each_accepted_line() { + // §2.2.1: range.endLine is the zero-based index of the last physical line consumed. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "line one"); + let (cmd, _) = accept_line(cmd, 2, "line two"); + let (cmd, _) = accept_line(cmd, 3, "```"); assert_eq!(cmd.start_line, 0); - assert_eq!(cmd.end_line, 2); + assert_eq!(cmd.end_line, 3); } - #[test] - fn slash_at_end_without_space_is_not_continuation() { - let header = header_from("/path /var/log/"); - let cmd = start_command(header, 0); + // --- Property tests --- - assert_eq!(cmd.mode, ArgumentMode::SingleLine); - assert!(!cmd.is_open); - assert_eq!(cmd.payload_lines, vec!["/var/log/"]); + use proptest::prelude::*; + + fn valid_command_name() -> impl Strategy { + "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + } + + proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn raw_lines_count_equals_lines_consumed( + // §8.2: raw accumulates opener + every body line + closer (if present). + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 0..8) + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0, 0); + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); + prop_assert_eq!(cmd.raw_lines.len(), body_lines.len() + 2); + } + + #[test] +#[cfg_attr(feature = "tdd", ignore)] +fn payload_never_contains_closer( + // §5.2.4: the closing fence line is never part of the payload. + // The check mirrors is_fence_closer: non-empty after trim, all backticks. + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 0..8) +) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0, 0); + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); +let no_closer = !cmd.payload_lines.iter().any(|l| { + let t = l.trim(); + !t.is_empty() && t.chars().all(|c| c == '`') +}); +prop_assert!(no_closer); +} + + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fence_closer_property_count_gte_opener( + // §5.2.4: any line of N or more backticks (N >= opener) closes the fence. + name in valid_command_name(), + extra in 0usize..5 + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0, 0); + let closer = "`".repeat(3 + extra); + let (cmd, res) = accept_line(cmd, 1, &closer); + prop_assert_eq!(res, AcceptResult::Completed); + prop_assert!(!cmd.is_open); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn id_is_preserved_through_accumulation( + // §7: the id assigned at start_command is stable throughout accumulation. + name in valid_command_name(), + id in 0usize..1000, + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 0..5) + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0, id); + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + prop_assert_eq!(cmd.id, id); + } } } diff --git a/parser/src/application/command_finalize.rs b/parser/src/application/command_finalize.rs index 7c3ca49..e89613a 100644 --- a/parser/src/application/command_finalize.rs +++ b/parser/src/application/command_finalize.rs @@ -1,34 +1,47 @@ use super::command_accumulate::PendingCommand; -use crate::domain::{ArgumentMode, Command, CommandArguments, LineRange, ParseWarning}; +use crate::domain::{ArgumentMode, Command, CommandArguments, LineRange, Warning}; #[derive(Debug)] pub struct FinalizedCommand { pub command: Command, - pub warnings: Vec, + pub warnings: Vec, } +/// Consume a PendingCommand and produce the finalized Command value plus any warnings. +/// +/// §7: id is formatted as cmd-{n} using the zero-based encounter index from start_command. +/// §8.2: raw is the physical lines joined with newline separators (opener + body + closer). +/// §5.2.5: an unclosed fence produces a Warning rather than a hard parse failure. pub fn finalize_command(pending: PendingCommand) -> FinalizedCommand { let mut warnings = Vec::new(); - match pending.mode { - ArgumentMode::Fence if pending.is_open => { - warnings.push(ParseWarning::UnclosedFence { - start_line: pending.start_line, - }); - } - ArgumentMode::Continuation if pending.is_open => { - warnings.push(ParseWarning::UnclosedContinuation { - start_line: pending.start_line, - }); - } - _ => {} + // §5.2.5: if the parser reached EOF without finding a closing fence, emit one warning + // and finalize with whatever payload was accumulated. + if pending.mode == ArgumentMode::Fence && pending.is_open { + warnings.push(Warning { + wtype: "unclosed-fence".to_string(), + start_line: Some(pending.start_line), + message: Some(format!( + "Fenced block opened at line {} was never closed.", + pending.start_line + )), + }); } + // §8.2: raw is built from all accumulated physical lines joined with newline separators. + // For single-line commands this is just the one header line. + // For fenced commands this includes the opener, all body lines, and the closer if present. + let raw = pending.raw_lines.join("\n"); + + // §5.1: single-line payload equals header_text (initialised at start_command). + // §5.2.2: fence payload is the verbatim body lines joined with newline separators. let payload = pending.payload_lines.join("\n"); let command = Command { + // §7: sequential zero-based id formatted as cmd-0, cmd-1, ... + id: format!("cmd-{}", pending.id), name: pending.name, - raw: pending.raw_header, + raw, range: LineRange { start_line: pending.start_line, end_line: pending.end_line, @@ -58,142 +71,208 @@ mod tests { fn header_from(line: &str) -> CommandHeader { match classify_line(line) { LineKind::Command(h) => h, - _ => panic!("expected command header"), + _ => panic!("expected command header from: {line:?}"), } } - // --- Warning tests --- + // --- id --- #[test] - fn unclosed_fence_at_eof_warns() { - let cmd = start_command(header_from("/cmd ```rust"), 0); - let (cmd, _) = accept_line(cmd, 1, "line1"); - let (cmd, _) = accept_line(cmd, 2, "line2"); + fn id_is_formatted_as_cmd_zero() { + // §7: commands are assigned sequential zero-based IDs formatted as cmd-0, cmd-1, ... + let cmd = start_command(header_from("/deploy staging"), 0, 0); + let result = finalize_command(cmd); + assert_eq!(result.command.id, "cmd-0"); + } + #[test] + fn id_reflects_encounter_index() { + // §7: the id counter is caller-supplied; cmd-5 identifies the sixth command encountered. + let cmd = start_command(header_from("/deploy staging"), 0, 5); let result = finalize_command(cmd); + assert_eq!(result.command.id, "cmd-5"); + } - assert_eq!(result.warnings.len(), 1); - assert!(matches!( - result.warnings[0], - ParseWarning::UnclosedFence { start_line: 0 } - )); + // --- raw --- + + #[test] + fn single_line_raw_is_header_line_only() { + // §8.2: for a single-line command raw contains only the original command line. + let cmd = start_command(header_from("/deploy production --region us-west-2"), 0, 0); + let result = finalize_command(cmd); + assert_eq!(result.command.raw, "/deploy production --region us-west-2"); } #[test] - fn unclosed_continuation_at_eof_warns() { - let cmd = start_command(header_from("/cmd first \\"), 0); + fn fenced_raw_includes_opener_body_and_closer() { + // §8.2: for fenced commands raw includes the command line, all body lines, + // and the closing fence line, joined with newline separators. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "line one"); + let (cmd, _) = accept_line(cmd, 2, "line two"); + let (cmd, _) = accept_line(cmd, 3, "```"); + let result = finalize_command(cmd); + assert_eq!(result.command.raw, "/cmd ```\nline one\nline two\n```"); + } + #[test] + fn unclosed_fence_raw_includes_opener_and_partial_body() { + // §8.2: if the fence is unclosed raw still contains all accumulated physical lines; + // there is simply no closer line to include. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "partial"); let result = finalize_command(cmd); + assert_eq!(result.command.raw, "/cmd ```\npartial"); + } + + // --- warnings --- + #[test] + fn unclosed_fence_emits_exactly_one_warning() { + // §5.2.5: an unclosed fence produces exactly one warning; the parser does not fail. + let cmd = start_command(header_from("/cmd ```rust"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "line1"); + let (cmd, _) = accept_line(cmd, 2, "line2"); + let result = finalize_command(cmd); assert_eq!(result.warnings.len(), 1); - assert!(matches!( - result.warnings[0], - ParseWarning::UnclosedContinuation { start_line: 0 } - )); } #[test] - fn closed_fence_has_no_warning() { - let cmd = start_command(header_from("/cmd ```"), 0); - let (cmd, _) = accept_line(cmd, 1, "content"); - let (cmd, _) = accept_line(cmd, 2, "```"); - + fn unclosed_fence_warning_wtype_is_unclosed_fence() { + // §5.2.5: the warning type string is "unclosed-fence". + let cmd = start_command(header_from("/cmd ```"), 0, 0); let result = finalize_command(cmd); + assert_eq!(result.warnings[0].wtype, "unclosed-fence"); + } - assert!(result.warnings.is_empty()); + #[test] + fn unclosed_fence_warning_start_line_is_opener_physical_line() { + // §5.2.5: start_line is set to the fence opener's physical line number. + let cmd = start_command(header_from("/cmd ```"), 4, 0); + let result = finalize_command(cmd); + assert_eq!(result.warnings[0].start_line, Some(4)); } #[test] - fn closed_continuation_has_no_warning() { - let cmd = start_command(header_from("/cmd first \\"), 0); - let (cmd, _) = accept_line(cmd, 1, "last line"); + fn unclosed_fence_warning_message_is_present() { + // §5.2.5 (implied): a human-readable message is present and non-empty. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let result = finalize_command(cmd); + assert!(result.warnings[0].message.as_deref().map(|m| !m.is_empty()).unwrap_or(false)); + } + #[test] + fn unclosed_fence_warning_message_references_start_line() { + // §5.2.5 (implied): the message should identify the offending line number for diagnostics. + let cmd = start_command(header_from("/cmd ```"), 7, 0); let result = finalize_command(cmd); + let msg = result.warnings[0].message.as_deref().unwrap_or(""); + assert!(msg.contains('7')); + } + #[test] + fn closed_fence_has_no_warning() { + // §5.2.3: once the closing fence is found the command is well-formed; no warning emitted. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "content"); + let (cmd, _) = accept_line(cmd, 2, "```"); + let result = finalize_command(cmd); assert!(result.warnings.is_empty()); } #[test] fn single_line_has_no_warning() { - let cmd = start_command(header_from("/help"), 0); - + // §5.1 (implied): a single-line command is always complete; no warning emitted. + let cmd = start_command(header_from("/help"), 0, 0); let result = finalize_command(cmd); - assert!(result.warnings.is_empty()); } - // --- Finalized command structure --- + // --- payload --- #[test] - fn finalized_command_has_correct_name() { - let cmd = start_command(header_from("/deploy production"), 0); - + fn single_line_payload_equals_header_text() { + // §5.1: in single-line mode header and payload contain the same string. + let cmd = start_command(header_from("/hello world"), 0, 0); let result = finalize_command(cmd); - - assert_eq!(result.command.name, "deploy"); + assert_eq!(result.command.arguments.payload, "world"); + assert_eq!(result.command.arguments.mode, ArgumentMode::SingleLine); } #[test] - fn finalized_command_has_correct_range() { - let cmd = start_command(header_from("/cmd ```"), 0); - let (cmd, _) = accept_line(cmd, 1, "body"); - let (cmd, _) = accept_line(cmd, 2, "```"); - + fn single_line_no_args_has_empty_payload() { + // §5.1 (implied): a command with no arguments has an empty header and empty payload. + let cmd = start_command(header_from("/ping"), 0, 0); let result = finalize_command(cmd); - - assert_eq!(result.command.range.start_line, 0); - assert_eq!(result.command.range.end_line, 2); + assert_eq!(result.command.arguments.payload, ""); + assert_eq!(result.command.arguments.header, ""); } #[test] - fn finalized_fence_payload_is_joined_lines() { - let cmd = start_command(header_from("/cmd ```"), 0); + fn fence_payload_is_body_lines_joined() { + // §5.2.2: fence payload is the verbatim body lines joined with newline separators. + // §5.2.4: the closing fence line is not included in the payload. + // Order matters: body accumulation (§5.2.2) must be verified against closer exclusion (§5.2.4). + let cmd = start_command(header_from("/cmd ```"), 0, 0); let (cmd, _) = accept_line(cmd, 1, "line one"); let (cmd, _) = accept_line(cmd, 2, "line two"); let (cmd, _) = accept_line(cmd, 3, "```"); - let result = finalize_command(cmd); - assert_eq!(result.command.arguments.payload, "line one\nline two"); assert_eq!(result.command.arguments.mode, ArgumentMode::Fence); } #[test] - fn finalized_continuation_payload_is_joined_lines() { - let cmd = start_command(header_from("/cmd first \\"), 0); - let (cmd, _) = accept_line(cmd, 1, "second \\"); - let (cmd, _) = accept_line(cmd, 2, "third"); - + fn empty_fence_body_has_empty_payload() { + // §5.2.2 (implied): a fence with no body lines between opener and closer has empty payload. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "```"); let result = finalize_command(cmd); - - assert_eq!(result.command.arguments.payload, "first\nsecond\nthird"); - assert_eq!(result.command.arguments.mode, ArgumentMode::Continuation); + assert_eq!(result.command.arguments.payload, ""); } - #[test] - fn finalized_single_line_payload_matches_header() { - let cmd = start_command(header_from("/hello world"), 0); + // --- range and metadata --- + #[test] + fn finalized_command_has_correct_range() { + // §2.2.1: range covers zero-based physical line numbers from opener through closer. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "body"); + let (cmd, _) = accept_line(cmd, 2, "```"); let result = finalize_command(cmd); + assert_eq!(result.command.range.start_line, 0); + assert_eq!(result.command.range.end_line, 2); + } - assert_eq!(result.command.arguments.payload, "world"); - assert_eq!(result.command.arguments.mode, ArgumentMode::SingleLine); + #[test] + fn finalized_command_name_matches_header() { + // §3.1 (implied): the command name is carried through the pipeline unchanged. + let cmd = start_command(header_from("/deploy production"), 0, 0); + let result = finalize_command(cmd); + assert_eq!(result.command.name, "deploy"); } #[test] - fn finalized_fence_captures_language() { - let cmd = start_command(header_from("/code ```rust"), 0); + fn fence_lang_preserved_in_arguments() { + // §5.2.1: fence_lang is the language identifier from the opener; null if absent. + let cmd = start_command(header_from("/code ```rust"), 0, 0); let (cmd, _) = accept_line(cmd, 1, "fn main() {}"); let (cmd, _) = accept_line(cmd, 2, "```"); - let result = finalize_command(cmd); + assert_eq!(result.command.arguments.fence_lang, Some("rust".to_string())); + } - assert_eq!( - result.command.arguments.fence_lang, - Some("rust".to_string()) - ); + #[test] + fn fence_without_lang_has_none_fence_lang() { + // §5.2.1: fence_lang is None when no language identifier follows the backtick run. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "```"); + let result = finalize_command(cmd); + assert_eq!(result.command.arguments.fence_lang, None); } + // --- Property tests --- + use proptest::prelude::*; fn valid_command_name() -> impl Strategy { @@ -201,29 +280,43 @@ mod tests { } proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn id_format_is_cmd_n( + // §7: the id field always matches the pattern cmd-[0-9]+. + name in valid_command_name(), + n in 0usize..1000 + ) { + let input = format!("/{name} arg"); + let cmd = start_command(header_from(&input), 0, n); + let result = finalize_command(cmd); + prop_assert_eq!(result.command.id, format!("cmd-{n}")); + } + #[test] #[cfg_attr(feature = "tdd", ignore)] fn finalized_name_matches_pending_name(name in valid_command_name()) { + // §3.1 (implied): name survives the full start_command -> finalize_command pipeline. let input = format!("/{name} arg"); - let cmd = start_command(header_from(&input), 0); + let cmd = start_command(header_from(&input), 0, 0); let result = finalize_command(cmd); prop_assert_eq!(result.command.name, name); } #[test] #[cfg_attr(feature = "tdd", ignore)] - fn closed_commands_never_warn( + fn closed_fence_never_warns( + // §5.2.3: a properly closed fence produces zero warnings regardless of body content. name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9]{1,30}", 1..8) + body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 1..8) ) { let input = format!("/{name} ```"); - let cmd = start_command(header_from(&input), 0); + let cmd = start_command(header_from(&input), 0, 0); let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { let (next, _) = accept_line(cmd, i + 1, line); next }); let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); - let result = finalize_command(cmd); prop_assert!(result.warnings.is_empty()); } @@ -231,38 +324,60 @@ mod tests { #[test] #[cfg_attr(feature = "tdd", ignore)] fn unclosed_fence_always_warns( + // §5.2.5: any fence that reaches EOF without a closer produces exactly one warning. name in valid_command_name(), body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 1..5) ) { let input = format!("/{name} ```"); - let cmd = start_command(header_from(&input), 0); + let cmd = start_command(header_from(&input), 0, 0); let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { let (next, _) = accept_line(cmd, i + 1, line); next }); - let result = finalize_command(cmd); prop_assert_eq!(result.warnings.len(), 1); - let is_unclosed_fence = matches!(result.warnings[0], ParseWarning::UnclosedFence { .. }); - prop_assert!(is_unclosed_fence); + prop_assert_eq!(&result.warnings[0].wtype, "unclosed-fence"); } #[test] #[cfg_attr(feature = "tdd", ignore)] - fn payload_is_payload_lines_joined( + fn fence_payload_equals_body_lines_joined( + // §5.2.2: payload is exactly the body lines joined with "\n", verbatim. name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 1..8) + body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,20}", 1..8) ) { let input = format!("/{name} ```"); - let cmd = start_command(header_from(&input), 0); + let cmd = start_command(header_from(&input), 0, 0); let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { let (next, _) = accept_line(cmd, i + 1, line); next }); let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); - let result = finalize_command(cmd); prop_assert_eq!(result.command.arguments.payload, body_lines.join("\n")); } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn raw_newline_count_equals_physical_lines_minus_one( + // §8.2: raw joins all physical lines so its newline count equals consumed lines - 1. + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 0..6) + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0, 0); + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); + let result = finalize_command(cmd); + // opener + body + closer = body_lines.len() + 2 physical lines + let expected_newlines = body_lines.len() + 1; + prop_assert_eq!( + result.command.raw.chars().filter(|&c| c == '\n').count(), + expected_newlines + ); + } } } diff --git a/parser/src/application/document_parse.rs b/parser/src/application/document_parse.rs index a9da645..5847630 100644 --- a/parser/src/application/document_parse.rs +++ b/parser/src/application/document_parse.rs @@ -1,452 +1,484 @@ use super::{ command_accumulate::{AcceptResult, PendingCommand, accept_line, start_command}, command_finalize::finalize_command, - line_classify::{LineKind, classify_line}, + line_classify::{CommandHeader, LineKind, classify_line}, + line_join::LineJoiner, + normalize::normalize, text_collect::{PendingText, append_text, finalize_text, start_text}, }; -use crate::domain::{Command, ParseResult, ParseWarning, ParserContext, SPEC_VERSION, TextBlock}; +use crate::domain::{Command, ParseResult, TextBlock, Warning, SPEC_VERSION}; -struct ParseState { - current_cmd: Option, +// --- Types --- + +#[derive(Debug)] +enum ParserState { + Idle, + InFence(PendingCommand), +} + +#[derive(PartialEq, Eq)] +enum LoopAction { + Continue, + Break, +} + +struct ParseCtx { + state: ParserState, current_text: Option, commands: Vec, - text_blocks: Vec, - warnings: Vec, + textblocks: Vec, + warnings: Vec, + cmd_seq: usize, + text_seq: usize, } +impl ParseCtx { + fn new() -> Self { + Self { + state: ParserState::Idle, + current_text: None, + commands: Vec::new(), + textblocks: Vec::new(), + warnings: Vec::new(), + cmd_seq: 0, + text_seq: 0, + } + } + + fn into_result(self) -> ParseResult { + ParseResult { + version: SPEC_VERSION.to_owned(), + commands: self.commands, + textblocks: self.textblocks, + warnings: self.warnings, + } + } +} + +// --- Public entry point --- + pub fn parse_document(input: &str) -> ParseResult { - let state = input - .lines() - .enumerate() - .fold(empty_state(), |state, (i, line)| step(state, i, line)); + let normalized = normalize(input); + let physical_lines = split_physical_lines(&normalized); + let owned: Vec = physical_lines.iter().map(|s| s.to_string()).collect(); + let mut joiner = LineJoiner::new(owned); + let mut ctx = ParseCtx::new(); - finalize(state) + while step(&mut ctx, &mut joiner, &physical_lines) == LoopAction::Continue {} + + flush_text(&mut ctx); + ctx.into_result() } -fn empty_state() -> ParseState { - ParseState { - current_cmd: None, - current_text: None, - commands: Vec::new(), - text_blocks: Vec::new(), - warnings: Vec::new(), +// --- Pipeline --- + +fn split_physical_lines(normalized: &str) -> Vec<&str> { + let mut lines: Vec<&str> = normalized.split('\n').collect(); + if lines.last() == Some(&"") { + lines.pop(); } + lines } -fn step(mut state: ParseState, line_index: usize, line: &str) -> ParseState { - if try_feed_command(&mut state, line_index, line) { - return state; +fn step(ctx: &mut ParseCtx, joiner: &mut LineJoiner, phys: &[&str]) -> LoopAction { + let state = std::mem::replace(&mut ctx.state, ParserState::Idle); + match state { + ParserState::Idle => step_idle(ctx, joiner, phys), + ParserState::InFence(cmd) => step_in_fence(ctx, joiner, cmd), } - - classify_fresh_line(&mut state, line_index, line); - state } -fn try_feed_command(state: &mut ParseState, line_index: usize, line: &str) -> bool { - let Some(cmd) = state.current_cmd.take() else { - return false; - }; +// --- State handlers --- - if !cmd.is_open { - absorb_command(state, cmd); - return false; +fn step_idle(ctx: &mut ParseCtx, joiner: &mut LineJoiner, phys: &[&str]) -> LoopAction { + let Some(ll) = joiner.next_logical() else { + return LoopAction::Break; + }; + match classify_line(&ll.text) { + LineKind::Command(mut header) => { + header.raw = phys[ll.first_physical..=ll.last_physical].join("\n"); + flush_text(ctx); + start_new_command(ctx, header, ll.first_physical); + } + LineKind::Text => { + accumulate_text(ctx, ll.first_physical, ll.last_physical, phys); + } } + LoopAction::Continue +} - let (updated_cmd, result) = accept_line(cmd, line_index, line); +fn step_in_fence( + ctx: &mut ParseCtx, + joiner: &mut LineJoiner, + cmd: PendingCommand, +) -> LoopAction { + let Some((line_idx, line)) = joiner.next_physical() else { + absorb_command(ctx, cmd); + return LoopAction::Break; + }; + let (updated, result) = accept_line(cmd, line_idx, &line); match result { AcceptResult::Consumed => { - state.current_cmd = Some(updated_cmd); - true - } - AcceptResult::Completed => { - absorb_command(state, updated_cmd); - true + ctx.state = ParserState::InFence(updated); } - AcceptResult::Rejected => { - absorb_command(state, updated_cmd); - false + AcceptResult::Completed | AcceptResult::Rejected => { + absorb_command(ctx, updated); } } + LoopAction::Continue } -fn classify_fresh_line(state: &mut ParseState, line_index: usize, line: &str) { - match classify_line(line) { - LineKind::Command(header) => { - flush_text(state); - state.current_cmd = Some(start_command(header, line_index)); - } - LineKind::Text => { - accumulate_text(state, line_index, line); - } +// --- Context helpers --- + +fn start_new_command(ctx: &mut ParseCtx, header: CommandHeader, first_physical: usize) { + let cmd = start_command(header, first_physical, ctx.cmd_seq); + ctx.cmd_seq += 1; + if cmd.is_open { + ctx.state = ParserState::InFence(cmd); + } else { + absorb_command(ctx, cmd); } } -fn absorb_command(state: &mut ParseState, cmd: PendingCommand) { +fn absorb_command(ctx: &mut ParseCtx, cmd: PendingCommand) { let finalized = finalize_command(cmd); - state.commands.push(finalized.command); - state.warnings.extend(finalized.warnings); + ctx.commands.push(finalized.command); + ctx.warnings.extend(finalized.warnings); } -fn flush_text(state: &mut ParseState) { - if let Some(text) = state.current_text.take() { - state.text_blocks.push(finalize_text(text)); - } +fn flush_text(ctx: &mut ParseCtx) { + let Some(text) = ctx.current_text.take() else { return }; + ctx.textblocks.push(finalize_text(text, ctx.text_seq)); + ctx.text_seq += 1; } -fn accumulate_text(state: &mut ParseState, line_index: usize, line: &str) { - state.current_text = match state.current_text.take() { - Some(text) => Some(append_text(text, line_index, line)), - None => Some(start_text(line_index, line)), +fn accumulate_text(ctx: &mut ParseCtx, first: usize, last: usize, phys: &[&str]) { + let text = match ctx.current_text.take() { + Some(existing) => fold_physical_lines(existing, first, last, phys), + None => { + let started = start_text(first, phys[first]); + fold_physical_lines(started, first + 1, last, phys) + } }; + ctx.current_text = Some(text); } -fn finalize(mut state: ParseState) -> ParseResult { - if let Some(cmd) = state.current_cmd.take() { - absorb_command(&mut state, cmd); - } - - flush_text(&mut state); - - ParseResult { - commands: state.commands, - text_blocks: state.text_blocks, - warnings: state.warnings, - version: SPEC_VERSION.to_owned(), - context: ParserContext::default(), +fn fold_physical_lines( + mut text: PendingText, + from: usize, + to: usize, + phys: &[&str], +) -> PendingText { + for (idx, line) in (from..=to).zip(&phys[from..=to]) { + text = append_text(text, idx, line); } + text } #[cfg(test)] mod tests { - use proptest::prelude::*; - - use crate::{application::document_parse::parse_document, domain::ArgumentMode}; - - fn parse(input: &str) -> crate::domain::ParseResult { - parse_document(input) - } + use super::parse_document; + use crate::domain::{ArgumentMode, SPEC_VERSION}; - fn assert_single_command(input: &str) -> crate::domain::Command { - let result = parse(input); - assert_eq!(result.commands.len(), 1, "expected exactly 1 command"); - result.commands.into_iter().next().unwrap() - } + // --- Category 1: Empty / trivial input --- #[test] - fn single_line_simple_command_parses_name() { - let cmd = assert_single_command("/hello world"); - assert_eq!(cmd.name, "hello"); + fn empty_input_produces_empty_result() { + // §8.1 (implied): empty input produces envelope with empty arrays. + let result = parse_document(""); + assert!(result.commands.is_empty()); + assert!(result.textblocks.is_empty()); + assert!(result.warnings.is_empty()); } #[test] - fn trailing_newline_does_not_create_empty_text_block() { - let result = parse("/cmd arg\n"); - assert_eq!(result.text_blocks.len(), 0); + fn whitespace_only_input_is_text() { + // §6 (implied): lines that are not commands become text blocks. + let result = parse_document(" "); + assert!(result.commands.is_empty()); + assert_eq!(result.textblocks.len(), 1); } + // --- Category 2: Single-line command threading --- + #[test] - fn single_line_mode_threads_through_from_classify() { - let cmd = assert_single_command("/hello world"); - assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); + fn single_line_command_parses_name() { + // §5.1: command name is extracted and threaded to output. + let result = parse_document("/deploy production"); + assert_eq!(result.commands.len(), 1); + assert_eq!(result.commands.first().unwrap().name, "deploy"); } #[test] - fn single_line_payload_threads_through_from_classify() { - let cmd = assert_single_command("/hello world"); - assert_eq!(cmd.arguments.payload, "world"); + fn single_line_command_with_no_args_has_empty_payload() { + // §5.1 (implied): command with no arguments produces empty header and payload. + let result = parse_document("/ping"); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.arguments.header, ""); + assert_eq!(cmd.arguments.payload, ""); } #[test] - fn unclosed_continuation_produces_warning() { - let input = "/cmd start \\"; - let result = parse(input); - assert_eq!(result.commands.len(), 1); - assert_eq!(result.warnings.len(), 1); + fn single_line_command_range_is_same_line() { + // §2.2.1: range uses physical line numbers. Single logical line -> start == end. + let result = parse_document("/deploy production"); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 0); } #[test] - fn leading_whitespace_command_threads_through() { - let cmd = assert_single_command(" /hello world"); - assert_eq!(cmd.name, "hello"); + fn single_line_mode_threads_through() { + // §5.1: mode is single-line when no fence opener present. + let result = parse_document("/deploy production"); + assert_eq!( + result.commands.first().unwrap().arguments.mode, + ArgumentMode::SingleLine + ); } #[test] - fn single_line_command_with_no_args_has_empty_header() { - let cmd = assert_single_command("/ping"); - assert_eq!(cmd.arguments.header, ""); - assert_eq!(cmd.arguments.payload, ""); + fn single_line_payload_threads_through() { + // §5.1: payload equals the full arguments text. + let result = parse_document("/deploy production --region us-west-2"); + assert_eq!( + result.commands.first().unwrap().arguments.payload, + "production --region us-west-2" + ); } + // --- Category 3: Fenced command pipeline --- + #[test] - fn single_line_command_with_complex_args() { - let cmd = assert_single_command(r#"/mcp call_tool read_file {"path": "src/index.ts"}"#); - assert_eq!(cmd.name, "mcp"); + fn fenced_command_parses_through_document() { + // §5.2.2: fence body lines appended verbatim, joined with \n separators. + let result = parse_document("/cmd ```\nline one\nline two\n```"); + assert_eq!(result.commands.len(), 1); assert_eq!( - cmd.arguments.header, - r#"call_tool read_file {"path": "src/index.ts"}"# + result.commands.first().unwrap().arguments.payload, + "line one\nline two" ); } #[test] - fn single_line_command_range_is_same_line() { - let cmd = assert_single_command("/test"); + fn fenced_command_range_covers_opener_through_closer() { + // §2.2.1: range.start_line and range.end_line are physical line numbers. + let result = parse_document("/cmd ```\nbody\n```"); + let cmd = result.commands.first().unwrap(); assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 0); + assert_eq!(cmd.range.end_line, 2); } #[test] - fn parses_text_and_single_command() { - let input = "intro\n/cmd arg\noutro"; - let result = parse_document(input); - assert_eq!(result.commands.len(), 1); - assert_eq!(result.text_blocks.len(), 2); - assert_eq!(result.commands[0].name, "cmd"); - assert_eq!(result.commands[0].arguments.mode, ArgumentMode::SingleLine); + fn fence_lang_threads_through() { + // §5.2.1: fence_lang is the language identifier from the opener. + let result = parse_document("/cmd ```json\n{}\n```"); + assert_eq!( + result.commands.first().unwrap().arguments.fence_lang, + Some("json".to_string()) + ); } - #[test] - fn command_on_line_two_has_correct_range() { - let input = "text\nmore text\n/cmd arg"; - let result = parse(input); - assert_eq!(result.commands[0].range.start_line, 2); - assert_eq!(result.commands[0].range.end_line, 2); - } + // --- Category 4: Unclosed fence --- #[test] - fn two_single_line_commands_both_parse() { - let result = parse("/first a\n/second b"); - assert_eq!(result.commands.len(), 2); - assert_eq!(result.commands[0].name, "first"); - assert_eq!(result.commands[1].name, "second"); + fn unclosed_fence_produces_warning() { + // §5.2.5: unclosed fence emits warning with type "unclosed-fence". + let result = parse_document("/cmd ```\norphaned body"); + assert_eq!(result.warnings.len(), 1); + assert_eq!(result.warnings.first().unwrap().wtype, "unclosed-fence"); } #[test] - fn empty_input_produces_empty_result() { - let result = parse(""); - assert_eq!(result.commands.len(), 0); - assert_eq!(result.text_blocks.len(), 0); - assert_eq!(result.warnings.len(), 0); + fn unclosed_fence_still_produces_command() { + // §5.2.5: parser emits the command with its partial payload. + let result = parse_document("/cmd ```\npartial"); + assert_eq!(result.commands.len(), 1); + assert_eq!( + result.commands.first().unwrap().arguments.payload, + "partial" + ); } + // --- Category 5: Text block accumulation --- + #[test] fn text_only_produces_one_text_block() { - let result = parse("just some text"); - assert_eq!(result.commands.len(), 0); - assert_eq!(result.text_blocks.len(), 1); + // §6: consecutive non-command lines form a single text block. + let result = parse_document("just some text"); + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().content, "just some text"); } #[test] fn adjacent_text_lines_merge_into_one_block() { - let result = parse("line one\nline two\nline three"); - assert_eq!(result.text_blocks.len(), 1); - assert_eq!(result.text_blocks[0].range.start_line, 0); - assert_eq!(result.text_blocks[0].range.end_line, 2); + // §6: consecutive non-command lines form a single text block. + let result = parse_document("line one\nline two\nline three"); + assert_eq!(result.textblocks.len(), 1); + assert_eq!( + result.textblocks.first().unwrap().content, + "line one\nline two\nline three" + ); } #[test] - fn text_block_before_command_has_correct_content() { - let result = parse("intro line\n/cmd arg"); - assert_eq!(result.text_blocks.len(), 1); - assert_eq!(result.text_blocks[0].content, "intro line"); + fn text_block_content_uses_physical_lines() { + // ADR-NNNN: text block content preserves original physical lines. + let result = parse_document("hello\nworld"); + assert_eq!(result.textblocks.first().unwrap().content, "hello\nworld"); } #[test] - fn text_block_after_command_has_correct_range() { - let result = parse("/cmd arg\noutro line"); - assert_eq!(result.text_blocks.len(), 1); - assert_eq!(result.text_blocks[0].range.start_line, 1); + fn text_block_range_covers_physical_lines() { + // §6: text blocks use physical line numbers for their range. + let result = parse_document("line one\nline two"); + let tb = result.textblocks.first().unwrap(); + assert_eq!(tb.range.start_line, 0); + assert_eq!(tb.range.end_line, 1); } + // --- Category 6: Interleaving commands and text --- + #[test] - fn continuation_command_parses_through_document() { - let input = "/cmd first \\\nsecond \\\nthird"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Continuation); - assert_eq!(cmd.arguments.payload, "first\nsecond\nthird"); + fn text_before_command_is_captured() { + // §6: non-command lines before a command form a text block. + let result = parse_document("preamble\n/cmd arg"); + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().content, "preamble"); + assert_eq!(result.commands.len(), 1); } #[test] - fn fenced_command_parses_through_document() { - let input = "/cmd ```\nline one\nline two\n```"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); - assert_eq!(cmd.arguments.payload, "line one\nline two"); + fn text_after_command_is_captured() { + // §6: new text block begins after a command is finalized. + let result = parse_document("/cmd arg\npostamble"); + assert_eq!(result.commands.len(), 1); + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().content, "postamble"); } #[test] - fn unclosed_fence_produces_warning() { - let input = "/cmd ```\nline one\nline two"; - let result = parse(input); - assert_eq!(result.commands.len(), 1); - assert_eq!(result.warnings.len(), 1); + fn text_between_commands_is_captured() { + // §6: text between two commands forms its own block. + let result = parse_document("/cmd1 a\nmiddle text\n/cmd2 b"); + assert_eq!(result.commands.len(), 2); + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().content, "middle text"); } #[test] - fn fence_followed_by_new_command() { - let input = "/first ```\nbody\n```\n/second arg"; - let result = parse(input); + fn two_single_line_commands_both_parse() { + // §7: multiple commands assigned sequential IDs. + let result = parse_document("/cmd1 a\n/cmd2 b"); assert_eq!(result.commands.len(), 2); - assert_eq!(result.commands[0].name, "first"); - assert_eq!(result.commands[1].name, "second"); + assert_eq!(result.commands.first().unwrap().name, "cmd1"); + assert_eq!(result.commands.get(1).unwrap().name, "cmd2"); } #[test] - fn continuation_followed_by_new_command() { - let input = "/first start \\\nend\n/second arg"; - let result = parse(input); + fn fence_followed_by_new_command() { + // §4.1: after fence closes, parser returns to idle and scans for next command. + let result = parse_document("/cmd1 ```\nbody\n```\n/cmd2 arg"); assert_eq!(result.commands.len(), 2); - assert_eq!(result.commands[0].name, "first"); - assert_eq!(result.commands[1].name, "second"); + assert_eq!(result.commands.first().unwrap().arguments.mode, ArgumentMode::Fence); + assert_eq!(result.commands.get(1).unwrap().arguments.mode, ArgumentMode::SingleLine); } + // --- Category 7: ID assignment --- + #[test] - fn version_is_set() { - let result = parse(""); - assert_eq!(result.version, crate::domain::SPEC_VERSION); + fn command_ids_are_sequential_zero_based() { + // §7: commands assigned cmd-0, cmd-1, cmd-2. + let result = parse_document("/a x\n/b y\n/c z"); + assert_eq!(result.commands.first().unwrap().id, "cmd-0"); + assert_eq!(result.commands.get(1).unwrap().id, "cmd-1"); + assert_eq!(result.commands.get(2).unwrap().id, "cmd-2"); } - // --- Property tests --- + #[test] + fn text_block_ids_are_sequential_zero_based() { + // §7: text blocks assigned text-0, text-1, text-2. + let result = parse_document("aaa\n/cmd x\nbbb\n/cmd y\nccc"); + assert_eq!(result.textblocks.first().unwrap().id, "text-0"); + assert_eq!(result.textblocks.get(1).unwrap().id, "text-1"); + assert_eq!(result.textblocks.get(2).unwrap().id, "text-2"); + } - fn valid_command_name() -> impl Strategy { - "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + #[test] + fn command_and_text_ids_are_independent() { + // §7: command and text block ID sequences are independent. + let result = parse_document("prose\n/cmd arg\nmore prose"); + assert_eq!(result.commands.first().unwrap().id, "cmd-0"); + assert_eq!(result.textblocks.first().unwrap().id, "text-0"); + assert_eq!(result.textblocks.get(1).unwrap().id, "text-1"); } - fn command_line() -> impl Strategy { - (valid_command_name(), "[a-z0-9 ]{0,30}").prop_map(|(name, args)| format!("/{name} {args}")) + // --- Category 8: raw field --- + + #[test] + fn single_line_raw_is_header_line_only() { + // §8.2: single-line command raw contains only the original command line. + let result = parse_document("/echo hello world"); + assert_eq!(result.commands.first().unwrap().raw, "/echo hello world"); } - fn text_line() -> impl Strategy { - "[a-zA-Z0-9 !.,]{1,60}".prop_filter("must not start with slash", |s| { - !s.trim_start().starts_with('/') - }) + #[test] + fn joined_command_raw_includes_backslashes() { + // §8.2 + ADR-NNNN: raw contains physical lines with backslashes and \n separators. + // The physical_lines slice and join logic is exclusively in document_parse.rs. + let result = parse_document("/deploy prod \\\n --region us-west-2"); + assert_eq!( + result.commands.first().unwrap().raw, + "/deploy prod \\\n --region us-west-2" + ); } - proptest! { - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn never_panics_on_arbitrary_input(input in "[\\x00-\\x7F]{0,500}") { - let _ = parse_document(&input); - } + #[test] + fn fenced_raw_includes_opener_body_and_closer() { + // §8.2: fenced command raw includes command line, body lines, and closing fence. + let result = parse_document("/cmd ```\nbody\n```"); + assert_eq!(result.commands.first().unwrap().raw, "/cmd ```\nbody\n```"); + } - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn command_count_matches_slash_lines( - commands in prop::collection::vec(command_line(), 1..10) - ) { - let input = commands.join("\n"); - let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), commands.len()); - } + // --- Category 10: Trailing newline edge case --- - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn text_only_input_produces_zero_commands( - lines in prop::collection::vec(text_line(), 1..20) - ) { - let input = lines.join("\n"); - let result = parse_document(&input); - prop_assert!(result.commands.is_empty()); - prop_assert!(!result.text_blocks.is_empty()); - } + #[test] + fn trailing_newline_does_not_create_empty_text_block() { + // §2.1 + ADR-NNNN: split_physical_lines pops the trailing empty element. + let result = parse_document("/cmd arg\n"); + assert_eq!(result.commands.len(), 1); + assert!(result.textblocks.is_empty()); + } - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn version_is_always_spec_version(input in "[\\x20-\\x7E]{0,200}") { - let result = parse_document(&input); - prop_assert_eq!(&result.version, crate::domain::SPEC_VERSION); - } + // --- Category 12: Version --- - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn fenced_content_preserved_verbatim( - name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9 !]{1,50}", 1..5) - ) { - let body = body_lines.join("\n"); - let input = format!("/{name} ```\n{body}\n```"); - let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), 1); - prop_assert_eq!(&result.commands[0].arguments.payload, &body); - prop_assert_eq!(&result.commands[0].arguments.mode, &ArgumentMode::Fence); - } + #[test] + fn version_is_set() { + // §8.1: envelope contains version field set to SPEC_VERSION. + let result = parse_document(""); + assert_eq!(result.version, SPEC_VERSION); + } - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn interleaved_text_and_commands_both_captured( - cmd_count in 1usize..5 - ) { - let mut lines = Vec::new(); - for i in 0..cmd_count { - lines.push(format!("text line {i}")); - lines.push(format!("/cmd{i} arg")); - } - let input = lines.join("\n"); - let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), cmd_count); - prop_assert_eq!(result.text_blocks.len(), cmd_count); - } + // --- Property tests --- + // Separated from watch mode via tdd feature flag. - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn unclosed_fence_always_produces_warning( - name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9]{1,30}", 1..5) - ) { - let body = body_lines.join("\n"); - let input = format!("/{name} ```\n{body}"); - let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), 1); - prop_assert_eq!(result.warnings.len(), 1); - } + use proptest::prelude::*; + proptest! { #[test] #[cfg_attr(feature = "tdd", ignore)] - fn text_block_content_matches_non_command_lines( - lines in prop::collection::vec(text_line(), 1..10) - ) { - let input = lines.join("\n"); - let result = parse_document(&input); - prop_assert_eq!(result.text_blocks.len(), 1); - prop_assert_eq!(&result.text_blocks[0].content, &input); - } - - fn continuation_payload_matches_joined_lines( - name in valid_command_name(), - middle_lines in prop::collection::vec("[a-z0-9]{1,20}", 0..5), - final_line in "[a-z0-9]{1,20}" - ) { - let mut input_lines = vec![format!("/{name} start \\")]; - let mut expected_parts = vec!["start".to_string()]; - for line in &middle_lines { - input_lines.push(format!("{line} \\")); - expected_parts.push(line.clone()); - } - input_lines.push(final_line.clone()); - expected_parts.push(final_line); - let input = input_lines.join("\n"); - - let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), 1); - prop_assert_eq!(&result.commands[0].arguments.mode, &ArgumentMode::Continuation); - prop_assert_eq!(&result.commands[0].arguments.payload, &expected_parts.join("\n")); + fn never_panics_on_arbitrary_input(input in "\\PC{0,500}") { + // §8.1 (total function): parser always produces a valid envelope, never panics. + let _ = parse_document(&input); } #[test] #[cfg_attr(feature = "tdd", ignore)] - fn unclosed_continuation_always_produces_warning( - name in valid_command_name(), - middle_lines in prop::collection::vec("[a-z0-9]{1,20}", 0..3) - ) { - let mut input_lines = vec![format!("/{name} start \\")]; - for line in &middle_lines { - input_lines.push(format!("{line} \\")); - } - let input = input_lines.join("\n"); - + fn version_is_always_spec_version(input in "\\PC{0,200}") { + // §8.1: version field is always SPEC_VERSION regardless of input. let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), 1); - prop_assert_eq!(result.warnings.len(), 1); + prop_assert_eq!(result.version, SPEC_VERSION); } - } + } diff --git a/parser/src/application/line_classify.rs b/parser/src/application/line_classify.rs index c95be82..c604c28 100644 --- a/parser/src/application/line_classify.rs +++ b/parser/src/application/line_classify.rs @@ -1,13 +1,17 @@ use crate::domain::ArgumentMode; -/// Raw classification of a single input line. +/// Raw classification of a single logical input line. #[derive(Debug, Clone, PartialEq, Eq)] pub enum LineKind { Command(CommandHeader), Text, } -/// Extracted header fields from a line that starts a slash command. +/// Extracted header fields from a line that opens a slash command. +/// +/// `fence_backtick_count` is the length of the backtick run that opened the +/// fence (e.g. 3 for ` ``` `, 4 for ` ```` `). It is 0 for single-line commands. +/// The state machine uses this to recognise the matching closer (§5.2.4). #[derive(Debug, Clone, PartialEq, Eq)] pub struct CommandHeader { pub raw: String, @@ -15,9 +19,10 @@ pub struct CommandHeader { pub header_text: String, pub mode: ArgumentMode, pub fence_lang: Option, + pub fence_backtick_count: usize, } -/// Classify a single line as either a command header or plain text. +/// Classify a single logical line as either a command header or plain text. pub fn classify_line(line: &str) -> LineKind { match try_parse_command(line) { Some(header) => LineKind::Command(header), @@ -25,6 +30,29 @@ pub fn classify_line(line: &str) -> LineKind { } } +/// Find the first run of 3 or more consecutive backtick characters in `s`. +/// +/// Returns `(start_byte_offset, backtick_count)`. +fn find_fence_opener(s: &str) -> Option<(usize, usize)> { + let bytes = s.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'`' { + let start = i; + while i < bytes.len() && bytes[i] == b'`' { + i += 1; + } + let count = i - start; + if count >= 3 { + return Some((start, count)); + } + } else { + i += 1; + } + } + None +} + fn try_parse_command(line: &str) -> Option { let trimmed = line.trim_start(); if !trimmed.starts_with('/') { @@ -33,62 +61,55 @@ fn try_parse_command(line: &str) -> Option { let without_slash = &trimmed[1..]; - // Command name is the first contiguous non-whitespace run. let mut parts = without_slash.splitn(2, char::is_whitespace); let name_raw = parts.next().filter(|n| !n.is_empty())?; - // Command must begin with a lowercase ASCII letter and may contain lowercase letters, digits, and hyphens + // §3.1: [a-z][a-z0-9-]* if !name_raw.starts_with(|c: char| c.is_ascii_lowercase()) { return None; } - if !name_raw .chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-' || c == '_') + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') { return None; } + let name = name_raw.to_string(); let rest = parts.next().unwrap_or("").trim_start(); - // Fence mode: rest starts with ``` - if let Some(stripped) = rest.strip_prefix("```") { - let after_ticks = stripped.trim(); - let lang = after_ticks - .split_whitespace() - .next() - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()); - - return Some(CommandHeader { - raw: line.to_string(), - name, - header_text: rest.to_string(), - mode: ArgumentMode::Fence, - fence_lang: lang, - }); - } - - // Continuation mode: rest ends with trailing backslash. - if rest.ends_with('\\') { - let header_text = rest.trim_end_matches('\\').trim_end().to_string(); + // §5.2.1: detect first occurrence of 3+ backticks anywhere in the args. + if let Some((fence_start, fence_count)) = find_fence_opener(rest) { + let header_text = rest[..fence_start].trim_end().to_string(); + let after_ticks = &rest[fence_start + fence_count..]; + // "trimmed of leading whitespace, if non-empty and consisting of a + // single token (no internal whitespace)" + let after_trimmed = after_ticks.trim(); + let fence_lang = + if !after_trimmed.is_empty() && !after_trimmed.contains(char::is_whitespace) { + Some(after_trimmed.to_string()) + } else { + None + }; return Some(CommandHeader { raw: line.to_string(), name, header_text, - mode: ArgumentMode::Continuation, - fence_lang: None, + mode: ArgumentMode::Fence, + fence_lang, + fence_backtick_count: fence_count, }); } - // Single-line mode (default). + // §5.1: single-line mode — no fence opener present. Some(CommandHeader { raw: line.to_string(), name, header_text: rest.to_string(), mode: ArgumentMode::SingleLine, fence_lang: None, + fence_backtick_count: 0, }) } @@ -96,107 +117,242 @@ fn try_parse_command(line: &str) -> Option { mod tests { use super::*; + // --- find_fence_opener --- + #[test] - fn plain_text_line() { - assert_eq!(classify_line("hello world"), LineKind::Text); + fn find_fence_opener_empty_returns_none() { + // Structural: no input, no opener. + assert!(find_fence_opener("").is_none()); } #[test] - fn bare_slash_is_text() { - assert_eq!(classify_line("/ "), LineKind::Text); + fn find_fence_opener_no_backticks_returns_none() { + // Structural: absence of backticks cannot produce an opener. + assert!(find_fence_opener("hello world").is_none()); } #[test] - fn simple_command() { - let result = classify_line("/cmd some args"); - match result { - LineKind::Command(h) => { - assert_eq!(h.name, "cmd"); - assert_eq!(h.header_text, "some args"); - assert_eq!(h.mode, ArgumentMode::SingleLine); - } - _ => panic!("expected Command"), - } + fn find_fence_opener_one_backtick_returns_none() { + // §5.2.1: opener requires "three or more consecutive backtick characters". + assert!(find_fence_opener("`").is_none()); } #[test] - fn fence_command_with_lang() { - let result = classify_line("/code ```rust"); - match result { - LineKind::Command(h) => { - assert_eq!(h.name, "code"); - assert_eq!(h.mode, ArgumentMode::Fence); - assert_eq!(h.fence_lang.as_deref(), Some("rust")); - } - _ => panic!("expected Command"), - } + fn find_fence_opener_two_backticks_returns_none() { + // §5.2.1: two consecutive backticks do not meet the three-or-more threshold. + assert!(find_fence_opener("``").is_none()); } #[test] - fn continuation_command() { - let result = classify_line("/cmd first part \\"); - match result { - LineKind::Command(h) => { - assert_eq!(h.name, "cmd"); - assert_eq!(h.header_text, "first part"); - assert_eq!(h.mode, ArgumentMode::Continuation); - } - _ => panic!("expected Command"), - } + fn find_fence_opener_three_backticks_at_start() { + // §5.2.1: exactly three backticks form a valid opener at offset 0 with count 3. + assert_eq!(find_fence_opener("```"), Some((0, 3))); } #[test] - fn command_with_no_args() { - let result = classify_line("/help"); - match result { - LineKind::Command(h) => { - assert_eq!(h.name, "help"); - assert_eq!(h.header_text, ""); - assert_eq!(h.mode, ArgumentMode::SingleLine); - } - _ => panic!("expected Command"), - } + fn find_fence_opener_four_backticks_counted_correctly() { + // §5.2.1: "variable-length backtick fence (three or more)" — four ticks + // produce count 4, not two separate runs of two. + assert_eq!(find_fence_opener("````"), Some((0, 4))); } + #[test] - fn slash_alone_no_space_is_text() { - assert_eq!(classify_line("/"), LineKind::Text); + fn find_fence_opener_backticks_after_prefix() { + // §5.2.1: "first occurrence" — offset must reflect the actual position in args. + assert_eq!(find_fence_opener("call_tool write_file ```"), Some((21, 3))); } #[test] - fn command_name_starting_with_uppercase_is_text() { - assert_eq!(classify_line("/Hello"), LineKind::Text); + fn find_fence_opener_returns_first_qualifying_run() { + // §5.2.1: "first occurrence" — a second run of 3+ is ignored. + assert_eq!(find_fence_opener("``` and ```"), Some((0, 3))); } #[test] - fn command_name_starting_with_digit_is_text() { - assert_eq!(classify_line("/2fast"), LineKind::Text); + fn find_fence_opener_two_runs_of_two_returns_none() { + // §5.2.1: non-adjacent pairs do not combine into a qualifying run. + assert!(find_fence_opener("`` ``").is_none()); } #[test] - fn leading_spaces_still_detected_as_command() { - match classify_line(" /cmd arg") { - LineKind::Command(h) => assert_eq!(h.name, "cmd"), - _ => panic!("expected Command"), - } + fn find_fence_opener_tilde_is_not_backtick() { + // §5.2: "Only backtick (`) fences are recognised. Tilde (~) fences are not supported." + assert!(find_fence_opener("~~~").is_none()); } + // --- try_parse_command --- + #[test] - fn fence_without_language() { - match classify_line("/cmd ```") { - LineKind::Command(h) => { - assert_eq!(h.mode, ArgumentMode::Fence); - assert_eq!(h.fence_lang, None); - } - _ => panic!("expected Command"), - } + fn no_slash_returns_none() { + // §3: "first non-whitespace character is `/`" is required for a command. + assert!(try_parse_command("hello").is_none()); } #[test] - fn raw_preserves_original_line() { - match classify_line(" /cmd arg") { - LineKind::Command(h) => assert_eq!(h.raw, " /cmd arg"), - _ => panic!("expected Command"), - } + fn bare_slash_returns_none() { + // §3.2: "a bare `/`" is an invalid slash line treated as text. + assert!(try_parse_command("/").is_none()); + } + + #[test] + fn slash_then_space_returns_none() { + // §3.2: "/ space" — space after slash means no command name follows. + assert!(try_parse_command("/ ").is_none()); + } + + #[test] + fn uppercase_name_returns_none() { + // §3.1: pattern is `[a-z][a-z0-9-]*`; uppercase start does not match. + // §3.2: "/Hello" is an example of an invalid slash line. + assert!(try_parse_command("/Hello").is_none()); + } + + #[test] + fn digit_name_returns_none() { + // §3.1: pattern requires lowercase letter start; digit start does not match. + // §3.2: "/123" is an example of an invalid slash line. + assert!(try_parse_command("/123").is_none()); + } + + #[test] + fn underscore_in_name_returns_none() { + // §3.1: pattern `[a-z][a-z0-9-]*` — hyphens allowed, underscores are not. + assert!(try_parse_command("/cmd_foo").is_none()); + } + + #[test] + fn hyphen_in_name_is_valid() { + // §3.1: "followed by zero or more lowercase ASCII letters, ASCII digits, or hyphens". + let h = try_parse_command("/call-tool args").unwrap(); + assert_eq!(h.name, "call-tool"); + } + + #[test] + fn leading_whitespace_before_slash_is_ignored() { + // §3: "first non-whitespace character is `/`" — leading spaces are stripped. + let h = try_parse_command(" /cmd arg").unwrap(); + assert_eq!(h.name, "cmd"); + } + + #[test] + fn raw_field_preserves_original_line_with_leading_whitespace() { + // §8.2: `raw` must contain "the exact source text … as it appeared in the + // normalized input". Leading whitespace is part of that text. + let h = try_parse_command(" /cmd arg").unwrap(); + assert_eq!(h.raw, " /cmd arg"); + } + + #[test] + fn no_args_produces_empty_header_single_line() { + // §3.3: "The arguments portion may be empty (command with no arguments)." + // §5.1: mode is single-line, header is the empty arguments string. + let h = try_parse_command("/help").unwrap(); + assert_eq!(h.name, "help"); + assert_eq!(h.header_text, ""); + assert_eq!(h.mode, ArgumentMode::SingleLine); + assert_eq!(h.fence_lang, None); + assert_eq!(h.fence_backtick_count, 0); + } + + #[test] + fn single_line_header_text_is_full_args() { + // §3.4: in single-line mode "header and payload contain the same string + // (the full arguments text)". + // §3.3: "The whitespace between the command name and the arguments is + // consumed as a separator and is not included in the arguments string." + let h = try_parse_command("/deploy production --region us-west-2").unwrap(); + assert_eq!(h.header_text, "production --region us-west-2"); + assert_eq!(h.mode, ArgumentMode::SingleLine); + assert_eq!(h.fence_backtick_count, 0); + } + + #[test] + fn fence_at_start_of_args_gives_empty_header() { + // §5.2.1: "Text before the backtick run (trimmed of trailing whitespace) + // becomes arguments.header." When the backtick run is first, header is empty. + let h = try_parse_command("/cmd ```json").unwrap(); + assert_eq!(h.header_text, ""); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, Some("json".to_string())); + assert_eq!(h.fence_backtick_count, 3); + } + + #[test] + fn fence_with_preceding_header_text() { + // §5.2.1: text before the backtick run, trimmed of trailing whitespace, + // becomes header. §3.4: header serves as the dispatch/routing portion. + let h = try_parse_command("/mcp call_tool write_file ```json").unwrap(); + assert_eq!(h.header_text, "call_tool write_file"); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, Some("json".to_string())); + assert_eq!(h.fence_backtick_count, 3); + } + + #[test] + fn fence_without_lang_gives_none() { + // §5.2.1: fence_lang is the optional language identifier — None when absent. + let h = try_parse_command("/cmd ```").unwrap(); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, None); + } + + #[test] + fn fence_lang_none_when_multiple_tokens_after_ticks() { + // §5.2.1: "consisting of a single token (no internal whitespace)" — + // multiple tokens disqualify the language identifier. + let h = try_parse_command("/cmd ``` foo bar").unwrap(); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, None); + } + + #[test] + fn fence_backtick_count_four() { + // §5.2.1: "variable-length backtick fence (three or more)" — count must + // match the actual run length so the closer check can match correctly (§5.2.4). + let h = try_parse_command("/cmd ````json").unwrap(); + assert_eq!(h.fence_backtick_count, 4); + assert_eq!(h.mode, ArgumentMode::Fence); + } + + #[test] + fn tilde_fence_not_recognised() { + // §5.2: "Only backtick (`) fences are recognised. Tilde (~) fences are not supported." + let h = try_parse_command("/cmd ~~~").unwrap(); + assert_eq!(h.mode, ArgumentMode::SingleLine); + assert_eq!(h.header_text, "~~~"); + } + + #[test] + fn fence_opener_anywhere_in_args_is_detected() { + // §5.2.1: "first occurrence of three or more consecutive backtick characters" + // — the opener does not need to be at the start of the arguments. + let h = try_parse_command("/mcp call_tool write_file -c```json").unwrap(); + assert_eq!(h.header_text, "call_tool write_file -c"); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, Some("json".to_string())); + } + + // --- classify_line (public API — composition only) --- + + #[test] + fn plain_text_line_produces_text_kind() { + // §3: lines whose first non-whitespace char is not `/` are not commands. + assert_eq!(classify_line("hello world"), LineKind::Text); + } + + #[test] + fn valid_command_produces_command_kind() { + // §3: valid command lines produce LineKind::Command. + assert!(matches!(classify_line("/cmd args"), LineKind::Command(_))); + } + + #[test] + fn invalid_slash_lines_produce_text_kind() { + // §3.2: invalid slash lines are treated as ordinary text in idle state. + assert_eq!(classify_line("/Hello"), LineKind::Text); + assert_eq!(classify_line("/123"), LineKind::Text); + assert_eq!(classify_line("/"), LineKind::Text); + assert_eq!(classify_line("/ "), LineKind::Text); } // --- Property tests --- @@ -204,18 +360,16 @@ mod tests { use proptest::prelude::*; fn valid_command_name() -> impl Strategy { - "[a-z][a-z0-9\\-]{0,20}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + // §3.1: [a-z][a-z0-9-]* + "[a-z][a-z0-9\\-]{0,20}".prop_map(|s| s) } fn arbitrary_line() -> impl Strategy { prop_oneof![ - // plain text "[a-zA-Z0-9 !.,]{0,80}", - // command-like valid_command_name().prop_flat_map(|name| { "[a-zA-Z0-9 ]{0,40}".prop_map(move |args| format!("/{name} {args}")) }), - // leading whitespace + command valid_command_name().prop_flat_map(|name| { (1usize..5, "[a-zA-Z0-9 ]{0,40}").prop_map(move |(spaces, args)| { format!("{}/{} {}", " ".repeat(spaces), name, args) @@ -228,12 +382,19 @@ mod tests { #[test] #[cfg_attr(feature = "tdd", ignore)] fn classify_never_panics(line in "[\\x00-\\x7F]{0,200}") { + // §8.1 (implied): "The parser is a total function: it always produces a + // valid envelope for any input." classify_line must never panic. let _ = classify_line(&line); } #[test] #[cfg_attr(feature = "tdd", ignore)] - fn valid_name_always_produces_command(name in valid_command_name(), args in "[a-z0-9 ]{0,40}") { + fn valid_name_always_produces_command( + name in valid_command_name(), + args in "[a-z0-9 ]{0,40}" + ) { + // §3, §3.1: any line whose first non-whitespace is `/` followed by a + // name matching `[a-z][a-z0-9-]*` must produce LineKind::Command. let input = format!("/{name} {args}"); match classify_line(&input) { LineKind::Command(h) => prop_assert_eq!(h.name, name), @@ -244,12 +405,15 @@ mod tests { #[test] #[cfg_attr(feature = "tdd", ignore)] fn text_without_slash_is_never_command(line in "[a-zA-Z0-9 !.,]{1,80}") { + // §3: command detection requires the first non-whitespace char to be `/`. prop_assert!(matches!(classify_line(&line), LineKind::Text)); } #[test] #[cfg_attr(feature = "tdd", ignore)] fn raw_field_preserves_original_input(line in arbitrary_line()) { + // §8.2: raw must contain "the exact source text … as it appeared in + // the normalized input". if let LineKind::Command(h) = classify_line(&line) { prop_assert_eq!(h.raw, line); } @@ -257,7 +421,12 @@ mod tests { #[test] #[cfg_attr(feature = "tdd", ignore)] - fn fence_mode_iff_backticks_present(name in valid_command_name(), lang in "[a-z]{0,10}") { + fn fence_mode_iff_three_or_more_backticks( + name in valid_command_name(), + lang in "[a-z]{0,10}" + ) { + // §5.2.1: the first occurrence of 3+ consecutive backticks in the args + // triggers fence mode regardless of position. let input = format!("/{name} ```{lang}"); match classify_line(&input) { LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Fence), @@ -265,16 +434,33 @@ mod tests { } } + #[test] +#[cfg_attr(feature = "tdd", ignore)] +fn single_line_fence_count_is_zero( + name in valid_command_name(), + args in "[a-zA-Z0-9 ]{0,40}" +) { + // §5.1: single-line mode has no fence opener, so fence_backtick_count + // must be 0. + let input = format!("/{name} {args}"); + let LineKind::Command(h) = classify_line(&input) else { return Ok(()); }; + prop_assert_eq!(h.mode, ArgumentMode::SingleLine); + prop_assert_eq!(h.fence_backtick_count, 0); +} + + #[test] #[cfg_attr(feature = "tdd", ignore)] - fn continuation_mode_iff_trailing_backslash( + fn fence_backtick_count_matches_opener_length( name in valid_command_name(), - args in "[a-z0-9 ]{1,30}" + extra in 0usize..5 ) { - let input = format!("/{name} {args} \\"); - match classify_line(&input) { - LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Continuation), - _ => panic!("expected Command"), + // §5.2.1: "The backtick run length is recorded as the fence marker length." + // §5.2.4: the closer must have >= this many backticks. + let ticks = "`".repeat(3 + extra); + let input = format!("/{name} {ticks}json"); + if let LineKind::Command(h) = classify_line(&input) { + prop_assert_eq!(h.fence_backtick_count, 3 + extra); } } } diff --git a/parser/src/application/line_join.rs b/parser/src/application/line_join.rs new file mode 100644 index 0000000..4291efd --- /dev/null +++ b/parser/src/application/line_join.rs @@ -0,0 +1,354 @@ +/// POSIX-style backslash line joining. +/// +/// Consumes physical lines and produces logical lines. A physical line ending +/// with `\` is joined with the next physical line, separated by a single space, +/// and the backslash is removed. Joining repeats while the accumulated line +/// still ends with `\`. At EOF, a trailing `\` is silently removed. +/// +/// Fence immunity is enforced by the caller: when the state machine enters +/// `InFence`, it calls `next_physical` directly instead of `next_logical`, +/// bypassing the joiner for those lines. +pub struct LogicalLine { + pub text: String, + pub first_physical: usize, + pub last_physical: usize, +} + +pub struct LineJoiner { + lines: Vec, + cursor: usize, +} + +/// Consume the next logical line from `lines` starting at `*cursor`, +/// joining any trailing-backslash continuations. +fn consume_logical(lines: &[String], cursor: &mut usize) -> Option { + if *cursor >= lines.len() { + return None; + } + + let first_physical = *cursor; + let mut text = lines[*cursor].clone(); + *cursor += 1; + + while text.ends_with('\\') { + text.truncate(text.len() - 1); + + if *cursor >= lines.len() { + // Trailing backslash at EOF: silently removed, line stands alone. + break; + } + + text.push(' '); + text.push_str(&lines[*cursor]); + *cursor += 1; + } + + Some(LogicalLine { + text, + first_physical, + last_physical: *cursor - 1, + }) +} + +/// Consume the next raw physical line from `lines` at `*cursor`, +/// bypassing join logic entirely. +fn consume_physical(lines: &[String], cursor: &mut usize) -> Option<(usize, String)> { + if *cursor >= lines.len() { + return None; + } + let idx = *cursor; + let line = lines[*cursor].clone(); + *cursor += 1; + Some((idx, line)) +} + +impl LineJoiner { + pub fn new(lines: Vec) -> Self { + Self { lines, cursor: 0 } + } + + pub fn next_logical(&mut self) -> Option { + consume_logical(&self.lines, &mut self.cursor) + } + + /// Used by the state machine when inside a fenced block. + pub fn next_physical(&mut self) -> Option<(usize, String)> { + consume_physical(&self.lines, &mut self.cursor) + } + + pub fn is_exhausted(&self) -> bool { + self.cursor >= self.lines.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + + fn lines(xs: &[&str]) -> Vec { + xs.iter().map(|s| s.to_string()).collect() + } + + fn joiner(xs: &[&str]) -> LineJoiner { + LineJoiner::new(lines(xs)) + } + + /// Build a string ending with a backslash without embedding `\\` in literals. + fn bsl(s: &str) -> String { + let mut r = s.to_string(); + r.push('\\'); + r + } + + // --- consume_logical (private free function) --- + + #[test] + fn consume_logical_empty_returns_none() { + // Structural: cursor must not advance and None must be returned on empty input. + let ls = lines(&[]); + let mut cursor = 0; + assert!(consume_logical(&ls, &mut cursor).is_none()); + assert_eq!(cursor, 0); + } + + #[test] + fn consume_logical_no_backslash_passes_through() { + // §2.2: "Lines that do not end with `\` are left unchanged." + let ls = lines(&["/echo hello"]); + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "/echo hello"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 0); + assert_eq!(cursor, 1); + } + + #[test] + fn consume_logical_joins_two_lines() { + // §2.2 steps 1-3: remove trailing backslash, concatenate with next physical + // line separated by a single space. + let ls = vec![bsl("a"), "b".to_string()]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "a b"); + assert_eq!(ll.last_physical, 1); + assert_eq!(cursor, 2); + } + + #[test] + fn consume_logical_trailing_backslash_at_eof_removed() { + // §2.2.2: "If the final physical line ends with a backslash and there is no + // subsequent line to join with, the trailing backslash is removed and the + // line stands alone." + let ls = vec![bsl("a")]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "a"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 0); + } + + #[test] + fn consume_logical_advances_cursor_past_consumed_lines() { + // §2.2.1: the cursor must advance past all physical lines consumed by a + // single logical line so that subsequent calls produce the next logical line. + let ls = vec![bsl("x"), "y".to_string(), "z".to_string()]; + let mut cursor = 0; + consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(cursor, 2); + let ll2 = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll2.text, "z"); + } + + // --- consume_physical (private free function) --- + + #[test] + fn consume_physical_empty_returns_none() { + // Structural: cursor must not advance and None must be returned on empty input. + let ls = lines(&[]); + let mut cursor = 0; + assert!(consume_physical(&ls, &mut cursor).is_none()); + assert_eq!(cursor, 0); + } + + #[test] + fn consume_physical_returns_raw_line_with_backslash() { + // §2.3: inside a fenced block "all physical lines are consumed verbatim", + // including trailing backslashes that would otherwise be join markers. + let ls = vec![bsl("line one"), "line two".to_string()]; + let mut cursor = 0; + let (idx, line) = consume_physical(&ls, &mut cursor).unwrap(); + assert_eq!(idx, 0); + assert!(line.ends_with('\\')); + assert_eq!(cursor, 1); + } + + #[test] + fn consume_physical_does_not_join() { + // §2.3: "A trailing `\` inside a fence is literal content, not a join marker." + let ls = vec![bsl("a"), "b".to_string()]; + let mut cursor = 0; + let (_, line) = consume_physical(&ls, &mut cursor).unwrap(); + assert!(line.ends_with('\\')); + let (_, line2) = consume_physical(&ls, &mut cursor).unwrap(); + assert_eq!(line2, "b"); + } + + // --- LineJoiner impl (delegation + composition) --- + + #[test] + fn next_logical_delegates_correctly() { + // Structural: next_logical must delegate to consume_logical and advance state. + let mut j = joiner(&["/echo hello"]); + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "/echo hello"); + assert!(j.is_exhausted()); + } + + #[test] + fn next_physical_delegates_correctly() { + // Structural: next_physical must delegate to consume_physical and advance state. + let input = vec![bsl("line one"), "line two".to_string()]; + let mut j = LineJoiner::new(input); + let (idx, line) = j.next_physical().unwrap(); + assert_eq!(idx, 0); + assert!(line.ends_with('\\')); + } + + #[test] + fn interleaving_logical_and_physical_shares_cursor() { + // §2.3: the state machine calls next_logical in idle state and next_physical + // in fence state; both must share a single cursor so no lines are skipped + // or double-consumed at the transition boundary. + let input = vec![bsl("/cmd a"), " b".to_string(), "fence body".to_string()]; + let mut j = LineJoiner::new(input); + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "/cmd a b"); + assert_eq!(ll.last_physical, 1); + let (idx, line) = j.next_physical().unwrap(); + assert_eq!(idx, 2); + assert_eq!(line, "fence body"); + assert!(j.is_exhausted()); + } + + #[test] + fn is_exhausted_false_when_lines_remain() { + // Structural: is_exhausted must reflect the current cursor position accurately. + let mut j = joiner(&["a", "b"]); + assert!(!j.is_exhausted()); + j.next_logical(); + assert!(!j.is_exhausted()); + j.next_logical(); + assert!(j.is_exhausted()); + } + + // --- Spec examples --- + + #[test] + fn spec_example_three_physical_lines_join_to_one() { + // §2.2.3 example: three physical lines collapse into one logical line. + // §2.2.1: first_physical and last_physical cover the full physical range. + let input = vec![ + bsl("/mcp call_tool read_file"), + bsl(" --path src/index.ts"), + " --format json".to_string(), + ]; + let mut j = LineJoiner::new(input); + let ll = j.next_logical().unwrap(); + assert_eq!( + ll.text, + "/mcp call_tool read_file --path src/index.ts --format json" + ); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 2); + } + + #[test] + fn spec_example_trailing_backslash_at_eof_space_preserved() { + // §2.2.2 example: "/echo hello \" -> "/echo hello " — the backslash is + // removed but the space before it is part of the content and stays. + let input = vec![bsl("/echo hello ")]; + let mut j = LineJoiner::new(input); + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "/echo hello "); + } + + #[test] + fn fence_closer_line_with_trailing_backslash_joins_normally() { + // §2.2: "The join marker is any backslash character immediately before the + // physical line terminator, regardless of what precedes it. This includes + // lines that serve other syntactic roles, such as a closing fence line + // followed by a trailing backslash." + // §5.2.4: the fence close is detected by the state machine, not the joiner; + // the joiner has no special case here. + let input = vec![bsl("```"), "next line".to_string()]; + let mut j = LineJoiner::new(input); + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "``` next line"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 1); + } + + // --- Property tests --- + + proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn logical_count_lte_physical_count( + ls in prop::collection::vec("[a-zA-Z0-9 ]{0,40}", 0..20) + ) { + // §2.2: joining only reduces or maintains line count, never increases it. + let count = ls.len(); + let mut cursor = 0; + let mut logical_count = 0; + while consume_logical(&ls, &mut cursor).is_some() { + logical_count += 1; + } + prop_assert!(logical_count <= count); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn clean_lines_pass_through_unchanged( + ls in prop::collection::vec("[a-zA-Z0-9 !.,]{1,40}", 1..10) + ) { + // §2.2: "Lines that do not end with `\` are left unchanged." + let expected = ls.clone(); + let mut cursor = 0; + for expected_text in expected { + let ll = consume_logical(&ls, &mut cursor).unwrap(); + prop_assert_eq!(ll.text, expected_text); + } + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn ranges_cover_all_physical_lines( + ls in prop::collection::vec("[a-zA-Z0-9 ]{0,40}", 1..20) + ) { + // §2.2.1: logical lines must partition the physical line sequence without + // gaps or overlaps — every physical line belongs to exactly one logical line. + let count = ls.len(); + let mut cursor = 0; + let mut next_expected = 0usize; + while let Some(ll) = consume_logical(&ls, &mut cursor) { + prop_assert_eq!(ll.first_physical, next_expected); + prop_assert!(ll.first_physical <= ll.last_physical); + next_expected = ll.last_physical + 1; + } + prop_assert_eq!(next_expected, count); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn exhausted_after_consuming_all_logical_lines( + ls in prop::collection::vec("[a-zA-Z0-9 ]{0,40}", 0..20) + ) { + // Structural: is_exhausted must be true once all lines have been consumed. + let mut j = LineJoiner::new(ls); + while j.next_logical().is_some() {} + prop_assert!(j.is_exhausted()); + } + } +} diff --git a/parser/src/application/mod.rs b/parser/src/application/mod.rs index 19c9b5b..56a2e6f 100644 --- a/parser/src/application/mod.rs +++ b/parser/src/application/mod.rs @@ -1,10 +1,12 @@ +mod normalize; +mod line_join; +mod line_classify; +mod text_collect; mod command_accumulate; mod command_finalize; mod document_parse; -mod line_classify; -mod text_collect; -pub use document_parse::parse_document; +// pub use document_parse::parse_document; #[cfg(test)] mod tests; diff --git a/parser/src/application/normalize.rs b/parser/src/application/normalize.rs new file mode 100644 index 0000000..c98a3bd --- /dev/null +++ b/parser/src/application/normalize.rs @@ -0,0 +1,141 @@ +/// Normalize line endings in `input` to LF only. +/// +/// Step 1: replace all CRLF (`\r\n`) with LF (`\n`). +/// Step 2: replace any remaining bare CR (`\r`) with LF (`\n`). +/// +/// All other bytes, including literal `\n` escape sequences inside content, +/// are preserved verbatim. +pub fn normalize(input: &str) -> String { + input.replace("\r\n", "\n").replace('\r', "\n") +} + + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + + + // --- Unit tests --- + + + #[test] + fn clean_lf_input_is_unchanged() { + // §2.1: LF is the normalized form; input already using LF needs no changes. + let input = "line one\nline two\nline three"; + assert_eq!(normalize(input), input); + } + + + #[test] + fn empty_input_is_unchanged() { + // §2.1 (implied): normalize is a total function — empty input produces empty output. + assert_eq!(normalize(""), ""); + } + + + #[test] + fn crlf_becomes_lf() { + // §2.1 rule 1: "Replace all \r\n (CRLF) sequences with \n (LF)." + assert_eq!(normalize("a\r\nb"), "a\nb"); + } + + + #[test] + fn bare_cr_becomes_lf() { + // §2.1 rule 2: "Replace all remaining \r (bare CR) characters with \n." + assert_eq!(normalize("a\rb"), "a\nb"); + } + + + #[test] + fn mixed_endings_all_become_lf() { + // §2.1 rules 1-2: CRLF replacement runs first so the CR in \r\n is not + // re-matched as a bare CR and doubled into \n\n. + assert_eq!(normalize("a\r\nb\rc\nd"), "a\nb\nc\nd"); + } + + + #[test] + fn multiple_crlf_all_converted() { + // §2.1 rule 1: all CRLF occurrences are replaced, not just the first. + assert_eq!(normalize("a\r\nb\r\nc"), "a\nb\nc"); + } + + + #[test] + fn crlf_at_start_and_end() { + // §2.1 rule 1: replacement applies anywhere in the input, including boundaries. + assert_eq!(normalize("\r\nhello\r\n"), "\nhello\n"); + } + + + #[test] + fn only_cr_sequence() { + // §2.1 rule 2: bare CR with no following LF is still converted. + assert_eq!(normalize("\r\r\r"), "\n\n\n"); + } + + + #[test] + fn content_without_any_line_endings_unchanged() { + // §2.1: content with no CR or LF characters is unaffected by normalization. + let input = "no line endings here"; + assert_eq!(normalize(input), input); + } + + + #[test] + fn literal_backslash_n_in_content_is_not_a_line_terminator() { + // §2.1: "Literal escape sequences inside content (e.g., \n in a JSON string + // \"blah\nblah\") are ordinary characters, not line terminators. They are + // preserved verbatim in the output." + // Rust "\\n" = two chars: backslash + n. Not a real newline. + let input = "before\\nafter"; + assert_eq!(normalize(input), "before\\nafter"); + } + + + // --- Property tests --- + + + proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn output_never_contains_cr(input in "[\x00-\x7F]{0,500}") { + // §2.1: "After normalization, all line terminators are LF." No CR may remain. + let result = normalize(&input); + prop_assert!(!result.contains('\r')); + } + + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn idempotent(input in "[\x00-\x7F]{0,500}") { + // §2.1 (implied): normalizing already-normalized output is a no-op. + let once = normalize(&input); + let twice = normalize(&once); + prop_assert_eq!(once, twice); + } + + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn clean_input_is_unchanged(input in "[\x20-\x7E\n]{0,500}") { + // §2.1: input containing no CR characters requires no changes. + prop_assert_eq!(normalize(&input), input); + } + + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn lf_count_gte_original_lf_count(input in "[\x00-\x7F]{0,500}") { + // §2.1 rule 2: each bare CR becomes an LF, so LF count can only stay + // the same or increase after normalization. + let original_lf = input.chars().filter(|&c| c == '\n').count(); + let result = normalize(&input); + let result_lf = result.chars().filter(|&c| c == '\n').count(); + prop_assert!(result_lf >= original_lf); + } + } +} diff --git a/parser/src/application/tests/mod.rs b/parser/src/application/tests/mod.rs index cd2f278..c3b7f8f 100644 --- a/parser/src/application/tests/mod.rs +++ b/parser/src/application/tests/mod.rs @@ -1 +1,415 @@ -mod proptest; +// Layer 2 integration tests: cross-module composition within the application layer. +// +// These tests exercise combinations of modules that no single file's tests can cover. +// They use only the public API of each module. + +use super::line_classify::{classify_line, LineKind}; +use super::line_join::LineJoiner; +use super::normalize::normalize; +use super::document_parse::parse_document; +use crate::domain::ArgumentMode; + +// --- normalize + line_join --- + +#[test] +fn crlf_continuation_joins_same_as_lf() { + // §2.1 rules 1-2: CRLF and bare CR both normalize to LF before any other processing. + // §2.2: line joining runs after normalization, so a backslash before a CRLF boundary + // must produce the same logical line as the equivalent LF-only input. + let crlf = "/deploy production\\\r\n --region us-west-2"; + let lf = "/deploy production\\\n --region us-west-2"; + + let crlf_lines: Vec = normalize(crlf).split('\n').map(|s| s.to_string()).collect(); + let lf_lines: Vec = normalize(lf).split('\n').map(|s| s.to_string()).collect(); + + let crlf_ll = LineJoiner::new(crlf_lines).next_logical().unwrap(); + let lf_ll = LineJoiner::new(lf_lines).next_logical().unwrap(); + + assert_eq!(crlf_ll.text, lf_ll.text); + assert_eq!(crlf_ll.first_physical, lf_ll.first_physical); + assert_eq!(crlf_ll.last_physical, lf_ll.last_physical); +} + +#[test] +fn bare_cr_continuation_joins_same_as_lf() { + // §2.1 rule 2: remaining bare CR characters are replaced with LF after CRLF removal. + // §2.2: joining is agnostic to the original line-ending style. + let cr = "/deploy production\\\r --region us-west-2"; + let lf = "/deploy production\\\n --region us-west-2"; + + let cr_lines: Vec = normalize(cr).split('\n').map(|s| s.to_string()).collect(); + let lf_lines: Vec = normalize(lf).split('\n').map(|s| s.to_string()).collect(); + + let cr_ll = LineJoiner::new(cr_lines).next_logical().unwrap(); + let lf_ll = LineJoiner::new(lf_lines).next_logical().unwrap(); + + assert_eq!(cr_ll.text, lf_ll.text); +} + +#[test] +fn mixed_crlf_multi_line_join_matches_lf() { + // §2.1: normalization applies to all line endings uniformly. + // §2.2 step 4: joining repeats while the accumulated line still ends with `\`, + // so three physical lines collapse into one logical line regardless of ending style. + // §2.2.1: last_physical must be the zero-based index of the last consumed physical line. + let crlf = "/mcp call_tool read_file\\\r\n --path src/index.ts\\\r\n --format json"; + let lf = "/mcp call_tool read_file\\\n --path src/index.ts\\\n --format json"; + + let crlf_lines: Vec = normalize(crlf).split('\n').map(|s| s.to_string()).collect(); + let lf_lines: Vec = normalize(lf).split('\n').map(|s| s.to_string()).collect(); + + let crlf_ll = LineJoiner::new(crlf_lines).next_logical().unwrap(); + let lf_ll = LineJoiner::new(lf_lines).next_logical().unwrap(); + + assert_eq!(crlf_ll.text, lf_ll.text); + assert_eq!(crlf_ll.first_physical, 0); + assert_eq!(crlf_ll.last_physical, 2); +} + +// --- normalize + line_join + line_classify --- + +#[test] +fn joining_into_fence_opener_spec_5_2_6() { + // §5.2.6: "When backslash joining merges a command line with a line containing + // a fence opener, the fence is detected in the resulting logical line." + // §2.2.1: the logical line maps back to physical lines 0-1. + // §5.2.1: text before the backtick run becomes header; text after becomes fence_lang. + // §5.2.1: fence_backtick_count records the run length for the closer check. + // + // Input (two physical lines): + // /mcp call_tool write_file \ + // ```json + // + // After join: "/mcp call_tool write_file ```json" + // (two spaces: one trailing space before `\` + one joiner separator) + let input = normalize("/mcp call_tool write_file \\\n```json"); + let lines: Vec = input.split('\n').map(|s| s.to_string()).collect(); + let mut joiner = LineJoiner::new(lines); + + let ll = joiner.next_logical().unwrap(); + assert_eq!(ll.text, "/mcp call_tool write_file ```json"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 1); + assert!(joiner.is_exhausted()); + + match classify_line(&ll.text) { + LineKind::Command(h) => { + assert_eq!(h.name, "mcp"); + assert_eq!(h.header_text, "call_tool write_file"); + assert_eq!(h.fence_lang, Some("json".to_string())); + assert_eq!(h.fence_backtick_count, 3); + } + LineKind::Text => panic!("expected fence command, got text"), + } +} + +#[test] +fn joined_logical_line_classifies_as_single_line_command() { + // §2.2: line joining produces logical lines; the state machine then classifies them. + // §5.1: a joined result with no fence opener in the args is a single-line command. + let input = normalize("/deploy production\\\n --region us-west-2"); + let lines: Vec = input.split('\n').map(|s| s.to_string()).collect(); + let ll = LineJoiner::new(lines).next_logical().unwrap(); + + match classify_line(&ll.text) { + LineKind::Command(h) => { + assert_eq!(h.name, "deploy"); + assert_eq!(h.header_text, "production --region us-west-2"); + assert_eq!(h.mode, ArgumentMode::SingleLine); + assert_eq!(h.fence_backtick_count, 0); + } + LineKind::Text => panic!("expected single-line command, got text"), + } +} + +#[test] +fn joined_invalid_slash_line_classifies_as_text() { + // §3.2: a line whose slash is not followed by a valid name is treated as text. + // §2.2: this check runs on the logical line after joining, not on the raw physical lines. + let input = normalize("/Hello world\\\n more args"); + let lines: Vec = input.split('\n').map(|s| s.to_string()).collect(); + let ll = LineJoiner::new(lines).next_logical().unwrap(); + + assert_eq!(classify_line(&ll.text), LineKind::Text); +} + + +// --- Full pipeline: document_parse --- +// These tests exercise parse_document end-to-end across multiple modules. +// They verify cross-module composition that no single file's Layer 1 tests can cover. + + +// --- Backslash joining through full pipeline (§2.2 + §2.3) --- + +#[test] +fn joined_multi_line_command_through_pipeline() { + // §2.2: backslash continuation joins physical lines into one logical line. + // §5.1: the joined result with no fence opener is single-line mode. + let result = parse_document("/deploy production\\\n --region us-west-2"); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.name, "deploy"); + assert_eq!(cmd.arguments.header, "production --region us-west-2"); + assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); +} + +#[test] +fn trailing_backslash_at_eof_removed_through_pipeline() { + // §2.2.2: trailing backslash at EOF is silently removed. + // The command parses as single-line with the backslash stripped. + let result = parse_document("/echo hello\\"); + assert_eq!(result.commands.len(), 1); + assert_eq!(result.commands.first().unwrap().arguments.payload, "hello"); + assert!(result.warnings.is_empty()); +} + +#[test] +fn fence_immunity_backslash_inside_fence_is_literal() { + // §2.3: trailing backslash inside fence is literal content, not a join marker. + let result = parse_document("/cmd ```\nline one\\\nline two\n```"); + assert_eq!(result.commands.len(), 1); + assert_eq!(result.commands.first().unwrap().arguments.payload, "line one\\\nline two"); +} + +#[test] +fn joining_into_fence_opener_through_pipeline() { + // §5.2.6: backslash joining merges command line with fence opener line. + // Physical lines 0-1 join into the command header; lines 2-3 are fence body + closer. + let result = parse_document("/mcp call_tool write_file \\\n```json\n{\"path\": \"foo\"}\n```"); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.arguments.header, "call_tool write_file"); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.arguments.payload, "{\"path\": \"foo\"}"); +} + +// --- Text block content with continuation lines (ADR-NNNN) --- + +#[test] +fn text_block_with_continuation_preserves_backslash() { + // ADR-NNNN: text block content stores physical lines. Backslashes are retained + // because text blocks capture pre-join content for round-trip fidelity. + let result = parse_document("hello \\\nworld"); + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().content, "hello \\\nworld"); +} + +#[test] +fn text_block_continuation_range_covers_all_physical_lines() { + // ADR-NNNN + §2.2.1: range spans all physical lines consumed by a joined logical line. + let result = parse_document("hello \\\nworld"); + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().range.start_line, 0); + assert_eq!(result.textblocks.first().unwrap().range.end_line, 1); +} + +// --- Appendix A spec examples --- +// Full pipeline assertions matching spec appendix examples exactly. +// These are the Layer 2 "acceptance test" equivalents. + +#[test] +fn spec_a1_single_line_command() { + // Appendix A.1: "/echo hello world" + let result = parse_document("/echo hello world"); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "echo"); + assert_eq!(cmd.raw, "/echo hello world"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 0); + assert_eq!(cmd.arguments.header, "hello world"); + assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); + assert_eq!(cmd.arguments.fence_lang, None); + assert_eq!(cmd.arguments.payload, "hello world"); +} + +#[test] +fn spec_a2_joined_multi_line_command() { + // Appendix A.2: three physical lines joined into one command. + let input = "/deploy production \\\n --region us-west-2 \\\n --canary"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "deploy"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 2); + assert_eq!(cmd.arguments.header, "production --region us-west-2 --canary"); + assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); + assert_eq!(cmd.arguments.payload, "production --region us-west-2 --canary"); + // §8.2: raw contains physical lines with backslashes and \n separators. + assert_eq!(cmd.raw, "/deploy production \\\n --region us-west-2 \\\n --canary"); +} + +#[test] +fn spec_a3_fenced_command_with_header() { + // Appendix A.3: "/mcp call_tool write_file ```json\n{...}\n```" + let input = "/mcp call_tool write_file ```json\n{\"path\": \"src/index.ts\"}\n```"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 2); + assert_eq!(cmd.arguments.header, "call_tool write_file"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.arguments.payload, "{\"path\": \"src/index.ts\"}"); +} + +#[test] +fn spec_a4_backslash_join_into_fence() { + // Appendix A.4: four physical lines, join merges 0+1 into command+fence opener. + let input = "/mcp call_tool write_file \\\n```json\n{\"path\": \"foo\"}\n```"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 3); + assert_eq!(cmd.arguments.header, "call_tool write_file"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.arguments.payload, "{\"path\": \"foo\"}"); + // §8.2: raw includes all physical lines from opener through closer. + assert_eq!(cmd.raw, "/mcp call_tool write_file \\\n```json\n{\"path\": \"foo\"}\n```"); +} + +#[test] +fn spec_a5_text_blocks_and_multiple_commands() { + // Appendix A.5: text, two commands, trailing text. + let input = "Welcome to the deployment system.\n\n/deploy staging\n/notify team --channel ops\nDeployment complete."; + let result = parse_document(input); + + // Two commands in order. + assert_eq!(result.commands.len(), 2); + assert_eq!(result.commands.first().unwrap().id, "cmd-0"); + assert_eq!(result.commands.first().unwrap().name, "deploy"); + assert_eq!(result.commands.first().unwrap().arguments.payload, "staging"); + assert_eq!(result.commands.first().unwrap().range.start_line, 2); + assert_eq!(result.commands.get(1).unwrap().id, "cmd-1"); + assert_eq!(result.commands.get(1).unwrap().name, "notify"); + assert_eq!(result.commands.get(1).unwrap().arguments.payload, "team --channel ops"); + assert_eq!(result.commands.get(1).unwrap().range.start_line, 3); + + // Two text blocks in order. + assert_eq!(result.textblocks.len(), 2); + assert_eq!(result.textblocks.first().unwrap().id, "text-0"); + assert_eq!(result.textblocks.first().unwrap().content, "Welcome to the deployment system.\n"); + assert_eq!(result.textblocks.first().unwrap().range.start_line, 0); + assert_eq!(result.textblocks.first().unwrap().range.end_line, 1); + assert_eq!(result.textblocks.get(1).unwrap().id, "text-1"); + assert_eq!(result.textblocks.get(1).unwrap().content, "Deployment complete."); + assert_eq!(result.textblocks.get(1).unwrap().range.start_line, 4); + assert_eq!(result.textblocks.get(1).unwrap().range.end_line, 4); +} + +#[test] +fn spec_a7_unclosed_fence() { + // Appendix A.7: fence never closed, command emitted with partial payload + warning. + let input = "/mcp call_tool ```json\n{\"incomplete\": true}"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.arguments.header, "call_tool"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.arguments.payload, "{\"incomplete\": true}"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 1); + + assert_eq!(result.warnings.len(), 1); + assert_eq!(result.warnings.first().unwrap().wtype, "unclosed-fence"); + assert_eq!(result.warnings.first().unwrap().start_line, Some(0)); +} + +#[test] +fn spec_a8_closing_fence_with_trailing_backslash() { + // Appendix A.8: "```\" is NOT a valid closer (not solely backticks after trim). + // The fence never closes; all remaining lines become payload. Warning emitted. + let input = "/mcp call_tool write_file -c \\\n```json\n{\"path\": \"foo\"}\n```\\\n\\\nproduction"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + // Fence never closes: lines 2-5 are all payload. + assert!(cmd.arguments.payload.contains("{\"path\": \"foo\"}")); + + assert_eq!(result.warnings.len(), 1); + assert_eq!(result.warnings.first().unwrap().wtype, "unclosed-fence"); +} + +#[test] +fn spec_a9_proper_fence_close_followed_by_content() { + // Appendix A.9: fence closes on line 3, lines 4-5 join into text block. + let input = "/mcp call_tool write_file -c \\\n```json\n{\"path\": \"foo\"}\n```\n\\\nproduction"; + let result = parse_document(input); + + // One command with closed fence. + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.arguments.header, "call_tool write_file -c"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.arguments.payload, "{\"path\": \"foo\"}"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 3); + + // Lines 4-5 join into "production" text block. + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().content, "production"); + assert_eq!(result.textblocks.first().unwrap().range.start_line, 4); + assert_eq!(result.textblocks.first().unwrap().range.end_line, 5); +} + +// --- Property tests (Layer 2) --- +// Cross-module invariants over arbitrary input. + +use proptest::prelude::*; + +proptest! { + + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_only_input_produces_zero_commands( + lines in prop::collection::vec("[^/\\n][^\\n]{0,40}", 1..10) + ) { + // §6 + §7: input with no slash-prefixed lines produces no commands. + // Lines are guaranteed not to start with '/' so none can be commands. + let input = lines.join("\n"); + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), 0); + prop_assert!(result.textblocks.len() >= 1); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fenced_content_preserved_verbatim(body in "[^`\\n]{1,80}") { + // §5.2.2: fence body is verbatim, no joining, no escaping. + let input = format!("/cmd ```\n{}\n```", body); + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), 1); + prop_assert_eq!(&result.commands.first().unwrap().arguments.payload, &body); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn unclosed_fence_always_produces_warning(body in "[^`]{0,80}") { + // §5.2.5: any fence reaching EOF without closer produces exactly one warning. + let input = format!("/cmd ```\n{}", body); + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), 1); + prop_assert_eq!(result.warnings.len(), 1); + prop_assert_eq!(&result.warnings.first().unwrap().wtype, "unclosed-fence"); + } +} diff --git a/parser/src/application/text_collect.rs b/parser/src/application/text_collect.rs index 21a212b..38856a5 100644 --- a/parser/src/application/text_collect.rs +++ b/parser/src/application/text_collect.rs @@ -1,6 +1,6 @@ use crate::domain::{LineRange, TextBlock}; -/// In-progress accumulation of contiguous non-command lines. +/// Accumulated state for a text block being built line by line. #[derive(Debug, Clone)] pub struct PendingText { pub start_line: usize, @@ -8,6 +8,7 @@ pub struct PendingText { pub lines: Vec, } +/// Start a new pending text block at the given physical line index. pub fn start_text(line_index: usize, line: &str) -> PendingText { PendingText { start_line: line_index, @@ -16,15 +17,22 @@ pub fn start_text(line_index: usize, line: &str) -> PendingText { } } +/// Append one more physical line to an in-progress text block. pub fn append_text(mut text: PendingText, line_index: usize, line: &str) -> PendingText { text.end_line = line_index; text.lines.push(line.to_string()); text } -pub fn finalize_text(text: PendingText) -> TextBlock { +/// Finalize a pending text block, assigning it the given sequential id. +/// +/// The caller supplies a zero-based counter; this function formats it as +/// `text-{id}` per §7. The caller is responsible for incrementing the counter +/// after each call so that IDs are unique and sequential within an envelope. +pub fn finalize_text(text: PendingText, id: usize) -> TextBlock { let content = text.lines.join("\n"); TextBlock { + id: format!("text-{id}"), range: LineRange { start_line: text.start_line, end_line: text.end_line, @@ -36,135 +44,191 @@ pub fn finalize_text(text: PendingText) -> TextBlock { #[cfg(test)] mod tests { use super::*; + use proptest::prelude::*; + + // --- start_text --- #[test] - fn single_line_text_block() { - let pending = start_text(0, "hello"); - let block = finalize_text(pending); - assert_eq!(block.content, "hello"); - assert_eq!(block.range.start_line, 0); - assert_eq!(block.range.end_line, 0); + fn start_text_single_line_range() { + // §6: "Text blocks use physical line numbers for their range." + // A freshly started block spans exactly one physical line. + let pt = start_text(3, "hello"); + assert_eq!(pt.start_line, 3); + assert_eq!(pt.end_line, 3); + assert_eq!(pt.lines, vec!["hello"]); } #[test] - fn multi_line_text_block() { - let text = start_text(2, "first"); - let text = append_text(text, 3, "second"); - let text = append_text(text, 4, "third"); - let block = finalize_text(text); - assert_eq!(block.content, "first\nsecond\nthird"); - assert_eq!(block.range.start_line, 2); - assert_eq!(block.range.end_line, 4); + fn start_text_preserves_content_verbatim() { + // §6: "Text block content preserves the original lines." Leading/trailing + // whitespace and punctuation are not modified. + let pt = start_text(0, " indented line! "); + assert_eq!(pt.lines[0], " indented line! "); } + // --- append_text --- + #[test] - fn empty_line_preserved_in_content() { - let text = start_text(0, "before"); - let text = append_text(text, 1, ""); - let text = append_text(text, 2, "after"); - let block = finalize_text(text); - assert_eq!(block.content, "before\n\nafter"); + fn append_text_updates_end_line_only() { + // §6: "Text blocks use physical line numbers for their range." Appending + // advances end_line but must never change start_line. + let pt = start_text(2, "first"); + let pt = append_text(pt, 3, "second"); + assert_eq!(pt.start_line, 2); + assert_eq!(pt.end_line, 3); } #[test] - fn whitespace_only_line_preserved() { - let pending = start_text(5, " "); - let block = finalize_text(pending); - assert_eq!(block.content, " "); - assert_eq!(block.range.start_line, 5); + fn append_text_accumulates_lines_in_order() { + // §6: "Consecutive non-command lines form a single text block." Lines must + // appear in document order. + let pt = start_text(0, "a"); + let pt = append_text(pt, 1, "b"); + let pt = append_text(pt, 2, "c"); + assert_eq!(pt.lines, vec!["a", "b", "c"]); } #[test] - fn append_updates_end_line_only() { - let text = start_text(3, "first"); - let text = append_text(text, 4, "second"); - assert_eq!(text.start_line, 3); - assert_eq!(text.end_line, 4); + fn append_text_preserves_blank_lines() { + // §6: "Blank lines that are part of a text region are included in the + // text block content." + let pt = start_text(0, "before"); + let pt = append_text(pt, 1, ""); + let pt = append_text(pt, 2, "after"); + assert_eq!(pt.lines, vec!["before", "", "after"]); } - // --- Property tests --- + #[test] + fn append_text_preserves_whitespace_only_lines() { + // §6: blank lines (including whitespace-only lines) are included verbatim. + let pt = start_text(0, "text"); + let pt = append_text(pt, 1, " "); + assert_eq!(pt.lines[1], " "); + } - use proptest::prelude::*; + // --- finalize_text --- - fn valid_command_name() -> impl Strategy { - "[a-z][a-z0-9\\-]{0,20}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + #[test] + fn finalize_text_id_format() { + // §7: "Text blocks are assigned IDs independently in the same manner: + // text-0, text-1, text-2, and so on." + let block = finalize_text(start_text(0, "x"), 0); + assert_eq!(block.id, "text-0"); + + let block = finalize_text(start_text(0, "x"), 7); + assert_eq!(block.id, "text-7"); } - fn arbitrary_line() -> impl Strategy { - prop_oneof![ - // plain text - "[a-zA-Z0-9 !.,]{0,80}", - // command-like - valid_command_name().prop_flat_map(|name| { - "[a-zA-Z0-9 ]{0,40}".prop_map(move |args| format!("/{name} {args}")) - }), - // leading whitespace + command - valid_command_name().prop_flat_map(|name| { - (1usize..5, "[a-zA-Z0-9 ]{0,40}").prop_map(move |(spaces, args)| { - format!("{}/{} {}", " ".repeat(spaces), name, args) - }) - }), - ] + #[test] + fn finalize_text_content_lines_joined_with_newline() { + // §6: "Text block content preserves the original lines joined with `\n` + // separators." + let pt = start_text(0, "line one"); + let pt = append_text(pt, 1, "line two"); + let pt = append_text(pt, 2, "line three"); + let block = finalize_text(pt, 0); + assert_eq!(block.content, "line one\nline two\nline three"); } - use crate::{ - application::line_classify::{LineKind, classify_line}, - domain::ArgumentMode, - }; - proptest! { - #[test] - #[cfg_attr(feature = "tdd", ignore)] + #[test] + fn finalize_text_single_line_no_trailing_newline() { + // §6 (implied): a single-line block has no separator — content equals + // the line itself with no added newline. + let block = finalize_text(start_text(0, "hello"), 0); + assert_eq!(block.content, "hello"); + } + #[test] + fn finalize_text_range_covers_physical_lines() { + // §6: "Text blocks use physical line numbers for their range." The range + // must span from the first to the last physical line consumed. + let pt = start_text(4, "a"); + let pt = append_text(pt, 5, "b"); + let block = finalize_text(pt, 1); + assert_eq!(block.range.start_line, 4); + assert_eq!(block.range.end_line, 5); + } - fn classify_never_panics(line in "[\\x00-\\x7F]{0,200}") { - let _ = classify_line(&line); - } + #[test] + fn finalize_text_blank_line_in_content() { + // §6: "Blank lines that are part of a text region are included in the + // text block content." + let pt = start_text(0, "before"); + let pt = append_text(pt, 1, ""); + let pt = append_text(pt, 2, "after"); + let block = finalize_text(pt, 0); + assert_eq!(block.content, "before\n\nafter"); + } - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn valid_name_always_produces_command(name in valid_command_name(), args in "[a-z0-9 ]{0,40}") { - let input = format!("/{name} {args}"); - match classify_line(&input) { - LineKind::Command(h) => prop_assert_eq!(h.name, name), - LineKind::Text => panic!("expected Command for input: {input}"), - } - } + #[test] + fn finalize_text_incremental_emission_possible() { + // §9.2: "A conforming implementation must support finalizing each … text + // block as soon as its last physical line has been consumed." finalize_text + // requires only the accumulated PendingText — no global state. + let block = finalize_text(start_text(10, "standalone"), 3); + assert_eq!(block.id, "text-3"); + assert_eq!(block.range.start_line, 10); + assert_eq!(block.content, "standalone"); + } + + // --- Property tests --- + proptest! { #[test] #[cfg_attr(feature = "tdd", ignore)] - fn text_without_slash_is_never_command(line in "[a-zA-Z0-9 !.,]{1,80}") { - prop_assert!(matches!(classify_line(&line), LineKind::Text)); + fn id_matches_text_n_pattern(id in 0usize..1000) { + // §7: IDs are "text-0, text-1, text-2, and so on" — the pattern is + // always `text-` followed by the decimal zero-based index. + let block = finalize_text(start_text(0, "x"), id); + prop_assert_eq!(block.id, format!("text-{id}")); } #[test] #[cfg_attr(feature = "tdd", ignore)] - fn raw_field_preserves_original_input(line in arbitrary_line()) { - if let LineKind::Command(h) = classify_line(&line) { - prop_assert_eq!(h.raw, line); - } + fn content_equals_lines_joined_with_newline( + lines in prop::collection::vec("[a-zA-Z0-9 !.,]{0,60}", 1..20) + ) { + // §6: "Text block content preserves the original lines joined with + // `\n` separators." + let expected = lines.join("\n"); + let pt = lines.iter().enumerate().fold( + start_text(0, &lines[0]), + |acc, (i, line)| if i == 0 { acc } else { append_text(acc, i, line) }, + ); + let block = finalize_text(pt, 0); + prop_assert_eq!(block.content, expected); } #[test] #[cfg_attr(feature = "tdd", ignore)] - fn fence_mode_iff_backticks_present(name in valid_command_name(), lang in "[a-z]{0,10}") { - let input = format!("/{name} ```{lang}"); - match classify_line(&input) { - LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Fence), - _ => panic!("expected Command"), + fn range_covers_exactly_the_lines_provided( + start in 0usize..100, + extra in 0usize..20 + ) { + // §6: "Text blocks use physical line numbers for their range." The range + // must be [start, start + extra] with no gaps or extensions. + let mut pt = start_text(start, "first"); + for i in 1..=extra { + pt = append_text(pt, start + i, "line"); } + let block = finalize_text(pt, 0); + prop_assert_eq!(block.range.start_line, start); + prop_assert_eq!(block.range.end_line, start + extra); } #[test] #[cfg_attr(feature = "tdd", ignore)] - fn continuation_mode_iff_trailing_backslash( - name in valid_command_name(), - args in "[a-z0-9 ]{1,30}" + fn append_never_changes_start_line( + start in 0usize..100, + lines in prop::collection::vec("[a-zA-Z]{1,20}", 1..10) ) { - let input = format!("/{name} {args} \\"); - match classify_line(&input) { - LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Continuation), - _ => panic!("expected Command"), + // §6: start_line is fixed at the first physical line of the block; + // subsequent appends must not modify it. + let mut pt = start_text(start, "first"); + for (i, line) in lines.iter().enumerate() { + pt = append_text(pt, start + i + 1, line); } + prop_assert_eq!(pt.start_line, start); } } } diff --git a/parser/src/lib.rs b/parser/src/lib.rs index 0001c4b..c7e8ab0 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -1,5 +1,5 @@ mod application; -pub mod domain; +mod domain; -// Public API re-export -pub use application::parse_document; +// Public API re-exports — restored as refactor completes +// pub use application::parse_document;