From eaff79961d3e6bc23cd23e0689cb8556d5d19a87 Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Wed, 18 Mar 2026 10:29:09 -0600 Subject: [PATCH 01/13] ci: linter & fmt config to follow tom's code patterns --- Cargo.toml | 87 +++++++++++++++++++ ...lt-output-with-pretty-json-array-option.md | 6 +- dprint.json | 22 ++--- 3 files changed, 94 insertions(+), 21 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6c249e7..0b82c27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,3 +14,90 @@ repository = "https://github.com/tomdavidson/slash-parser" serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "2" + +[workspace.lints.rust] +# Safety baseline: no unsafe unless explicitly justified +unsafe_code = "forbid" +# Debug is cheap observability; helpful at boundaries +missing_debug_implementations = "warn" +# Keep APIs intentionally small; helps encapsulation +unreachable_pub = "warn" + +[workspace.lints.clippy] +# ── Baseline coverage (from lang-rust.md) ─────────────────────── +cargo = { level = "warn", priority = -1 } +# Broad coverage; per-lint overrides below win +pedantic = { level = "warn", priority = -1 } +# Feature modules + re-exports make this noisy +module_name_repetitions = "allow" +# You use #[must_use] intentionally, not everywhere +must_use_candidate = "allow" +# Reduce doc-noise; rely on clear types + thiserror +missing_errors_doc = "allow" +# You deny panics anyway (below) +missing_panics_doc = "allow" + +# ── String & borrowing conventions (lang-rust.md) ─────────────── +# Prefer &str over &String; &[T] over &Vec Date: Wed, 18 Mar 2026 10:34:11 -0600 Subject: [PATCH 02/13] add application & domain layers --- parser/src/application/command_accumulate.rs | 249 ++++++++++ parser/src/application/command_finalize.rs | 268 +++++++++++ parser/src/application/document_parse.rs | 452 +++++++++++++++++++ parser/src/application/line_classify.rs | 281 ++++++++++++ parser/src/application/mod.rs | 10 + parser/src/application/tests/mod.rs | 1 + parser/src/application/tests/proptest.rs | 62 +++ parser/src/application/text_collect.rs | 170 +++++++ parser/src/lib.rs | 5 + parser/tests/mod.rs | 21 + 10 files changed, 1519 insertions(+) create mode 100644 parser/src/application/command_accumulate.rs create mode 100644 parser/src/application/command_finalize.rs create mode 100644 parser/src/application/document_parse.rs create mode 100644 parser/src/application/line_classify.rs create mode 100644 parser/src/application/mod.rs create mode 100644 parser/src/application/tests/mod.rs create mode 100644 parser/src/application/tests/proptest.rs create mode 100644 parser/src/application/text_collect.rs create mode 100644 parser/src/lib.rs create mode 100644 parser/tests/mod.rs diff --git a/parser/src/application/command_accumulate.rs b/parser/src/application/command_accumulate.rs new file mode 100644 index 0000000..d89f76f --- /dev/null +++ b/parser/src/application/command_accumulate.rs @@ -0,0 +1,249 @@ +use super::line_classify::{CommandHeader, LineKind, classify_line}; +use crate::domain::ArgumentMode; + +/// Result of offering a line to an in-progress command. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AcceptResult { + Consumed, + Completed, + Rejected, +} + +/// In-progress command being assembled from consecutive lines. +#[derive(Debug, Clone)] +pub struct PendingCommand { + pub name: String, + pub raw_header: String, + pub header_text: String, + pub mode: ArgumentMode, + pub fence_lang: Option, + pub start_line: usize, + pub end_line: usize, + pub payload_lines: Vec, + pub is_open: bool, +} + +pub fn start_command(header: CommandHeader, line_index: usize) -> PendingCommand { + let initial_lines = if header.header_text.is_empty() { + vec![] + } else { + vec![header.header_text.clone()] + }; + + let (payload_lines, is_open) = match &header.mode { + ArgumentMode::SingleLine => (initial_lines, false), + ArgumentMode::Continuation => (initial_lines, true), + ArgumentMode::Fence => (vec![], true), + }; + + PendingCommand { + name: header.name, + raw_header: header.raw, + header_text: header.header_text, + mode: header.mode, + fence_lang: header.fence_lang, + start_line: line_index, + end_line: line_index, + payload_lines, + is_open, + } +} + +pub fn accept_line( + cmd: PendingCommand, + line_index: usize, + line: &str, +) -> (PendingCommand, AcceptResult) { + if !cmd.is_open { + return (cmd, AcceptResult::Rejected); + } + + match cmd.mode { + ArgumentMode::SingleLine => (cmd, AcceptResult::Rejected), + ArgumentMode::Continuation => accept_continuation(cmd, line_index, line), + ArgumentMode::Fence => accept_fence(cmd, line_index, line), + } +} + +fn accept_continuation( + mut cmd: PendingCommand, + line_index: usize, + line: &str, +) -> (PendingCommand, AcceptResult) { + if line.trim().is_empty() { + cmd.is_open = false; + return (cmd, AcceptResult::Rejected); + } + + if matches!(classify_line(line), LineKind::Command(_)) { + cmd.is_open = false; + return (cmd, AcceptResult::Rejected); + } + + if line.trim_end().ends_with('\\') { + let content = line.trim_end().trim_end_matches('\\').trim_end(); + cmd.payload_lines.push(content.to_string()); + cmd.end_line = line_index; + (cmd, AcceptResult::Consumed) + } else { + cmd.payload_lines.push(line.to_string()); + cmd.end_line = line_index; + cmd.is_open = false; + (cmd, AcceptResult::Completed) + } +} + +fn accept_fence( + mut cmd: PendingCommand, + line_index: usize, + line: &str, +) -> (PendingCommand, AcceptResult) { + if line.trim_start().starts_with("```") { + cmd.end_line = line_index; + cmd.is_open = false; + (cmd, AcceptResult::Completed) + } else { + cmd.end_line = line_index; + cmd.payload_lines.push(line.to_string()); + (cmd, AcceptResult::Consumed) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn header_from(line: &str) -> CommandHeader { + match classify_line(line) { + LineKind::Command(h) => h, + _ => panic!("expected command header"), + } + } + + #[test] + fn single_line_is_immediately_closed() { + let cmd = start_command(header_from("/help"), 0); + assert!(!cmd.is_open); + } + + #[test] + fn fence_stays_open_until_closing_ticks() { + let cmd = start_command(header_from("/code ```rust"), 0); + assert!(cmd.is_open); + + let (cmd, res) = accept_line(cmd, 1, "fn main() {}"); + assert_eq!(res, AcceptResult::Consumed); + assert!(cmd.is_open); + + let (cmd, res) = accept_line(cmd, 2, "```"); + assert_eq!(res, AcceptResult::Completed); + assert!(!cmd.is_open); + assert_eq!(cmd.payload_lines, vec!["fn main() {}"]); + } + + #[test] + fn continuation_ends_on_line_without_backslash() { + let cmd = start_command(header_from("/cmd first \\"), 0); + assert!(cmd.is_open); + + let (cmd, res) = accept_line(cmd, 1, "second \\"); + assert_eq!(res, AcceptResult::Consumed); + + let (cmd, res) = accept_line(cmd, 2, "last line"); + assert_eq!(res, AcceptResult::Completed); + assert!(!cmd.is_open); + } + + #[test] + fn continuation_rejects_blank_line() { + let cmd = start_command(header_from("/cmd start \\"), 0); + let (cmd, res) = accept_line(cmd, 1, ""); + assert_eq!(res, AcceptResult::Rejected); + assert!(!cmd.is_open); + } + + #[test] + fn continuation_rejects_new_command() { + let cmd = start_command(header_from("/cmd start \\"), 0); + let (cmd, res) = accept_line(cmd, 1, "/other"); + assert_eq!(res, AcceptResult::Rejected); + assert!(!cmd.is_open); + } + + #[test] + fn continuation_two_lines_produces_continuation_mode() { + let header = header_from("/mcp call_tool read_file \\"); + let cmd = start_command(header, 0); + + assert_eq!(cmd.mode, ArgumentMode::Continuation); + assert!(cmd.is_open); + } + + #[test] + fn continuation_header_strips_trailing_marker() { + let header = header_from("/mcp call_tool read_file \\"); + let cmd = start_command(header, 0); + + assert_eq!(cmd.header_text, "call_tool read_file"); + assert_eq!(cmd.payload_lines, vec!["call_tool read_file"]); + } + + #[test] + fn continuation_payload_preserves_newlines() { + let cmd = start_command(header_from("/mcp call_tool read_file \\"), 0); + + let line1 = r#"{"path": "src/index.ts"}"#; + let (cmd, res) = accept_line(cmd, 1, line1); + assert_eq!(res, AcceptResult::Completed); + assert!(!cmd.is_open); + + assert_eq!( + cmd.payload_lines, + vec!["call_tool read_file".to_string(), line1.to_string()] + ); + + assert_eq!(cmd.start_line, 0); + assert_eq!(cmd.end_line, 1); + } + + #[test] + fn continuation_three_lines() { + let cmd = start_command(header_from("/cmd first \\"), 0); + + let (cmd, res) = accept_line(cmd, 1, "second \\"); + assert_eq!(res, AcceptResult::Consumed); + let (cmd, res) = accept_line(cmd, 2, "third"); + assert_eq!(res, AcceptResult::Completed); + assert!(!cmd.is_open); + + assert_eq!( + cmd.payload_lines, + vec![ + "first".to_string(), + "second".to_string(), + "third".to_string() + ] + ); + } + + #[test] + fn continuation_range_spans_all_lines() { + let cmd = start_command(header_from("/cmd first \\"), 0); + + let (cmd, _) = accept_line(cmd, 1, "second \\"); + let (cmd, _) = accept_line(cmd, 2, "third"); + + assert_eq!(cmd.start_line, 0); + assert_eq!(cmd.end_line, 2); + } + + #[test] + fn slash_at_end_without_space_is_not_continuation() { + let header = header_from("/path /var/log/"); + let cmd = start_command(header, 0); + + assert_eq!(cmd.mode, ArgumentMode::SingleLine); + assert!(!cmd.is_open); + assert_eq!(cmd.payload_lines, vec!["/var/log/"]); + } +} diff --git a/parser/src/application/command_finalize.rs b/parser/src/application/command_finalize.rs new file mode 100644 index 0000000..7c3ca49 --- /dev/null +++ b/parser/src/application/command_finalize.rs @@ -0,0 +1,268 @@ +use super::command_accumulate::PendingCommand; +use crate::domain::{ArgumentMode, Command, CommandArguments, LineRange, ParseWarning}; + +#[derive(Debug)] +pub struct FinalizedCommand { + pub command: Command, + pub warnings: Vec, +} + +pub fn finalize_command(pending: PendingCommand) -> FinalizedCommand { + let mut warnings = Vec::new(); + + match pending.mode { + ArgumentMode::Fence if pending.is_open => { + warnings.push(ParseWarning::UnclosedFence { + start_line: pending.start_line, + }); + } + ArgumentMode::Continuation if pending.is_open => { + warnings.push(ParseWarning::UnclosedContinuation { + start_line: pending.start_line, + }); + } + _ => {} + } + + let payload = pending.payload_lines.join("\n"); + + let command = Command { + name: pending.name, + raw: pending.raw_header, + range: LineRange { + start_line: pending.start_line, + end_line: pending.end_line, + }, + arguments: CommandArguments { + header: pending.header_text, + mode: pending.mode, + fence_lang: pending.fence_lang, + payload, + }, + }; + + FinalizedCommand { command, warnings } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + application::{ + command_accumulate::{accept_line, start_command}, + line_classify::{CommandHeader, LineKind, classify_line}, + }, + domain::ArgumentMode, + }; + + fn header_from(line: &str) -> CommandHeader { + match classify_line(line) { + LineKind::Command(h) => h, + _ => panic!("expected command header"), + } + } + + // --- Warning tests --- + + #[test] + fn unclosed_fence_at_eof_warns() { + let cmd = start_command(header_from("/cmd ```rust"), 0); + let (cmd, _) = accept_line(cmd, 1, "line1"); + let (cmd, _) = accept_line(cmd, 2, "line2"); + + let result = finalize_command(cmd); + + assert_eq!(result.warnings.len(), 1); + assert!(matches!( + result.warnings[0], + ParseWarning::UnclosedFence { start_line: 0 } + )); + } + + #[test] + fn unclosed_continuation_at_eof_warns() { + let cmd = start_command(header_from("/cmd first \\"), 0); + + let result = finalize_command(cmd); + + assert_eq!(result.warnings.len(), 1); + assert!(matches!( + result.warnings[0], + ParseWarning::UnclosedContinuation { start_line: 0 } + )); + } + + #[test] + fn closed_fence_has_no_warning() { + let cmd = start_command(header_from("/cmd ```"), 0); + let (cmd, _) = accept_line(cmd, 1, "content"); + let (cmd, _) = accept_line(cmd, 2, "```"); + + let result = finalize_command(cmd); + + assert!(result.warnings.is_empty()); + } + + #[test] + fn closed_continuation_has_no_warning() { + let cmd = start_command(header_from("/cmd first \\"), 0); + let (cmd, _) = accept_line(cmd, 1, "last line"); + + let result = finalize_command(cmd); + + assert!(result.warnings.is_empty()); + } + + #[test] + fn single_line_has_no_warning() { + let cmd = start_command(header_from("/help"), 0); + + let result = finalize_command(cmd); + + assert!(result.warnings.is_empty()); + } + + // --- Finalized command structure --- + + #[test] + fn finalized_command_has_correct_name() { + let cmd = start_command(header_from("/deploy production"), 0); + + let result = finalize_command(cmd); + + assert_eq!(result.command.name, "deploy"); + } + + #[test] + fn finalized_command_has_correct_range() { + let cmd = start_command(header_from("/cmd ```"), 0); + let (cmd, _) = accept_line(cmd, 1, "body"); + let (cmd, _) = accept_line(cmd, 2, "```"); + + let result = finalize_command(cmd); + + assert_eq!(result.command.range.start_line, 0); + assert_eq!(result.command.range.end_line, 2); + } + + #[test] + fn finalized_fence_payload_is_joined_lines() { + let cmd = start_command(header_from("/cmd ```"), 0); + let (cmd, _) = accept_line(cmd, 1, "line one"); + let (cmd, _) = accept_line(cmd, 2, "line two"); + let (cmd, _) = accept_line(cmd, 3, "```"); + + let result = finalize_command(cmd); + + assert_eq!(result.command.arguments.payload, "line one\nline two"); + assert_eq!(result.command.arguments.mode, ArgumentMode::Fence); + } + + #[test] + fn finalized_continuation_payload_is_joined_lines() { + let cmd = start_command(header_from("/cmd first \\"), 0); + let (cmd, _) = accept_line(cmd, 1, "second \\"); + let (cmd, _) = accept_line(cmd, 2, "third"); + + let result = finalize_command(cmd); + + assert_eq!(result.command.arguments.payload, "first\nsecond\nthird"); + assert_eq!(result.command.arguments.mode, ArgumentMode::Continuation); + } + + #[test] + fn finalized_single_line_payload_matches_header() { + let cmd = start_command(header_from("/hello world"), 0); + + let result = finalize_command(cmd); + + assert_eq!(result.command.arguments.payload, "world"); + assert_eq!(result.command.arguments.mode, ArgumentMode::SingleLine); + } + + #[test] + fn finalized_fence_captures_language() { + let cmd = start_command(header_from("/code ```rust"), 0); + let (cmd, _) = accept_line(cmd, 1, "fn main() {}"); + let (cmd, _) = accept_line(cmd, 2, "```"); + + let result = finalize_command(cmd); + + assert_eq!( + result.command.arguments.fence_lang, + Some("rust".to_string()) + ); + } + + use proptest::prelude::*; + + fn valid_command_name() -> impl Strategy { + "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + } + + proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn finalized_name_matches_pending_name(name in valid_command_name()) { + let input = format!("/{name} arg"); + let cmd = start_command(header_from(&input), 0); + let result = finalize_command(cmd); + prop_assert_eq!(result.command.name, name); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn closed_commands_never_warn( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,30}", 1..8) + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0); + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); + + let result = finalize_command(cmd); + prop_assert!(result.warnings.is_empty()); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn unclosed_fence_always_warns( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 1..5) + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0); + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + + let result = finalize_command(cmd); + prop_assert_eq!(result.warnings.len(), 1); + let is_unclosed_fence = matches!(result.warnings[0], ParseWarning::UnclosedFence { .. }); + prop_assert!(is_unclosed_fence); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn payload_is_payload_lines_joined( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 1..8) + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0); + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); + + let result = finalize_command(cmd); + prop_assert_eq!(result.command.arguments.payload, body_lines.join("\n")); + } + } +} diff --git a/parser/src/application/document_parse.rs b/parser/src/application/document_parse.rs new file mode 100644 index 0000000..a9da645 --- /dev/null +++ b/parser/src/application/document_parse.rs @@ -0,0 +1,452 @@ +use super::{ + command_accumulate::{AcceptResult, PendingCommand, accept_line, start_command}, + command_finalize::finalize_command, + line_classify::{LineKind, classify_line}, + text_collect::{PendingText, append_text, finalize_text, start_text}, +}; +use crate::domain::{Command, ParseResult, ParseWarning, ParserContext, SPEC_VERSION, TextBlock}; + +struct ParseState { + current_cmd: Option, + current_text: Option, + commands: Vec, + text_blocks: Vec, + warnings: Vec, +} + +pub fn parse_document(input: &str) -> ParseResult { + let state = input + .lines() + .enumerate() + .fold(empty_state(), |state, (i, line)| step(state, i, line)); + + finalize(state) +} + +fn empty_state() -> ParseState { + ParseState { + current_cmd: None, + current_text: None, + commands: Vec::new(), + text_blocks: Vec::new(), + warnings: Vec::new(), + } +} + +fn step(mut state: ParseState, line_index: usize, line: &str) -> ParseState { + if try_feed_command(&mut state, line_index, line) { + return state; + } + + classify_fresh_line(&mut state, line_index, line); + state +} + +fn try_feed_command(state: &mut ParseState, line_index: usize, line: &str) -> bool { + let Some(cmd) = state.current_cmd.take() else { + return false; + }; + + if !cmd.is_open { + absorb_command(state, cmd); + return false; + } + + let (updated_cmd, result) = accept_line(cmd, line_index, line); + match result { + AcceptResult::Consumed => { + state.current_cmd = Some(updated_cmd); + true + } + AcceptResult::Completed => { + absorb_command(state, updated_cmd); + true + } + AcceptResult::Rejected => { + absorb_command(state, updated_cmd); + false + } + } +} + +fn classify_fresh_line(state: &mut ParseState, line_index: usize, line: &str) { + match classify_line(line) { + LineKind::Command(header) => { + flush_text(state); + state.current_cmd = Some(start_command(header, line_index)); + } + LineKind::Text => { + accumulate_text(state, line_index, line); + } + } +} + +fn absorb_command(state: &mut ParseState, cmd: PendingCommand) { + let finalized = finalize_command(cmd); + state.commands.push(finalized.command); + state.warnings.extend(finalized.warnings); +} + +fn flush_text(state: &mut ParseState) { + if let Some(text) = state.current_text.take() { + state.text_blocks.push(finalize_text(text)); + } +} + +fn accumulate_text(state: &mut ParseState, line_index: usize, line: &str) { + state.current_text = match state.current_text.take() { + Some(text) => Some(append_text(text, line_index, line)), + None => Some(start_text(line_index, line)), + }; +} + +fn finalize(mut state: ParseState) -> ParseResult { + if let Some(cmd) = state.current_cmd.take() { + absorb_command(&mut state, cmd); + } + + flush_text(&mut state); + + ParseResult { + commands: state.commands, + text_blocks: state.text_blocks, + warnings: state.warnings, + version: SPEC_VERSION.to_owned(), + context: ParserContext::default(), + } +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use crate::{application::document_parse::parse_document, domain::ArgumentMode}; + + fn parse(input: &str) -> crate::domain::ParseResult { + parse_document(input) + } + + fn assert_single_command(input: &str) -> crate::domain::Command { + let result = parse(input); + assert_eq!(result.commands.len(), 1, "expected exactly 1 command"); + result.commands.into_iter().next().unwrap() + } + + #[test] + fn single_line_simple_command_parses_name() { + let cmd = assert_single_command("/hello world"); + assert_eq!(cmd.name, "hello"); + } + + #[test] + fn trailing_newline_does_not_create_empty_text_block() { + let result = parse("/cmd arg\n"); + assert_eq!(result.text_blocks.len(), 0); + } + + #[test] + fn single_line_mode_threads_through_from_classify() { + let cmd = assert_single_command("/hello world"); + assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); + } + + #[test] + fn single_line_payload_threads_through_from_classify() { + let cmd = assert_single_command("/hello world"); + assert_eq!(cmd.arguments.payload, "world"); + } + + #[test] + fn unclosed_continuation_produces_warning() { + let input = "/cmd start \\"; + let result = parse(input); + assert_eq!(result.commands.len(), 1); + assert_eq!(result.warnings.len(), 1); + } + + #[test] + fn leading_whitespace_command_threads_through() { + let cmd = assert_single_command(" /hello world"); + assert_eq!(cmd.name, "hello"); + } + + #[test] + fn single_line_command_with_no_args_has_empty_header() { + let cmd = assert_single_command("/ping"); + assert_eq!(cmd.arguments.header, ""); + assert_eq!(cmd.arguments.payload, ""); + } + + #[test] + fn single_line_command_with_complex_args() { + let cmd = assert_single_command(r#"/mcp call_tool read_file {"path": "src/index.ts"}"#); + assert_eq!(cmd.name, "mcp"); + assert_eq!( + cmd.arguments.header, + r#"call_tool read_file {"path": "src/index.ts"}"# + ); + } + + #[test] + fn single_line_command_range_is_same_line() { + let cmd = assert_single_command("/test"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 0); + } + + #[test] + fn parses_text_and_single_command() { + let input = "intro\n/cmd arg\noutro"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + assert_eq!(result.text_blocks.len(), 2); + assert_eq!(result.commands[0].name, "cmd"); + assert_eq!(result.commands[0].arguments.mode, ArgumentMode::SingleLine); + } + + #[test] + fn command_on_line_two_has_correct_range() { + let input = "text\nmore text\n/cmd arg"; + let result = parse(input); + assert_eq!(result.commands[0].range.start_line, 2); + assert_eq!(result.commands[0].range.end_line, 2); + } + + #[test] + fn two_single_line_commands_both_parse() { + let result = parse("/first a\n/second b"); + assert_eq!(result.commands.len(), 2); + assert_eq!(result.commands[0].name, "first"); + assert_eq!(result.commands[1].name, "second"); + } + + #[test] + fn empty_input_produces_empty_result() { + let result = parse(""); + assert_eq!(result.commands.len(), 0); + assert_eq!(result.text_blocks.len(), 0); + assert_eq!(result.warnings.len(), 0); + } + + #[test] + fn text_only_produces_one_text_block() { + let result = parse("just some text"); + assert_eq!(result.commands.len(), 0); + assert_eq!(result.text_blocks.len(), 1); + } + + #[test] + fn adjacent_text_lines_merge_into_one_block() { + let result = parse("line one\nline two\nline three"); + assert_eq!(result.text_blocks.len(), 1); + assert_eq!(result.text_blocks[0].range.start_line, 0); + assert_eq!(result.text_blocks[0].range.end_line, 2); + } + + #[test] + fn text_block_before_command_has_correct_content() { + let result = parse("intro line\n/cmd arg"); + assert_eq!(result.text_blocks.len(), 1); + assert_eq!(result.text_blocks[0].content, "intro line"); + } + + #[test] + fn text_block_after_command_has_correct_range() { + let result = parse("/cmd arg\noutro line"); + assert_eq!(result.text_blocks.len(), 1); + assert_eq!(result.text_blocks[0].range.start_line, 1); + } + + #[test] + fn continuation_command_parses_through_document() { + let input = "/cmd first \\\nsecond \\\nthird"; + let cmd = assert_single_command(input); + assert_eq!(cmd.arguments.mode, ArgumentMode::Continuation); + assert_eq!(cmd.arguments.payload, "first\nsecond\nthird"); + } + + #[test] + fn fenced_command_parses_through_document() { + let input = "/cmd ```\nline one\nline two\n```"; + let cmd = assert_single_command(input); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.payload, "line one\nline two"); + } + + #[test] + fn unclosed_fence_produces_warning() { + let input = "/cmd ```\nline one\nline two"; + let result = parse(input); + assert_eq!(result.commands.len(), 1); + assert_eq!(result.warnings.len(), 1); + } + + #[test] + fn fence_followed_by_new_command() { + let input = "/first ```\nbody\n```\n/second arg"; + let result = parse(input); + assert_eq!(result.commands.len(), 2); + assert_eq!(result.commands[0].name, "first"); + assert_eq!(result.commands[1].name, "second"); + } + + #[test] + fn continuation_followed_by_new_command() { + let input = "/first start \\\nend\n/second arg"; + let result = parse(input); + assert_eq!(result.commands.len(), 2); + assert_eq!(result.commands[0].name, "first"); + assert_eq!(result.commands[1].name, "second"); + } + + #[test] + fn version_is_set() { + let result = parse(""); + assert_eq!(result.version, crate::domain::SPEC_VERSION); + } + + // --- Property tests --- + + fn valid_command_name() -> impl Strategy { + "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + } + + fn command_line() -> impl Strategy { + (valid_command_name(), "[a-z0-9 ]{0,30}").prop_map(|(name, args)| format!("/{name} {args}")) + } + + fn text_line() -> impl Strategy { + "[a-zA-Z0-9 !.,]{1,60}".prop_filter("must not start with slash", |s| { + !s.trim_start().starts_with('/') + }) + } + + proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn never_panics_on_arbitrary_input(input in "[\\x00-\\x7F]{0,500}") { + let _ = parse_document(&input); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn command_count_matches_slash_lines( + commands in prop::collection::vec(command_line(), 1..10) + ) { + let input = commands.join("\n"); + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), commands.len()); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_only_input_produces_zero_commands( + lines in prop::collection::vec(text_line(), 1..20) + ) { + let input = lines.join("\n"); + let result = parse_document(&input); + prop_assert!(result.commands.is_empty()); + prop_assert!(!result.text_blocks.is_empty()); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn version_is_always_spec_version(input in "[\\x20-\\x7E]{0,200}") { + let result = parse_document(&input); + prop_assert_eq!(&result.version, crate::domain::SPEC_VERSION); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fenced_content_preserved_verbatim( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9 !]{1,50}", 1..5) + ) { + let body = body_lines.join("\n"); + let input = format!("/{name} ```\n{body}\n```"); + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), 1); + prop_assert_eq!(&result.commands[0].arguments.payload, &body); + prop_assert_eq!(&result.commands[0].arguments.mode, &ArgumentMode::Fence); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn interleaved_text_and_commands_both_captured( + cmd_count in 1usize..5 + ) { + let mut lines = Vec::new(); + for i in 0..cmd_count { + lines.push(format!("text line {i}")); + lines.push(format!("/cmd{i} arg")); + } + let input = lines.join("\n"); + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), cmd_count); + prop_assert_eq!(result.text_blocks.len(), cmd_count); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn unclosed_fence_always_produces_warning( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,30}", 1..5) + ) { + let body = body_lines.join("\n"); + let input = format!("/{name} ```\n{body}"); + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), 1); + prop_assert_eq!(result.warnings.len(), 1); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_block_content_matches_non_command_lines( + lines in prop::collection::vec(text_line(), 1..10) + ) { + let input = lines.join("\n"); + let result = parse_document(&input); + prop_assert_eq!(result.text_blocks.len(), 1); + prop_assert_eq!(&result.text_blocks[0].content, &input); + } + + fn continuation_payload_matches_joined_lines( + name in valid_command_name(), + middle_lines in prop::collection::vec("[a-z0-9]{1,20}", 0..5), + final_line in "[a-z0-9]{1,20}" + ) { + let mut input_lines = vec![format!("/{name} start \\")]; + let mut expected_parts = vec!["start".to_string()]; + for line in &middle_lines { + input_lines.push(format!("{line} \\")); + expected_parts.push(line.clone()); + } + input_lines.push(final_line.clone()); + expected_parts.push(final_line); + let input = input_lines.join("\n"); + + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), 1); + prop_assert_eq!(&result.commands[0].arguments.mode, &ArgumentMode::Continuation); + prop_assert_eq!(&result.commands[0].arguments.payload, &expected_parts.join("\n")); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn unclosed_continuation_always_produces_warning( + name in valid_command_name(), + middle_lines in prop::collection::vec("[a-z0-9]{1,20}", 0..3) + ) { + let mut input_lines = vec![format!("/{name} start \\")]; + for line in &middle_lines { + input_lines.push(format!("{line} \\")); + } + let input = input_lines.join("\n"); + + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), 1); + prop_assert_eq!(result.warnings.len(), 1); + } + + } +} diff --git a/parser/src/application/line_classify.rs b/parser/src/application/line_classify.rs new file mode 100644 index 0000000..c95be82 --- /dev/null +++ b/parser/src/application/line_classify.rs @@ -0,0 +1,281 @@ +use crate::domain::ArgumentMode; + +/// Raw classification of a single input line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LineKind { + Command(CommandHeader), + Text, +} + +/// Extracted header fields from a line that starts a slash command. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommandHeader { + pub raw: String, + pub name: String, + pub header_text: String, + pub mode: ArgumentMode, + pub fence_lang: Option, +} + +/// Classify a single line as either a command header or plain text. +pub fn classify_line(line: &str) -> LineKind { + match try_parse_command(line) { + Some(header) => LineKind::Command(header), + None => LineKind::Text, + } +} + +fn try_parse_command(line: &str) -> Option { + let trimmed = line.trim_start(); + if !trimmed.starts_with('/') { + return None; + } + + let without_slash = &trimmed[1..]; + + // Command name is the first contiguous non-whitespace run. + let mut parts = without_slash.splitn(2, char::is_whitespace); + let name_raw = parts.next().filter(|n| !n.is_empty())?; + + // Command must begin with a lowercase ASCII letter and may contain lowercase letters, digits, and hyphens + if !name_raw.starts_with(|c: char| c.is_ascii_lowercase()) { + return None; + } + + if !name_raw + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-' || c == '_') + { + return None; + } + let name = name_raw.to_string(); + let rest = parts.next().unwrap_or("").trim_start(); + + // Fence mode: rest starts with ``` + if let Some(stripped) = rest.strip_prefix("```") { + let after_ticks = stripped.trim(); + let lang = after_ticks + .split_whitespace() + .next() + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + + return Some(CommandHeader { + raw: line.to_string(), + name, + header_text: rest.to_string(), + mode: ArgumentMode::Fence, + fence_lang: lang, + }); + } + + // Continuation mode: rest ends with trailing backslash. + if rest.ends_with('\\') { + let header_text = rest.trim_end_matches('\\').trim_end().to_string(); + + return Some(CommandHeader { + raw: line.to_string(), + name, + header_text, + mode: ArgumentMode::Continuation, + fence_lang: None, + }); + } + + // Single-line mode (default). + Some(CommandHeader { + raw: line.to_string(), + name, + header_text: rest.to_string(), + mode: ArgumentMode::SingleLine, + fence_lang: None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plain_text_line() { + assert_eq!(classify_line("hello world"), LineKind::Text); + } + + #[test] + fn bare_slash_is_text() { + assert_eq!(classify_line("/ "), LineKind::Text); + } + + #[test] + fn simple_command() { + let result = classify_line("/cmd some args"); + match result { + LineKind::Command(h) => { + assert_eq!(h.name, "cmd"); + assert_eq!(h.header_text, "some args"); + assert_eq!(h.mode, ArgumentMode::SingleLine); + } + _ => panic!("expected Command"), + } + } + + #[test] + fn fence_command_with_lang() { + let result = classify_line("/code ```rust"); + match result { + LineKind::Command(h) => { + assert_eq!(h.name, "code"); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang.as_deref(), Some("rust")); + } + _ => panic!("expected Command"), + } + } + + #[test] + fn continuation_command() { + let result = classify_line("/cmd first part \\"); + match result { + LineKind::Command(h) => { + assert_eq!(h.name, "cmd"); + assert_eq!(h.header_text, "first part"); + assert_eq!(h.mode, ArgumentMode::Continuation); + } + _ => panic!("expected Command"), + } + } + + #[test] + fn command_with_no_args() { + let result = classify_line("/help"); + match result { + LineKind::Command(h) => { + assert_eq!(h.name, "help"); + assert_eq!(h.header_text, ""); + assert_eq!(h.mode, ArgumentMode::SingleLine); + } + _ => panic!("expected Command"), + } + } + #[test] + fn slash_alone_no_space_is_text() { + assert_eq!(classify_line("/"), LineKind::Text); + } + + #[test] + fn command_name_starting_with_uppercase_is_text() { + assert_eq!(classify_line("/Hello"), LineKind::Text); + } + + #[test] + fn command_name_starting_with_digit_is_text() { + assert_eq!(classify_line("/2fast"), LineKind::Text); + } + + #[test] + fn leading_spaces_still_detected_as_command() { + match classify_line(" /cmd arg") { + LineKind::Command(h) => assert_eq!(h.name, "cmd"), + _ => panic!("expected Command"), + } + } + + #[test] + fn fence_without_language() { + match classify_line("/cmd ```") { + LineKind::Command(h) => { + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, None); + } + _ => panic!("expected Command"), + } + } + + #[test] + fn raw_preserves_original_line() { + match classify_line(" /cmd arg") { + LineKind::Command(h) => assert_eq!(h.raw, " /cmd arg"), + _ => panic!("expected Command"), + } + } + + // --- Property tests --- + + use proptest::prelude::*; + + fn valid_command_name() -> impl Strategy { + "[a-z][a-z0-9\\-]{0,20}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + } + + fn arbitrary_line() -> impl Strategy { + prop_oneof![ + // plain text + "[a-zA-Z0-9 !.,]{0,80}", + // command-like + valid_command_name().prop_flat_map(|name| { + "[a-zA-Z0-9 ]{0,40}".prop_map(move |args| format!("/{name} {args}")) + }), + // leading whitespace + command + valid_command_name().prop_flat_map(|name| { + (1usize..5, "[a-zA-Z0-9 ]{0,40}").prop_map(move |(spaces, args)| { + format!("{}/{} {}", " ".repeat(spaces), name, args) + }) + }), + ] + } + + proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn classify_never_panics(line in "[\\x00-\\x7F]{0,200}") { + let _ = classify_line(&line); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn valid_name_always_produces_command(name in valid_command_name(), args in "[a-z0-9 ]{0,40}") { + let input = format!("/{name} {args}"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.name, name), + LineKind::Text => panic!("expected Command for input: {input}"), + } + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_without_slash_is_never_command(line in "[a-zA-Z0-9 !.,]{1,80}") { + prop_assert!(matches!(classify_line(&line), LineKind::Text)); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn raw_field_preserves_original_input(line in arbitrary_line()) { + if let LineKind::Command(h) = classify_line(&line) { + prop_assert_eq!(h.raw, line); + } + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fence_mode_iff_backticks_present(name in valid_command_name(), lang in "[a-z]{0,10}") { + let input = format!("/{name} ```{lang}"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Fence), + _ => panic!("expected Command"), + } + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn continuation_mode_iff_trailing_backslash( + name in valid_command_name(), + args in "[a-z0-9 ]{1,30}" + ) { + let input = format!("/{name} {args} \\"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Continuation), + _ => panic!("expected Command"), + } + } + } +} diff --git a/parser/src/application/mod.rs b/parser/src/application/mod.rs new file mode 100644 index 0000000..19c9b5b --- /dev/null +++ b/parser/src/application/mod.rs @@ -0,0 +1,10 @@ +mod command_accumulate; +mod command_finalize; +mod document_parse; +mod line_classify; +mod text_collect; + +pub use document_parse::parse_document; + +#[cfg(test)] +mod tests; diff --git a/parser/src/application/tests/mod.rs b/parser/src/application/tests/mod.rs new file mode 100644 index 0000000..cd2f278 --- /dev/null +++ b/parser/src/application/tests/mod.rs @@ -0,0 +1 @@ +mod proptest; diff --git a/parser/src/application/tests/proptest.rs b/parser/src/application/tests/proptest.rs new file mode 100644 index 0000000..c994daa --- /dev/null +++ b/parser/src/application/tests/proptest.rs @@ -0,0 +1,62 @@ +use proptest::prelude::*; + +use crate::{ + application::{ + command_accumulate::{accept_line, start_command}, + command_finalize::finalize_command, + line_classify::{LineKind, classify_line}, + text_collect::{append_text, finalize_text, start_text}, + }, + domain::ArgumentMode, +}; + +fn valid_command_name() -> impl Strategy { + "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) +} + +proptest! { + /// classify -> accumulate -> finalize roundtrip never panics. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn classify_accumulate_finalize_roundtrip( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,30}", 0..8) + ) { + let input = format!("/{name} ```"); + let header = match classify_line(&input) { + LineKind::Command(h) => h, + LineKind::Text => panic!("expected command"), + }; + + let cmd = start_command(header, 0); + + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); + + let finalized = finalize_command(cmd); + prop_assert_eq!(finalized.command.name, name); + prop_assert_eq!(finalized.command.arguments.mode, ArgumentMode::Fence); + prop_assert!(finalized.warnings.is_empty()); + } + + /// Text lines classified as Text always produce valid text blocks. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_lines_through_classify_and_collect( + lines in prop::collection::vec("[a-zA-Z0-9 !.,]{1,40}", 1..15) + ) { + for line in &lines { + prop_assert!(matches!(classify_line(line), LineKind::Text)); + } + + let pending = lines.iter().enumerate().skip(1).fold( + start_text(0, &lines[0]), + |text, (i, line)| append_text(text, i, line), + ); + let block = finalize_text(pending); + prop_assert_eq!(block.content, lines.join("\n")); + } +} diff --git a/parser/src/application/text_collect.rs b/parser/src/application/text_collect.rs new file mode 100644 index 0000000..21a212b --- /dev/null +++ b/parser/src/application/text_collect.rs @@ -0,0 +1,170 @@ +use crate::domain::{LineRange, TextBlock}; + +/// In-progress accumulation of contiguous non-command lines. +#[derive(Debug, Clone)] +pub struct PendingText { + pub start_line: usize, + pub end_line: usize, + pub lines: Vec, +} + +pub fn start_text(line_index: usize, line: &str) -> PendingText { + PendingText { + start_line: line_index, + end_line: line_index, + lines: vec![line.to_string()], + } +} + +pub fn append_text(mut text: PendingText, line_index: usize, line: &str) -> PendingText { + text.end_line = line_index; + text.lines.push(line.to_string()); + text +} + +pub fn finalize_text(text: PendingText) -> TextBlock { + let content = text.lines.join("\n"); + TextBlock { + range: LineRange { + start_line: text.start_line, + end_line: text.end_line, + }, + content, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn single_line_text_block() { + let pending = start_text(0, "hello"); + let block = finalize_text(pending); + assert_eq!(block.content, "hello"); + assert_eq!(block.range.start_line, 0); + assert_eq!(block.range.end_line, 0); + } + + #[test] + fn multi_line_text_block() { + let text = start_text(2, "first"); + let text = append_text(text, 3, "second"); + let text = append_text(text, 4, "third"); + let block = finalize_text(text); + assert_eq!(block.content, "first\nsecond\nthird"); + assert_eq!(block.range.start_line, 2); + assert_eq!(block.range.end_line, 4); + } + + #[test] + fn empty_line_preserved_in_content() { + let text = start_text(0, "before"); + let text = append_text(text, 1, ""); + let text = append_text(text, 2, "after"); + let block = finalize_text(text); + assert_eq!(block.content, "before\n\nafter"); + } + + #[test] + fn whitespace_only_line_preserved() { + let pending = start_text(5, " "); + let block = finalize_text(pending); + assert_eq!(block.content, " "); + assert_eq!(block.range.start_line, 5); + } + + #[test] + fn append_updates_end_line_only() { + let text = start_text(3, "first"); + let text = append_text(text, 4, "second"); + assert_eq!(text.start_line, 3); + assert_eq!(text.end_line, 4); + } + + // --- Property tests --- + + use proptest::prelude::*; + + fn valid_command_name() -> impl Strategy { + "[a-z][a-z0-9\\-]{0,20}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + } + + fn arbitrary_line() -> impl Strategy { + prop_oneof![ + // plain text + "[a-zA-Z0-9 !.,]{0,80}", + // command-like + valid_command_name().prop_flat_map(|name| { + "[a-zA-Z0-9 ]{0,40}".prop_map(move |args| format!("/{name} {args}")) + }), + // leading whitespace + command + valid_command_name().prop_flat_map(|name| { + (1usize..5, "[a-zA-Z0-9 ]{0,40}").prop_map(move |(spaces, args)| { + format!("{}/{} {}", " ".repeat(spaces), name, args) + }) + }), + ] + } + use crate::{ + application::line_classify::{LineKind, classify_line}, + domain::ArgumentMode, + }; + proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + + + + fn classify_never_panics(line in "[\\x00-\\x7F]{0,200}") { + let _ = classify_line(&line); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn valid_name_always_produces_command(name in valid_command_name(), args in "[a-z0-9 ]{0,40}") { + let input = format!("/{name} {args}"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.name, name), + LineKind::Text => panic!("expected Command for input: {input}"), + } + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_without_slash_is_never_command(line in "[a-zA-Z0-9 !.,]{1,80}") { + prop_assert!(matches!(classify_line(&line), LineKind::Text)); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn raw_field_preserves_original_input(line in arbitrary_line()) { + if let LineKind::Command(h) = classify_line(&line) { + prop_assert_eq!(h.raw, line); + } + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fence_mode_iff_backticks_present(name in valid_command_name(), lang in "[a-z]{0,10}") { + let input = format!("/{name} ```{lang}"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Fence), + _ => panic!("expected Command"), + } + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn continuation_mode_iff_trailing_backslash( + name in valid_command_name(), + args in "[a-z0-9 ]{1,30}" + ) { + let input = format!("/{name} {args} \\"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Continuation), + _ => panic!("expected Command"), + } + } + } +} diff --git a/parser/src/lib.rs b/parser/src/lib.rs new file mode 100644 index 0000000..0001c4b --- /dev/null +++ b/parser/src/lib.rs @@ -0,0 +1,5 @@ +mod application; +pub mod domain; + +// Public API re-export +pub use application::parse_document; diff --git a/parser/tests/mod.rs b/parser/tests/mod.rs new file mode 100644 index 0000000..be37b0f --- /dev/null +++ b/parser/tests/mod.rs @@ -0,0 +1,21 @@ +// use super::*; +// use crate::domain::{ArgumentMode, ParserContext}; + +// mod proptest; + +// // --- Test helpers --- + +// fn default_context() -> ParserContext { ParserContext::default() } + +// fn parse(input: &str) -> SlashParseResult { parse_to_domain(input, default_context()) } + +// fn assert_single_command(input: &str) -> Command { +// let result = parse(input); +// assert_eq!(result.commands.len(), 1, "expected exactly 1 command"); +// result.commands.into_iter().next().unwrap() +// } + +// fn assert_no_commands(input: &str) { +// let result = parse(input); +// assert!(result.commands.is_empty(), "expected no commands, got {}", result.commands.len()); +// } From 11d1fb9034ae8be1d5108b88a12cc91ee262e6e7 Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Wed, 18 Mar 2026 20:46:33 -0600 Subject: [PATCH 03/13] doc: new 0.3.0 specification, clean slated on ADRs --- ...ward-slash-as-command-trigger-character.md | 62 - ...-continuation-with-trailing-space-slash.md | 98 -- ...lock-syntax-for-raw-multi-line-payloads.md | 89 -- .../0004-state-machine-parser-with-no-ast.md | 78 -- ...ment-payload-with-command-level-parsing.md | 87 -- ...sitory-structure-and-project-boundaries.md | 116 -- ...-build-tooling-and-toolchain-management.md | 84 -- .../0008-sequential-zero-based-command-ids.md | 61 - ...lt-output-with-pretty-json-array-option.md | 109 -- ...put-model-and-line-ending-normalization.md | 57 - docs/slash-cmd parseing sematic.md | 314 ----- docs/slash-parser CLI Specification.md | 19 +- docs/slash-parser-spec-v0.3.0.md | 1096 +++++++++++++++++ docs/slash-parser.md | 333 ----- 14 files changed, 1098 insertions(+), 1505 deletions(-) delete mode 100644 docs/adrs/0001-use-forward-slash-as-command-trigger-character.md delete mode 100644 docs/adrs/0002-line-continuation-with-trailing-space-slash.md delete mode 100644 docs/adrs/0003-fenced-block-syntax-for-raw-multi-line-payloads.md delete mode 100644 docs/adrs/0004-state-machine-parser-with-no-ast.md delete mode 100644 docs/adrs/0005-opaque-argument-payload-with-command-level-parsing.md delete mode 100644 docs/adrs/0006-repository-structure-and-project-boundaries.md delete mode 100644 docs/adrs/0007-build-tooling-and-toolchain-management.md delete mode 100644 docs/adrs/0008-sequential-zero-based-command-ids.md delete mode 100644 docs/adrs/0009-jsonl-default-output-with-pretty-json-array-option.md delete mode 100644 docs/adrs/0010-input-model-and-line-ending-normalization.md delete mode 100644 docs/slash-cmd parseing sematic.md create mode 100644 docs/slash-parser-spec-v0.3.0.md delete mode 100644 docs/slash-parser.md diff --git a/docs/adrs/0001-use-forward-slash-as-command-trigger-character.md b/docs/adrs/0001-use-forward-slash-as-command-trigger-character.md deleted file mode 100644 index e3c78e0..0000000 --- a/docs/adrs/0001-use-forward-slash-as-command-trigger-character.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -number: 1 -title: Use Forward Slash as Command Trigger Character -date: 2026-03-13 -status: accepted ---- - -# 1. Use Forward Slash as Command Trigger Character - -Date: 2026-03-13 - -## Status - -Accepted - -## Context - -The parser needs a reliable way to distinguish command lines from free-form text in a mixed-content -input stream. The trigger character must be unambiguous in the first column, easy to type, and -familiar to users of chat-style slash command interfaces (Discord, Slack, etc.). It must also allow -the parser to detect commands in a single left-to-right scan without backtracking. - -## Decision - -A command line is any line whose first non-whitespace character is `/` (forward slash, U+002F). - -Command line structure: - -```text -/[] -``` - -Command name rules: - -- Regex: `[a-z][a-z0-9-]*` -- Starts immediately after the leading `/` with no intervening space. -- Ends at the first whitespace character or end-of-line. -- Must begin with a lowercase ASCII letter and may contain lowercase letters, digits, and hyphens. - -Arguments prefix (optional): - -- Everything after the first whitespace following the command name. -- May contain inline arguments, an inline fence opener (see ADR-0003), or a continuation marker (see - ADR-0002). - -Any line that does not begin (after optional leading whitespace) with `/` is a non-command line. -Non-command lines are collected into text blocks (see ADR-0009). - -A line where the first non-whitespace character is `/` but the text after `/` does not match the -command name regex (e.g., a bare `/` or `/123`) is treated context-dependently: during accumulation -it becomes a literal payload line; in idle state it is treated as a non-command line. - -## Consequences - -- Command detection is a single character check on each line after skipping leading whitespace, - making it O(1) per line. -- The forward slash convention aligns with established chat/CLI slash command patterns, reducing - user learning curve. -- The strict command name regex prevents ambiguity with path-like content (e.g., `/usr/bin` fails - the regex because `usr/bin` contains `/`). -- Optional leading whitespace tolerance means indented commands (e.g., in nested contexts) are still - recognized. diff --git a/docs/adrs/0002-line-continuation-with-trailing-space-slash.md b/docs/adrs/0002-line-continuation-with-trailing-space-slash.md deleted file mode 100644 index d92896c..0000000 --- a/docs/adrs/0002-line-continuation-with-trailing-space-slash.md +++ /dev/null @@ -1,98 +0,0 @@ ---- -number: 2 -title: Line Continuation with Trailing Space-Slash -date: 2026-03-13 -status: accepted ---- - -# 2. Line Continuation with Trailing Space-Slash - -Date: 2026-03-13 - -## Status - -Accepted - -## Context - -Commands often need arguments that are too long or too complex for a single line. The parser needs -an explicit, opt-in mechanism for spanning a logical command across multiple physical lines. The -marker must be unambiguous (not confused with path separators or content containing slashes) and -must compose cleanly with the other argument modes (single-line and fenced blocks). - -## Decision - -Continuation mode is activated by a trailing space-slash (`/`) marker at the end of a line. - -### Marker definition - -A continuation marker is a line (after normalization per ADR-0010) whose content ends with a space -followed by a slash (`/`) immediately before the line terminator, with nothing after the slash -except optional trailing whitespace. - -Formally, a line is a continuation marker if the last two non-whitespace-trailing characters are `/` -(space + slash). - -Optional leading whitespace is allowed. For example, `/echo /` and `/` (space+slash with -indentation) both qualify. - -A special case: a line that, aside from leading whitespace, is exactly `/` represents a blank -payload line in continuation mode. - -### First command line with continuation - -When a command's first line ends with `/`: - -1. The parser strips the final `/` from that line's content. -2. The remaining content after the command name (which may be empty) becomes the first segment of - `arguments.payload`, followed by a newline separator. -3. `arguments.mode` is set to `continuation`. -4. The parser transitions to the `accumulating` state (see ADR-0004). - -If the first line does not end with `/` and has no fence opener, the command is single-line (see -ADR-0005). - -### Lines in accumulating state - -For each subsequent line while in `accumulating`: - -- Continuation marker line (ends in `/`): represents a blank payload line. The parser appends a - newline to `arguments.payload` and remains in `accumulating`. -- Empty line (zero characters after normalization): the parser finalizes the command and transitions - to `idle`. This blank line is not appended to the payload. -- Any other non-empty line: the parser appends the line content plus a newline to - `arguments.payload` and remains in `accumulating`. - -A fence opener encountered on a continuation line transitions the command from `accumulating` to -`inFence` (see ADR-0003). - -### Bare slash is not continuation - -A bare `/` at the end of a line (no preceding space) is not a continuation marker. If the line's -first non-whitespace character is `/`, it is processed via command detection (ADR-0001). If the `/` -appears elsewhere, it is literal content. Users needing complex or ambiguous multi-line payloads -should use fenced blocks (ADR-0003). - -Example: the input - -```text -/echo / -ooga booga -/ -testing 123 -``` - -does not include `testing 123` in `/echo`'s payload because the bare `/` line is not a `/` marker. - -## Consequences - -- Continuation is explicit and opt-in; lines without the marker never accidentally join. -- The space-slash marker avoids ambiguity with path-like content (e.g., `/usr/bin/` has no preceding - space before the trailing slash). -- Empty lines serve as a natural, visible terminator for multi-line commands. -- Blank payload lines can be represented by the `/` special case, so continuation mode can express - payloads containing empty lines. -- Round-trip fidelity of original whitespace at continuation boundaries is intentionally lost; the - canonical form uses newline separators between payload segments. -- Fenced blocks (ADR-0003) remain available as the preferred alternative when payloads are complex - or contain content that could be confused with markers. diff --git a/docs/adrs/0003-fenced-block-syntax-for-raw-multi-line-payloads.md b/docs/adrs/0003-fenced-block-syntax-for-raw-multi-line-payloads.md deleted file mode 100644 index a9b907b..0000000 --- a/docs/adrs/0003-fenced-block-syntax-for-raw-multi-line-payloads.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -number: 3 -title: Fenced Block Syntax for Raw Multi-line Payloads -date: 2026-03-13 -status: accepted ---- - -# 3. Fenced Block Syntax for Raw Multi-line Payloads - -Date: 2026-03-13 - -## Status - -Accepted - -## Context - -Some commands need to carry large, raw, multi-line payloads (JSON documents, code snippets, -configuration blocks) where the content itself may contain sequences that would otherwise be -interpreted as continuation markers (`/`) or command triggers (`/`). Continuation mode (ADR-0002) -cannot safely transport such content because every line is subject to marker detection. A verbatim -transport mechanism is needed that suspends all parsing rules until an explicit closing delimiter. - -## Decision - -Fenced blocks use markdown-style backtick fences to wrap raw payloads attached to a command. - -### Fence openers - -There are two ways to enter fence mode: - -1. Inline fence on the command line: - - ````text - / ```[lang] - - ```` - ``` - The first occurrence of three or more consecutive backticks in the arguments prefix is treated as the fence opener. Text before the opener is kept in `arguments.header`. The parser records the fence marker length (number of backticks) and the optional language identifier (e.g., `jsonl`). - ``` - -2. Fence on the next line after continuation: - - ````text - / / - ```[lang] - - ```` - ``` - The first line ends with ` /` (entering continuation per ADR-0002), but the next line is recognized as a fence opener. The parser transitions from `accumulating` to `inFence` for this command. - ``` - -### Fence mode semantics - -While in `inFence` state (see ADR-0004): - -- All lines (including blank lines) are appended to `arguments.payload` with newline separators. -- Continuation markers (`/`) inside a fence are treated as literal content, not syntax. -- Command triggers (`/`) at the start of a line inside a fence are treated as literal content. - -### Closing fence - -A line is a closing fence if, after trimming leading and trailing whitespace: - -- It consists solely of backticks. -- The number of backticks is greater than or equal to the opener's count. - -The closing fence line itself is not appended to `arguments.payload`. - -### Resulting command state - -Once the closing fence is found: - -- `arguments.mode` is set to `fence`. -- `arguments.fence_lang` is the captured language tag or null. -- The command is finalized and the parser transitions back to `idle`. - -## Consequences - -- Payloads inside fenced blocks are completely verbatim; no parsing rules apply to content lines, - eliminating escaping concerns. -- The variable-length backtick fence (3 or more) means content containing triple backticks can be - fenced with four or more backticks, avoiding collision. -- The inline and continuation-then-fence entry paths give authors flexibility in how they structure - command lines. -- Language tags (e.g., `jsonl`, `yaml`) are captured as metadata, enabling downstream consumers to - apply format-specific parsing to the payload. -- Fenced blocks have a clear, mandatory closing delimiter, so the parser always knows when the - payload ends (unlike continuation mode which relies on empty line termination). diff --git a/docs/adrs/0004-state-machine-parser-with-no-ast.md b/docs/adrs/0004-state-machine-parser-with-no-ast.md deleted file mode 100644 index 4e851d6..0000000 --- a/docs/adrs/0004-state-machine-parser-with-no-ast.md +++ /dev/null @@ -1,78 +0,0 @@ ---- -number: 4 -title: State Machine Parser with No AST -date: 2026-03-13 -status: accepted ---- - -# 4. State Machine Parser with No AST - -Date: 2026-03-13 - -## Status - -Accepted - -## Context - -The slash-cmd format is line-oriented with three distinct argument modes (single-line, continuation, -fenced). A traditional recursive-descent parser or AST-building approach would add unnecessary -complexity for a format that can be fully parsed in a single forward pass over lines. The parser -needs to track which mode it is in, accumulate payloads accordingly, and emit finalized commands and -text blocks without constructing an intermediate tree representation. - -## Decision - -The parser uses a line-based state machine with three primary states and no intermediate AST. - -### States - -- `idle`: not currently accumulating a command. The parser scans each line for command detection - (ADR-0001). -- `accumulating`: collecting continuation-mode arguments for a command (ADR-0002). Each line is - tested for continuation markers, empty-line terminators, fence openers, or literal content. -- `inFence`: collecting raw lines inside a fenced block for a command (ADR-0003). All lines are - literal until a closing fence is detected. - -### Tracked state per command - -While processing the current command, the parser tracks: - -- `name`: the command name from the first line. -- `arguments.header`: the header text after the command name on the first line, before any fence - opener. -- `arguments.mode`: one of `single-line`, `continuation`, or `fence`. -- `arguments.fence_lang`: optional language identifier when in fence mode. -- `arguments.payload`: the assembled argument string, with newlines between logical payload lines. - -### Multi-command scanning - -The parser walks the normalized input (ADR-0010) line by line: - -1. In `idle`, when it encounters a command line, it starts a new command according to argument mode - rules (ADR-0005 for single-line, ADR-0002 for continuation, ADR-0003 for fenced). -2. After a command is finalized (single-line completion, empty-line termination in continuation, or - fence close), the parser returns to `idle` and continues scanning. -3. Non-command lines encountered in `idle` are collected into text blocks. Each text block records - its line range (start_line, end_line inclusive) and content with internal newline separators - reflecting the normalized input. -4. Commands and text blocks are emitted in document order. - -### No AST - -The parser emits a flat list of commands and text blocks directly. There is no intermediate abstract -syntax tree. The `children` field on commands is reserved for future hierarchical structures but is -currently always empty. - -## Consequences - -- The single-pass, line-based approach is simple to implement, test, and reason about. Each state - has a small, well-defined set of transitions. -- Memory usage is proportional to the largest single command payload, not the entire document, since - commands are finalized and emitted incrementally. -- No AST means no tree-manipulation overhead. Consumers work directly with the flat command and text - block lists. -- Adding new states or argument modes in the future requires extending the state machine rather than - restructuring a grammar. -- The three-state design maps directly to the three argument modes, making the correspondence - between spec and implementation transparent. diff --git a/docs/adrs/0005-opaque-argument-payload-with-command-level-parsing.md b/docs/adrs/0005-opaque-argument-payload-with-command-level-parsing.md deleted file mode 100644 index d4f9232..0000000 --- a/docs/adrs/0005-opaque-argument-payload-with-command-level-parsing.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -number: 5 -title: Opaque Argument Payload with Command-Level Parsing -date: 2026-03-13 -status: accepted ---- - -# 5. Opaque Argument Payload with Command-Level Parsing - -Date: 2026-03-13 - -## Status - -Accepted - -## Context - -Different commands have wildly different argument shapes: some take a single keyword, others take -key-value pairs, and others need entire JSON or YAML documents. If the parser tried to understand -argument structure (quoting, escaping, key-value syntax), it would need to know every command's -grammar. This couples the parser to command definitions and makes the format difficult to extend. - -The parser needs to deliver arguments to commands as an opaque payload, letting each command define -its own interpretation. - -## Decision - -The parser treats all argument content as opaque byte sequences. It determines argument boundaries -and transport mode but never interprets argument semantics. - -### Argument structure - -Every command carries an `arguments` object with the following fields: - -- `header`: the text after the command name on the first line, before any fence opener. This is the - inline portion of the arguments. -- `mode`: one of `single-line`, `continuation`, or `fence`, indicating how the payload was - assembled. -- `fence_lang`: the language tag from a fence opener (e.g., `jsonl`), or null if not in fence mode. -- `payload`: the final assembled argument string with newlines between logical payload lines. - -### Three argument modes - -The parser supports exactly three modes for assembling arguments: - -1. Single-line mode (this ADR): the command and all its arguments fit on one line. -2. Continuation mode (ADR-0002): arguments span multiple lines via trailing `/` markers. -3. Fence mode (ADR-0003): arguments are wrapped in backtick fences for verbatim transport. - -### Single-line mode - -A command is single-line if its first line does not end with a continuation marker (`/` per -ADR-0002) and does not contain a fence opener (per ADR-0003). - -In single-line mode: - -- The text after the command name, trimmed of the initial separating space, is the entire argument. -- The command is finalized immediately on that line. -- `arguments.mode` is set to `single-line`. -- `arguments.payload` is exactly that argument string. The parser does not append a trailing - newline. - -Example: `/echo hello world` produces `arguments.payload = "hello world"` with -`arguments.mode = "single-line"`. - -### Command-level parsing - -The parser's job ends at delivering the opaque payload. Each command implementation is responsible -for interpreting its own payload content (splitting on spaces, parsing JSON, etc.). The parser makes -no assumptions about quoting, escaping, or structure within the payload. - -## Consequences - -- The parser is command-agnostic. New commands can be added without modifying parser logic. -- Argument parsing errors are the command's responsibility, not the parser's, which simplifies error - boundaries. -- The three-mode system gives authors a clear choice: single-line for simple arguments, continuation - for moderate multi-line, and fenced blocks for complex or ambiguous payloads. -- The `header` field preserves inline arguments even when a fence is used, allowing commands to - accept both positional arguments and a fenced body (e.g., ``/mcp call_tool write_file ```jsonl``). -- No quoting or escaping conventions are imposed by the parser, avoiding conflicts with - payload-specific syntax like JSON strings or shell expressions. -- Textual reconstruction (format_text) guarantees spec-level semantic roundtrip fidelity, not - byte-for-byte reproduction of the original input. Line ending normalization (ADR-0010), whitespace - handling, and continuation marker stripping are lossy transformations. The roundtrip invariant is: - `parse(to_plaintext(parse(input)))` produces structurally equivalent commands and text blocks - (same names, modes, payloads, and text block content), not identical bytes. diff --git a/docs/adrs/0006-repository-structure-and-project-boundaries.md b/docs/adrs/0006-repository-structure-and-project-boundaries.md deleted file mode 100644 index f5f7bf4..0000000 --- a/docs/adrs/0006-repository-structure-and-project-boundaries.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -number: 6 -title: Repository Structure and Project Boundaries -date: 2026-03-14 -status: accepted ---- - -# 6. Repository Structure and Project Boundaries - -Date: 2026-03-14 - -## Status - -Accepted - -## Context - -The slash project encompasses a Rust parser core, WASM build targets, language-specific SDKs, a CLI -tool (riff), distribution packaging, a documentation website, and a formal spec. - -Deep directory nesting (e.g., `core/parser/`, `sdk/slash-python/`, `cli/riff/`) adds navigation -friction. A flat layout with a strict naming convention communicates category membership and -dependency tiers without requiring a nested filesystem hierarchy. - -Naming: "slash" is the format and project name (80s rock reference). "riff" is the CLI tool (guitar -riff; it reads stdin, writes stdout, stays simple). All SDK packages are prefixed `slash-` for -discoverability in package registries. All CLI packaging projects are prefixed `riff-`. - -Because this is a polyglot monorepo, we need strict architectural constraints to ensure SDKs don't -accidentally import the CLI, and that WASM builds don't depend on downstream consumers. The -mechanisms for enforcing these constraints (e.g., build system tooling) will be handled in a -separate tooling ADR. - -## Decision - -The repository uses a flat, single-level project layout. Logical grouping and dependency enforcement -are handled via project metadata and tags, not directory depth. - -### Project Layout - -```text -wit/ core WIT interface definitions (single source of truth) -parser/ core Rust library, internal to repo, not published -wasm-javascript/ wasm wasm-bindgen module for JS/TS runtimes -wasm-wasi/ wasm WASI component for polyglot SDK consumption -slash-web/ sdk Browser runtime SDK (depends on wasm-javascript) -slash-javascript/ sdk ESM server-side SDK (depends on wasm-javascript) -slash-python/ sdk Python SDK (depends on wasm-wasi) -slash-ruby/ sdk Ruby SDK (depends on wasm-wasi) -slash-php/ sdk PHP SDK (depends on wasm-wasi) -slash-elixir/ sdk Elixir SDK (depends on wasm-wasi) -slash-ocaml/ sdk OCaml SDK (depends on wasm-wasi) -slash-haskell/ sdk Haskell SDK (depends on wasm-wasi) -slash-dart/ sdk Dart SDK (depends on wasm-wasi) -slash-java/ sdk Java SDK (depends on wasm-wasi) -slash-go/ sdk Go SDK (depends on wasm-wasi) -slash-zig/ sdk Zig SDK (native FFI or wasm-wasi) -slash-rust/ sdk Rust SDK, thin published crate wrapping parser -riff-cli/ cli CLI binary (depends on slash-rust) -riff-deb/ pkg Debian package -riff-rpm/ pkg RPM package -riff-oci/ pkg OCI container image -riff-proto/ pkg Proto toolchain plugin -website/ docs Static site for documentation and promotion -docs/ docs Formal specifications -docs/adrs/ docs Architecture Decision Records -``` - -### Module Boundaries - -Projects are grouped into six distinct tags/tiers to define architectural boundaries. Build tooling -must enforce that dependencies only flow downward through these layers: - -- `core`: the internal parser library and WIT definitions. Depends on nothing. -- `wasm`: WASM build targets that compile `parser`. Can only depend on `core`. -- `sdk`: language-specific packages wrapping a WASM module (or native FFI). Can only depend on - `wasm` and `core`. -- `cli`: the riff binary. Can only depend on `sdk` and `core`. -- `pkg`: distribution packaging for riff. Can only depend on `cli`. -- `docs`: website, specifications, and ADRs. Can only depend on `docs`. - -### Dependency Flow and The WIT Contract - -```text -wit -> parser -parser -> wasm-javascript -> slash-javascript, slash-web -parser -> wasm-wasi -> slash-python, slash-ruby, slash-php, ... -parser -> slash-rust -> riff-cli -> riff-deb, riff-rpm, riff-oci, riff-proto -docs -> website -``` - -The `wit/` directory contains the WebAssembly Interface Types definitions, acting as the single -source of truth for the parser's API surface. Language SDKs generate types and thin wrappers based -on this contract. - -### Internal vs Published Code - -`parser/` is an internal Rust crate. It is consumed only by `wasm-javascript`, `wasm-wasi`, and -`slash-rust` within this repo. External Rust consumers use the `slash-rust` crate, which provides -the stable public API. - -To keep the repository root clean, all fuzzing, profiling, and testing infrastructure for the core -engine must live entirely within the `parser/` directory, avoiding root-level artifact clutter. - -## Consequences - -- Every project is one `cd` from root. No navigating through intermediate grouping directories. -- Prefix conventions (`slash-*`, `riff-*`, `wasm-*`) make filesystem navigation and glob-based - targeting self-documenting. -- The `parser` crate stays internal, giving freedom to change its API without semver concerns. Only - `slash-rust` carries the public contract. -- Adding a new language SDK requires only a single new root directory; no restructuring is needed. -- Clear structural boundaries physically prevent architectural regressions (e.g., an SDK cannot - accidentally depend on the CLI). -- The specification and ADRs live in `docs/` alongside the code. The website project pulls from - `docs/` at build time, ensuring documentation is version-controlled with the codebase. diff --git a/docs/adrs/0007-build-tooling-and-toolchain-management.md b/docs/adrs/0007-build-tooling-and-toolchain-management.md deleted file mode 100644 index a9d58e6..0000000 --- a/docs/adrs/0007-build-tooling-and-toolchain-management.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -number: 7 -title: Build Tooling and Toolchain Management -date: 2026-03-14 -status: accepted ---- - -# 7. Build Tooling and Toolchain Management - -Date: 2026-03-14 - -## Status - -Accepted - -## Context - -The slash project is a polyglot monorepo containing Rust (core parser), WebAssembly components, and -multiple language-specific SDKs. Managing this requires a build system that can orchestrate a -directed acyclic graph (DAG) of tasks across different language ecosystems. We also need tooling to -physically enforce the architectural boundaries established in ADR 6, both between projects (macro) -and within projects (micro). - -Furthermore, we must prevent "works on my machine" issues by guaranteeing that all contributors and -CI runners use the exact same toolchain versions, while preventing our monorepo orchestrator from -getting into a tug-of-war with workspace-aware tools like Cargo and pnpm. - -## Decision - -We will use Moon as our primary task orchestrator and monorepo manager, alongside its companion tool -`proto` for toolchain management. Moon will wrap, rather than replace, the native tools for each -language ecosystem. - -### Boundary Enforcement (Macro vs Micro) - -Tooling will explicitly enforce the architectural constraints defined in ADR 6 at two levels: - -1. **Macro-Boundaries (Project Level):** We use Moon's `tagRelationships` in `.moon/workspace.yml`. - Projects are tagged (`core`, `wasm`, `sdk`, `cli`, `pkg`, `docs`). If an `sdk` project attempts - to depend on a `cli` project, Moon will fail the build graph generation. This physically prevents - architectural regressions across the monorepo. - -2. **Micro-Boundaries (File Level):** Within individual projects (like the `parser/` crate), we use - ecosystem-specific tools configured as Moon lint tasks to enforce Clean Architecture: - - **Rust:** We use `clippy.toml` restrictions (`disallowed-methods`, `disallowed-types`) to - prevent Infrastructure IO and WASM types from leaking into the `src/domain/` and - `src/application/` modules. We may adopt `tach` in the future for stricter folder-to-folder DAG - enforcement. - - **TypeScript:** We use `eslint-plugin-boundaries` to prevent illegal cross-folder imports. - -### Toolchain Pinning (Proto) - -All toolchain versions are strictly pinned at the repository root: - -- `.prototools` acts as the single source of truth for exact version numbers (`rust = "1.85.0"`). -- `.moon/toolchains.yml` handles configuration behavior (e.g., auto-installing the `wasm32-wasip1` - target) but explicitly omits version numbers to prevent configuration drift. - -### Virtual Workspaces and Task Scoping - -To support shared lockfiles and efficient caching, we use root-level virtual workspaces (a root -`Cargo.toml` and `pnpm-workspace.yaml`). - -**The Task Scoping Rule:** To prevent Cargo and pnpm from attempting to build the entire monorepo -when invoked inside a specific project, all inherited Moon tasks must use scope flags injected with -Moon's `$project` variable (e.g., `cargo build -p $project` instead of bare `cargo build`). - -### Editor Configuration & Validation - -All Moon configuration files will utilize remote JSON `$schema` headers. To prevent the VS Code YAML -extension from reporting false-positive validation errors, contributors must use the -`moon: Append YAML schemas configuration to settings` command provided by the official Moon VS Code -extension. - -## Consequences - -- Complete reproducibility: anyone cloning the repo will automatically download the correct - compilers and toolchains via `proto`. -- Clean Architecture is structurally enforced: Moon tags prevent illegal cross-project imports, and - Clippy/ESLint prevent illegal intra-project imports. -- By using `$project` variables with scope flags (`-p`), Moon and Cargo cooperate peacefully rather - than fighting over workspace control. -- The use of `$schema` headers combined with the VS Code extension provides immediate IDE validation - and prevents silent configuration errors. diff --git a/docs/adrs/0008-sequential-zero-based-command-ids.md b/docs/adrs/0008-sequential-zero-based-command-ids.md deleted file mode 100644 index e15f761..0000000 --- a/docs/adrs/0008-sequential-zero-based-command-ids.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -number: 8 -title: Sequential Zero-Based Command IDs -date: 2026-03-13 -status: accepted ---- - -# 8. Sequential Zero-Based Command IDs - -Date: 2026-03-13 - -## Status - -Accepted - -## Context - -The parser output contains multiple commands and text blocks that consumers need to reference, -correlate, and process. Each element needs a stable, predictable identifier that conveys its type -and position within the document. UUIDs or random IDs would make output non-deterministic and harder -to test. Line numbers alone are fragile because they change when content is inserted or removed -above. - -## Decision - -Every command and text block receives a sequential, zero-based identifier with a type prefix. - -### ID format - -- Commands: `cmd-0`, `cmd-1`, `cmd-2`, etc. -- Text blocks: `text-0`, `text-1`, `text-2`, etc. - -The prefix indicates the element type. The numeric suffix is a zero-based counter that increments -independently for each type, in document order. - -### Assignment rules - -1. IDs are assigned in the order elements appear in the normalized input. -2. The command counter and text block counter are independent sequences, both starting at 0. -3. IDs are deterministic: the same input always produces the same IDs. - -### Range tracking - -In addition to the ID, every command and text block carries a `range` object with: - -- `start_line`: the first line of the element (inclusive). -- `end_line`: the last line of the element (inclusive). - -Line numbers provide a secondary reference for diagnostics and error reporting, complementing the -stable ID. - -## Consequences - -- Output is fully deterministic, making snapshot testing and diffing straightforward. -- The type prefix eliminates ambiguity when referencing elements (no confusion between command 1 and - text block 1). -- Zero-based indexing aligns with array indexing in most programming languages, simplifying consumer - code. -- Independent counters mean inserting a new text block does not change command IDs, and vice versa. -- The `range` field preserves source location for error reporting without overloading the ID with - positional information. diff --git a/docs/adrs/0009-jsonl-default-output-with-pretty-json-array-option.md b/docs/adrs/0009-jsonl-default-output-with-pretty-json-array-option.md deleted file mode 100644 index 5ef2fa7..0000000 --- a/docs/adrs/0009-jsonl-default-output-with-pretty-json-array-option.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -number: 9 -title: JSONL Default Output with Pretty JSON Array Option -date: 2026-03-13 -status: accepted ---- - -# 9. JSONL Default Output with Pretty JSON Array Option - -Date: 2026-03-13 - -## Status - -Accepted - -## Context - -The parser needs a serialization format for its output that is both machine-readable for downstream -tooling and human-inspectable for debugging. The output must represent the full result of a parse -run, including metadata, commands with their argument payloads, and interleaved text blocks. Two -common JSON-based approaches exist: a single JSON document (easy to pretty-print, harder to stream) -and JSONL (one object per line, streamable, harder to read). - -## Decision - -The parser supports two output formats, selectable via CLI flag (see ADR-0007): - -- JSONL (default): one JSON object per line, suitable for streaming and piping. -- Pretty JSON array: a single formatted JSON document, suitable for human inspection. - -### Output envelope - -A parser run produces a JSON object with the following top-level structure: - -```json -{ - "version": "0.1.0", - "context": { - "source": "string", - "timestamp": "2026-03-13T10:40:00Z", - "user": "string", - "session_id": "string", - "extra": {} - }, - "commands": [], - "text_blocks": [] -} -``` - -- `version`: semantic version of the output schema. -- `context`: metadata about the parse run. Fields `source`, `timestamp`, `user`, and `session_id` - are populated from CLI flags (ADR-0007) or defaults. `extra` is a free-form object for additional - metadata. -- `commands`: ordered array of Command objects. -- `text_blocks`: ordered array of TextBlock objects. - -### Command schema - -Each element in `commands` is: - -````json -{ - "id": "cmd-0", - "name": "mcp", - "raw": "/mcp call_tool write_file ```jsonl\n...\n```", - "range": { "start_line": 10, "end_line": 20 }, - "arguments": { - "header": "call_tool write_file", - "mode": "fence", - "fence_lang": "jsonl", - "payload": "{\n \"path\": \"...\"\n}" - }, - "children": [] -} -```` - -- `id`: unique identifier per ADR-0008 (`cmd-0`, `cmd-1`, etc.). -- `name`: command name without the leading `/`. -- `raw`: exact source slice for the command (header + all argument lines). -- `range`: inclusive line range the command covers (`start_line`, `end_line`). -- `arguments`: argument payload object per ADR-0005 with `header`, `mode`, `fence_lang`, and - `payload` fields. -- `children`: reserved for future hierarchical structures, currently always empty. - -### TextBlock schema - -Each element in `text_blocks` is: - -```json -{ "id": "text-0", "range": { "start_line": 0, "end_line": 9 }, "content": "arbitrary text\n..." } -``` - -- `id`: unique identifier per ADR-0008 (`text-0`, `text-1`, etc.). -- `range`: inclusive line range. -- `content`: exact text for the block with internal newline separators from the normalized input. - -## Consequences - -- JSONL as default enables streaming pipelines where consumers process commands as they are emitted - without waiting for the full document. -- Pretty JSON mode provides a single-document view for debugging and manual inspection. -- The envelope schema captures both parse results and run metadata, making output self-describing - and reproducible. -- The `raw` field on commands preserves the original source text for round-trip inspection or error - reporting. -- The `children` field is a forward-compatible extension point that does not affect current - consumers. -- Text blocks between commands are explicitly captured rather than discarded, ensuring no input - content is silently lost. diff --git a/docs/adrs/0010-input-model-and-line-ending-normalization.md b/docs/adrs/0010-input-model-and-line-ending-normalization.md deleted file mode 100644 index c3d0bd9..0000000 --- a/docs/adrs/0010-input-model-and-line-ending-normalization.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -number: 10 -title: Input Model and Line Ending Normalization -date: 2026-03-14 -status: accepted ---- - -# 10. Input Model and Line Ending Normalization - -Date: 2026-03-14 - -## Status - -Accepted - -## Context - -The parser consumes text input from various sources (files, stdin, editor buffers) that may use -different line ending conventions: CRLF (Windows), LF (Unix), or bare CR (legacy Mac). Without a -normalization step, every downstream rule (command detection, continuation markers, fence -boundaries) would need to handle all three variants independently, increasing complexity and bug -surface. - -A clear input model also needs to distinguish between structural line terminators (which delimit -parser lines) and literal backslash-n sequences that appear inside content such as JSON strings. - -## Decision - -The parser applies line ending normalization as a preprocessing step before any parsing logic runs. - -The normalization rules are: - -1. Replace all CRLF (\r\n) sequences with LF (\n). -2. Replace all remaining bare CR (\r) characters with LF (\n). - -After normalization: - -- All line terminators are LF regardless of the original encoding. -- The parser iterates over the normalized input using LF as the line separator. -- Each LF terminates a line; the LF itself is not part of the line content. -- A trailing LF at the end of input produces a trailing empty line. -- Literal escape sequences like `\\n` inside content (e.g., JSON strings `"blah\\nblah"`) are - ordinary characters, not line terminators. They are preserved verbatim in payloads. - -The parser then operates on a sequence of logical lines (possibly including blank lines) derived -solely from the normalized line terminators. - -## Consequences - -- All downstream parsing rules (command detection per ADR-0001, continuation per ADR-0002, fencing - per ADR-0003) operate on a uniform LF-only input model. -- Input from any OS line ending convention is handled identically with no special-casing in the - state machine (ADR-0004). -- The normalization is a single pass over the input, adding negligible overhead. -- Content-embedded escape sequences are never confused with structural line terminators. -- The preprocessing step is the first thing that runs, establishing a clean invariant that all - subsequent parsing logic can rely on. diff --git a/docs/slash-cmd parseing sematic.md b/docs/slash-cmd parseing sematic.md deleted file mode 100644 index 9650ddf..0000000 --- a/docs/slash-cmd parseing sematic.md +++ /dev/null @@ -1,314 +0,0 @@ -## 1. Input model and line normalization - -- The parser consumes a single text input as a byte/char stream. -- It treats certain characters as **line terminators** in the input stream, distinct from literal - `\n` characters that might appear inside string data. - -### 1.1 Line ending normalization - -Before any parsing: - -- Replace all `"\r\n"` with `"\n"`. -- Replace all remaining `"\r"` with `"\n"`. - -After normalization: - -- `CRLF`, `LF`, and bare `CR` are all treated equivalently as a single `LF`. -- The parser then **iterates over the input using these `\n` characters as separators** to obtain - lines. Conceptually: - - - Each `\n` in the normalized input terminates a line. - - The `\n` itself is not part of the line’s content. - - A final `\n` at the end of input produces a trailing empty line. - -- Literal `\n` sequences that appear inside line content (for example in JSON strings like - `"blah\\nblah"`) are **not** treated as line terminators. They are ordinary characters and are - preserved verbatim in the payload. - -Result: the parser operates on a sequence of logical lines (possibly including blank lines) derived -solely from the normalized line terminators. - -## 2. Command detection - -A **command line** is any line whose first non-whitespace character is `/`. - -Command line structure: - -```text -/[] -``` - -- ``: - - Regex: `[a-z][a-z0-9-]*`. - - Starts immediately after the leading `/`. - - Ends at the first whitespace or end-of-line. -- `` (optional): - - Everything after the first whitespace following ``. - - May contain: - - Inline arguments. - - An inline fence opener (```lang). - -Any line that does **not** begin (after leading whitespace) with `/` is a **non-command line**. - -## 3. Parser state machine - -The parser is line-based and uses three primary states: - -- `idle` – not currently accumulating a command. -- `accumulating` – collecting continuation-mode arguments for a command. -- `inFence` – collecting raw lines inside a fenced block for a command. - -For the **current command**, the parser tracks: - -- `name`: the command name from the first line. -- `arguments.header`: the header text after the command name on the first line, **before** any fence - opener. -- `arguments.mode`: one of `"single-line"`, `"continuation"`, or `"fence"`. -- `arguments.fence_lang`: optional language identifier when in fence mode. -- `arguments.payload`: the assembled argument string, with `\n` between logical payload lines. - -## 4. Argument modes - -### 4.1 Single-line mode - -If a command line: - -- Does **not** end with a continuation marker (see 4.2), and -- Does **not** contain a fence opener, - -then: - -- The text after `` (trimmed once for the initial space) is the entire argument. -- The command is finalized on that line. - -Properties: - -- `arguments.mode = "single-line"`. -- `arguments.payload` is exactly that argument string (the parser does not append a newline simply - for being the last argument). - -### 4.2 Continuation mode (explicit `" /"` marker) - -Lines that end with " /" are continuation markers and contribute payload lines as described. - -Lines that are completely empty end the command. - -Lines whose first non-whitespace is / but do not form a valid command name are treated as literal -payload lines. - -Lines that form a valid new command name are out of scope for this command (you can either treat -them as payload or define that they end the current command; your current implementation treats the -invalid / as payload). - -Continuation is **opt-in** and **strict**. - -#### 4.2.1 Continuation marker definition - -- A **continuation marker** is defined as a line (after normalization) whose content ends with a - **space + slash** (`" /"`) immediately before the line terminator, with nothing after that slash. -- Optional leading whitespace is allowed. For example, `"/echo /"` or `" /"` (space+slash with - indentation) both qualify as continuation markers. - -Formally, a line is a continuation marker if it matches: - -- `^[ \t]*.*[ ]/[ ]*$` and the last two non-newline characters are `" /"`. - -A special case of this is a line that, aside from leading whitespace, is exactly `" /"`; this -represents a **blank payload line** in continuation mode. - -#### 4.2.2 First command line with continuation - -For a command’s first line: - -- If it ends with `" /"` (continuation marker): - - The parser **strips** the final `" /"` from that line’s content. - - The remaining content (which might be empty after the header) is appended to - `arguments.payload`, followed by `"\n"` if you treat it as part of the payload body. - - `arguments.mode` is set to `"continuation"`. - - The parser moves to `accumulating`. - -- If it does **not** end with `" /"` and has no fence opener: - - The parser treats the command as a single-line command (4.1). - -#### 4.2.3 Lines in `accumulating` state - -In `accumulating`, for each subsequent line: - -Let `line` be the current line content (without the trailing `\n`): - -- **Continuation marker line** (matches the pattern, ends in `" /"` with no content after it): - - This represents a **blank payload line**. - - The parser appends `"\n"` to `arguments.payload`. - - The parser remains in `accumulating`. - -- **Empty line** (`line == ""` after normalization): - - This is a **true blank line** with no continuation marker. - - The parser **finalizes the command** and transitions to `idle`. - - This blank line is **not** appended to `arguments.payload`. - - Subsequent lines are parsed independently (either as text or new commands). - -- **Any other non-empty line** (not a marker): - - The parser appends `line + "\n"` to `arguments.payload`. - - The parser remains in `accumulating`. - -#### 4.2.4 Bare slash is not continuation - -- A bare `/` at the end of a line (with no preceding space as `" /"`) is **not** a continuation - marker. - -Behavior: - -- If the line’s first non-whitespace character is `/`, it is processed via command detection (either - a new command or invalid command name). -- If the `/` appears elsewhere in the line, it is just part of the literal content. -- The parser does not introduce any special handling for this edge case; users are encouraged to use - fenced blocks when they need more complex or ambiguous multi-line payloads. - -**Implication:** examples like: - -```text -/echo / -ooga booga -/ -testing 123 -``` - -do **not** cause `testing 123` to be included in `/echo`’s payload, because the line with just `/` -is not a `" /"` marker and either starts a new command or is treated as content depending on -position. - -### 4.3 Fenced block mode - -Fenced blocks allow attaching a raw, multi-line payload to a command, similar to markdown code -fences. - -#### 4.3.1 Fence openers - -Two ways to enter fence mode: - -1. **Inline fence on the command line** - - ````text - / ```[lang] - ```` - - - In ``, the first occurrence of three or more consecutive backticks (```…) is - treated as the fence opener. - - Text before the opener is kept in `arguments.header`. - - The parser records: - - Fence marker length (number of backticks). - - Optional language identifier (e.g., `jsonl`). - - The parser enters `inFence` state. - -2. **Fence on the next line after continuation** - - ````text - / / - ```[lang] - - ```` - ``` - - First line ends with `" /"` → command enters `continuation` mode, but the next line is recognized as a fence opener. - - The parser transitions to `inFence` for this command. - - Subsequent lines until the fence close are payload lines for this command. - ``` - -#### 4.3.2 Fence mode semantics - -While in `inFence`: - -- All lines (including blank lines) are appended to `arguments.payload` with `"\n"` separators. -- Continuation markers (`" /"`) inside a fence are ignored as syntax and treated as literal content. -- The parser looks for a **closing fence**: - - - After trimming leading/trailing whitespace, a line is a closing fence if: - - It consists solely of backticks. - - The number of backticks is **greater than or equal** to the opener’s count. - -- The closing fence line itself is **not** appended to `arguments.payload`. -- Once the closing fence is found: - - `arguments.mode = "fence"`. - - `arguments.fence_lang` is the captured language (`lang`) or `null`. - - The command is finalized, and the parser transitions back to `idle`. - -## 5. Multiple commands and text blocks - -- The parser walks the normalized input line by line. -- In `idle`, when it encounters a command line, it starts a new command according to the rules - above. -- After a command is finalized (single-line, continuation termination, or fence close), the parser - returns to `idle` and continues scanning for the next command. -- Non-command text segments between commands can optionally be represented as `text_blocks`: - - - Each `text_block` has: - - `range.start_line`, `range.end_line` (inclusive, zero-based or one-based by convention). - - `content`: the exact text for that block, with internal `\n` separators reflecting the - normalized input. - -## 6. JSON output schematics (summary) - -A parser run produces a JSON object: - -````json -{ - "version": "0.1.0", - "context": { - "source": "string", - "timestamp": "2026-03-13T10:40:00Z", - "user": "string", - "session_id": "string", - "extra": {} - }, - "commands": [ /* Command[] */ ], - "text_blocks": [ /* TextBlock[] */ ] -} -```[] - -### 6.1 Command - -Each `commands[i]` is: - -```json -{ - "id": "cmd-1", - "name": "mcp", - "raw": "/mcp call_tool write_file ```jsonl\n...\n```", - "range": { - "start_line": 10, - "end_line": 20 - }, - "arguments": { - "header": "call_tool write_file", - "mode": "fence", - "fence_lang": "jsonl", - "payload": "{\n \"path\": \"...\"\n}" - }, - "children": [] -} -```[] - -- `id`: unique identifier for this command instance. -- `name`: command name (without the leading `/`). -- `raw`: exact source slice for this command (header + argument lines). -- `range`: inclusive line range that the command covers. -- `arguments`: - - `header`: header arguments from the command line before any fence opener. - - `mode`: `"single-line" | "continuation" | "fence"`. - - `fence_lang`: language tag or `null`. - - `payload`: final assembled argument string with `\n` between logical payload lines. -- `children`: reserved for future hierarchical structures (may be empty). - -### 6.2 TextBlock - -Each `text_blocks[i]` is: - -```json -{ - "id": "text-1", - "range": { - "start_line": 0, - "end_line": 9 - }, - "content": "arbitrary text\n..." -} -```` diff --git a/docs/slash-parser CLI Specification.md b/docs/slash-parser CLI Specification.md index f5fd861..4ff3199 100644 --- a/docs/slash-parser CLI Specification.md +++ b/docs/slash-parser CLI Specification.md @@ -1,6 +1,4 @@ - - -## `riff` CLI Specification +## `riff` CLI Specification - WIP ### 1. Overview @@ -229,17 +227,4 @@ riff -p -c '{"user":"tom","run_id":"abc-123"}' ./prompt.md # Context from multiple sources merged together riff -c ./defaults.json -c ./overrides.env -c debug=true ./prompt.md -``` - -parser/ core Rust library, internal to repo, not published wasm-js/ wasm wasm-bindgen module for -JS/TS runtimes wasm-wasi/ wasm WASI module for polyglot SDK consumption slash-js-web/ sdk Browser -runtime SDK (depends on wasm-js) slash-js-bundle/ sdk ESM server-side SDK (depends on wasm-js) -slash-py/ sdk Python SDK (depends on wasm-wasi) slash-ruby/ sdk Ruby SDK (depends on wasm-wasi) -slash-php/ sdk PHP SDK (depends on wasm-wasi) slash-elixir/ sdk Elixir SDK (depends on wasm-wasi) -slash-ocaml/ sdk OCaml SDK (depends on wasm-wasi) slash-haskell/ sdk Haskell SDK (depends on -wasm-wasi) slash-dart/ sdk Dart SDK (depends on wasm-wasi) slash-java/ sdk Java SDK (depends on -wasm-wasi) slash-go/ sdk Go SDK (depends on wasm-wasi) slash-zig/ sdk Zig SDK (native FFI or -wasm-wasi) slash-rs/ sdk Rust SDK, thin published crate wrapping parser riff/ cli CLI binary -(depends on slash-rs) riff-deb/ pkg Debian package riff-rpm/ pkg RPM package riff-oci/ pkg OCI -container image riff-proto/ pkg Proto toolchain plugin website/ docs Static site for documentation -and promotion docs/ docs ADRs and formal spec (bundled into website) +``` \ No newline at end of file diff --git a/docs/slash-parser-spec-v0.3.0.md b/docs/slash-parser-spec-v0.3.0.md new file mode 100644 index 0000000..413c3be --- /dev/null +++ b/docs/slash-parser-spec-v0.3.0.md @@ -0,0 +1,1096 @@ +# Slash Command Parser Specification + +Version: 0.3.0 Date: 2026-03-18 + +## 1. Overview + +The Slash Command Parser consumes a single UTF-8 text input and produces a structured JSON result +containing all detected slash commands, interleaved text blocks, optional caller-supplied context, +and any parser warnings. + +The parser is deterministic and pure: given the same input and context, it always produces the same +output. It performs no I/O, maintains no global mutable state, and accepts all configuration through +its input parameters. + +### 1.1 Design Principles + +1. **Command-agnostic.** The parser never interprets argument content. It determines argument + boundaries and transport mode but treats all argument strings as opaque byte sequences. + Tokenization, quoting, key-value parsing, and any other semantic interpretation of arguments are + the sole responsibility of the command implementation. + +2. **Flat output, no AST.** The parser emits a flat list of commands and text blocks in document + order. There is no intermediate abstract syntax tree. + +3. **Incremental emission.** A conforming implementation must be capable of finalizing and emitting + each command or text block as soon as its last line has been consumed, without buffering the + entire document. Memory usage should be bounded by the size of the largest single command payload + or text block, not the total input size. + +4. **Single forward pass.** The parser processes the input in a single left-to-right pass (after + normalization and line joining). It never backtracks. + +## 2. Input Model + +### 2.1 Line Ending Normalization + +Before any other processing: + +1. Replace all `\r\n` (CRLF) sequences with `\n` (LF). +2. Replace all remaining `\r` (bare CR) characters with `\n`. + +After normalization, all line terminators are LF. The parser splits the normalized input on `\n` to +produce a sequence of physical lines. Each `\n` terminates a line and is not part of the line +content. A trailing `\n` at the end of input produces a trailing empty line. + +Literal escape sequences inside content (e.g., `\n` in a JSON string `"blah\nblah"`) are ordinary +characters, not line terminators. They are preserved verbatim in the output. + +### 2.2 Line Joining (Backslash Continuation) + +After line-ending normalization and before command parsing, the parser performs a line-joining +pre-pass. This mechanism is identical in behavior to POSIX shell backslash-newline removal. + +For each physical line, if the line ends with a backslash (`\`) character: + +1. Remove the trailing backslash. +2. Remove the line terminator (the split boundary). +3. Concatenate the remainder with the next physical line, separated by a single space. +4. Repeat: if the joined result still ends with `\`, continue joining with subsequent lines. + +Lines that do not end with `\` are left unchanged. + +The join marker is any backslash character immediately before the physical line terminator, +regardless of what precedes it. There is no requirement for a space or any other character before +the backslash. This includes lines that serve other syntactic roles, such as a closing fence line +followed by a trailing backslash (e.g., `` ``` \ ``). In all cases, a trailing `\` triggers line +joining. This matches POSIX shell behavior. + +After this pass, the parser operates on a sequence of logical lines. Each logical line maps back to +one or more physical lines from the original input. + +#### 2.2.1 Physical Line Tracking + +The parser must track the mapping from each logical line back to its original physical line range. +This mapping is used to populate `range.start_line` and `range.end_line` in the output, which always +refer to zero-based physical line numbers from the normalized input (before joining). + +#### 2.2.2 Trailing Backslash at EOF + +If the final physical line ends with a backslash and there is no subsequent line to join with, the +trailing backslash is removed and the line stands alone. This mirrors POSIX shell behavior where a +backslash-newline pair removes the newline; at EOF, the backslash simply disappears. + +Example: + +```text +/echo hello \ +``` + +Logical line: `/echo hello` + +The command parses as a single-line command with `arguments.payload` of `"hello "`. + +#### 2.2.3 Joining Examples + +Physical input (three physical lines): + +```text +/mcp call_tool read_file \ + --path src/index.ts \ + --format json +``` + +Logical line after joining: + +```text +/mcp call_tool read_file --path src/index.ts --format json +``` + +This logical line maps to physical lines 0 through 2. + +A line ending with a literal backslash that is not intended as a join marker should use a fenced +block instead (see Section 5.2). + +### 2.3 Fence Immunity + +Line joining does not apply inside fenced blocks. Once the parser enters fence mode (see Section +5.2), all physical lines are consumed verbatim until the closing fence or EOF. A trailing `\` inside +a fence is literal content, not a join marker. + +Because line joining is conceptually a pre-pass and fences are detected during command parsing, the +implementation must either (a) perform joining lazily during parsing, skipping lines inside fences, +or (b) use a two-pass approach where fence boundaries are identified first. Either strategy is +acceptable as long as fence content is never subject to line joining. + +## 3. Command Detection + +A command line is any logical line (after normalization and joining) whose first non-whitespace +character is `/` (U+002F) and whose subsequent characters form a valid command name. + +### 3.1 Command Name + +The command name: + +- Starts immediately after the leading `/` with no intervening space. +- Ends at the first whitespace character or end-of-line. +- Must match the pattern `[a-z][a-z0-9-]*`: + - Begins with a lowercase ASCII letter (a-z). + - Followed by zero or more lowercase ASCII letters, ASCII digits, or hyphens. + +### 3.2 Invalid Slash Lines + +If a logical line's first non-whitespace character is `/` but the text after `/` does not match the +command name pattern (for example, a bare `/`, `/123`, `/Hello`, or `/ space`), the line is not a +command. In `idle` state, such lines are treated as ordinary text and may become part of a text +block (Section 6). In `in_fence` state, all lines (including invalid slash lines) are literal +payload content. + +### 3.3 Arguments + +Everything after the first whitespace following the command name is the arguments portion. The +whitespace between the command name and the arguments is consumed as a separator and is not included +in the arguments string. + +The arguments portion may be: + +- Empty (command with no arguments). +- Inline text (single-line mode, Section 5.1). +- A fence opener (fence mode, Section 5.2). +- Inline text followed by a fence opener (the text before the fence becomes the `header`, the fenced + content becomes the `payload`). + +### 3.4 The `header` Field + +The `header` field contains the inline, non-fenced argument portion of the command line. It serves +as the dispatch or routing segment of the command and is present in both argument modes: + +- In single-line mode, `header` and `payload` contain the same string (the full arguments text). +- In fence mode, `header` contains the arguments text that appears before the fence opener on the + command line. This is typically used for subcommand names and flags, while the fenced `payload` + carries bulk data such as JSON or code. + +## 4. Parser States + +The parser uses two states: + +- `idle`: not inside a fenced block. The parser scans logical lines for commands and collects + non-command lines into text blocks. +- `in_fence`: collecting raw physical lines inside a fenced block for the current command. + +There is no "accumulating" or "continuation" state. Multi-physical-line commands that are not fenced +are handled entirely by line joining (Section 2.2) before the state machine runs. + +### 4.1 State Transitions + +| Current State | Condition | Action | Next State | +| ------------- | ---------------------------------------------------- | --------------------------------------------- | ---------- | +| `idle` | Logical line is a valid command with no fence opener | Finalize as single-line command | `idle` | +| `idle` | Logical line is a valid command with a fence opener | Begin fence; record header and fence metadata | `in_fence` | +| `idle` | Logical line is not a command | Append to current text block | `idle` | +| `in_fence` | Physical line is a closing fence | Finalize fenced command | `idle` | +| `in_fence` | Physical line is not a closing fence | Append to payload | `in_fence` | +| `in_fence` | EOF reached without closing fence | Finalize command with warning | `idle` | + +## 5. Argument Modes + +### 5.1 Single-Line Mode + +If a command's arguments portion (after joining) does not contain a fence opener: + +- The full arguments text is stored in both `header` and `payload`. +- `mode` is `"single-line"`. +- `fence_lang` is `null`. +- The command is finalized on that logical line. + +Example: + +```text +/deploy production --region us-west-2 +``` + +Result: + +- `name`: `"deploy"` +- `arguments.header`: `"production --region us-west-2"` +- `arguments.mode`: `"single-line"` +- `arguments.fence_lang`: `null` +- `arguments.payload`: `"production --region us-west-2"` + +### 5.2 Fence Mode + +Fenced blocks allow attaching a raw, multi-line payload to a command using markdown-style code fence +syntax. Fenced payloads are completely verbatim: no parsing rules (including line joining) apply to +content lines, eliminating escaping concerns. + +Only backtick (`` ` ``) fences are recognized. Tilde (`~`) fences are not supported. + +#### 5.2.1 Fence Opener + +In the arguments portion of a command line, the first occurrence of three or more consecutive +backtick characters is treated as a fence opener. + +- Text before the backtick run (trimmed of trailing whitespace) becomes `arguments.header`. +- The backtick run length is recorded as the fence marker length. +- Text after the backtick run (trimmed of leading whitespace), if non-empty and consisting of a + single token (no internal whitespace), is the optional language identifier stored in + `arguments.fence_lang`. +- The parser transitions to `in_fence`. + +The variable-length backtick fence (three or more) means content containing triple backticks can be +fenced with four or more backticks, avoiding collision. + +#### 5.2.2 Fence Body + +While in `in_fence`, the parser reads physical lines from the normalized input (not joined logical +lines). All lines are appended to the payload verbatim, preserving their original content including +any trailing backslashes. + +Lines are joined in the payload with `\n` separators. The payload does not include a trailing `\n` +after the last content line. + +#### 5.2.3 Fence Lifetime + +A fenced argument block extends from the fence opener line through either: + +- The first closing fence line, or +- End of input (EOF), in which case the fence is considered unclosed and a warning is emitted (see + Section 5.2.5). + +There is no other mechanism that terminates a fence. Inside a fence, command triggers, invalid slash +lines, blank lines, and all other content are literal payload. + +#### 5.2.4 Fence Closer + +A physical line is a closing fence if, after trimming leading and trailing whitespace: + +- It consists solely of backtick characters. +- The number of backticks is greater than or equal to the opener's backtick count. + +The closing fence line is not included in the payload. Once found, the parser finalizes the command +and returns to `idle`. + +Note that line joining still applies to lines outside fences. If a closing fence line ends with a +trailing backslash (e.g., `` ``` \ ``), the fence closes normally, and then the backslash triggers a +line join between the closing fence and the next physical line. The joined result is outside the +fenced command and is parsed normally (see Appendix A.8 for an example). + +#### 5.2.5 Unclosed Fence + +If the input ends before a closing fence is found: + +- The parser finalizes the command with whatever payload has been accumulated through EOF. +- A warning object with `"type": "unclosed_fence"` and `"start_line"` set to the fence opener's + physical line number is added to the `warnings` array in the result envelope. + +The parser emits the command with its partial payload. Consumers that require strictly well-formed +fenced payloads SHOULD treat `unclosed_fence` warnings as hard errors and reject the affected +command. + +#### 5.2.6 Backslash Joining Around Fences + +Backslash line joining applies to physical lines outside of fences. This means joining can merge +lines before the fence opener and lines after the fence closer, but never lines inside the fence +body (see Section 2.3). + +Joining into a fence opener. When backslash joining merges a command line with a line containing a +fence opener, the fence is detected in the resulting logical line. This is the natural way to place +a fence opener on a separate physical line from the command name. + +Example (four physical lines): + +/mcp call_tool write_file \ + +```json +{ "path": "foo" } +``` + +After joining, physical lines 0 and 1 become one logical line: + +/mcp call_tool write_file ```json + +The backtick run is detected as a fence opener. arguments.header is "call_tool write_file", +arguments.fence_lang is "json", and the JSON body is collected as the fenced payload. + +Joining after a fence closer. Once the closing fence is found and the parser returns to idle, any +subsequent physical lines are again subject to normal line joining. If lines following the closing +fence end with trailing backslashes, they join with each other as usual. + +Example (six physical lines): + +/mcp call_tool write_file -c\ + +```json +{ "path": "foo" } +``` + +\ +production + + Lines 0 and 1 join into: /mcp call_tool write_file -c ```json + + Lines 2 and 3 are inside and closing the fence (no joining applies). + + Lines 4 and 5 join into: production (the backslash on line 4 is removed and line 5 is + appended). + +The fenced command has header of "call_tool write_file -c", fence_lang of "json", and payload of "{ +\"path\": \"foo\" }". The joined production line is outside the command and becomes a text block. +See Appendix A.9 for the full output. + +## 6. Text Blocks + +Non-command logical lines encountered in `idle` state are collected into text blocks. + +- Consecutive non-command lines form a single text block. +- Blank lines that are part of a text region are included in the text block content. +- Text block content preserves the original lines joined with `\n` separators. +- Text blocks use physical line numbers for their `range`. +- A new text block begins after a command is finalized, if non-command lines follow. + +Text blocks exist so that: + +- The roundtrip invariant (Section 9.4) can be satisfied: a formatter needs to know where + non-command regions are to reconstruct the input. +- Tooling can distinguish "what will run" from "what is commentary" when rendering or analyzing + documents that mix commands with prose. + +Consumers that do not need text blocks may ignore the `text_blocks` array. + +## 7. Multiple Commands and Ordering + +The parser walks the input line by line: + +1. In `idle`, when it encounters a command line, it starts a new command according to the argument + mode rules (Section 5). +2. After a command is finalized, the parser returns to `idle` and continues scanning. +3. Non-command lines in `idle` are collected into text blocks (Section 6). + +Commands are assigned sequential zero-based IDs in encounter order: `cmd-0`, `cmd-1`, `cmd-2`, and +so on. Text blocks are assigned IDs independently in the same manner: `text-0`, `text-1`, `text-2`, +and so on. + +Commands and text blocks appear in the output arrays in document order. + +## 8. Output Format + +### 8.1 Envelope + +A parser run produces a single JSON object (the "envelope") conforming to the `SlashParserResult` +schema defined in Section 8.5. The envelope contains: + +- `version`: the spec version (`"0.3.0"`). +- `context`: caller-supplied metadata passed through to the output unchanged. +- `commands`: an ordered array of all detected commands. +- `text_blocks`: an ordered array of all non-command text regions. +- `warnings`: an array of parser warnings (e.g., unclosed fences). Empty if no warnings. + +The parser is a total function: it always produces a valid envelope for any input. There is no input +that causes the parser to fail or return an error instead of an envelope. An empty input produces an +envelope with empty `commands`, `text_blocks`, and `warnings` arrays. Malformed constructs (such as +unclosed fences) produce commands with partial data and corresponding entries in the `warnings` +array, never a parse failure. + +### 8.2 The `raw` Field + +The `raw` field on each command contains the exact source text for that command as it appeared in +the normalized input (after line-ending normalization but before line joining). For single-line +commands that span multiple physical lines via backslash joining, `raw` contains all the physical +lines including the backslash characters and the `\n` separators between them. For fenced commands, +`raw` includes the command line, all fence body lines, and the closing fence line (if present). + +### 8.3 JSONL Streaming Mode + +When a parser implementation supports JSON Lines (JSONL) output, each line of the JSONL stream is +exactly one complete `SlashParserResult` envelope, encoded as a single JSON object on one line. + +JSONL mode does not emit one command per line. Commands remain elements of the `commands` array +inside each envelope. In a pipeline processing multiple input files, each input produces one JSONL +line. + +This design was chosen because: + +- Each JSONL line is a self-contained parse result with commands, text blocks, warnings, and + context, making each line independently meaningful. +- Command IDs (`cmd-0`, `cmd-1`) are scoped per envelope, so per-envelope lines maintain stable ID + semantics. +- Consumers can treat each line as a complete unit of work corresponding to one input document. + +Per-command JSONL (one command per line) is out of scope for this specification. Implementations may +offer it as a non-standard extension. + +### 8.4 Envelope Example + +```json +{ + "version": "0.3.0", + "context": { "source": "deploy.md", "user": "tom" }, + "commands": [ + { + "id": "cmd-0", + "name": "deploy", + "raw": "/deploy production \\\n --region us-west-2 \\\n --canary", + "range": { "start_line": 0, "end_line": 2 }, + "arguments": { + "header": "production --region us-west-2 --canary", + "mode": "single-line", + "fence_lang": null, + "payload": "production --region us-west-2 --canary" + } + } + ], + "text_blocks": [ + { "id": "text-0", "range": { "start_line": 3, "end_line": 3 }, "content": "Deploy initiated." } + ], + "warnings": [] +} +``` + +### 8.5 JSON Schema + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://slash-parser.dev/schema/v0.3.0/parse-result.json", + "title": "SlashParserResult", + "description": "Output envelope for the Slash Command Parser v0.3.0.", + "type": "object", + "required": ["version", "context", "commands", "text_blocks", "warnings"], + "additionalProperties": false, + "properties": { + "version": { + "type": "string", + "const": "0.3.0", + "description": "Specification version that produced this result." + }, + "context": { + "type": "object", + "description": "Caller-supplied metadata passed through to the output unchanged.", + "additionalProperties": true, + "properties": { + "source": { + "type": ["string", "null"], + "description": "Identifier for the input source (e.g., filename, URI)." + }, + "timestamp": { + "type": ["string", "null"], + "description": "ISO 8601 timestamp, if provided by the caller." + }, + "user": { + "type": ["string", "null"], + "description": "User identifier, if provided by the caller." + }, + "session_id": { + "type": ["string", "null"], + "description": "Session identifier, if provided by the caller." + }, + "extra": { + "type": ["object", "null"], + "description": "Arbitrary additional metadata.", + "additionalProperties": true + } + } + }, + "commands": { + "type": "array", + "description": "Ordered array of all detected commands, in document order.", + "items": { "$ref": "#/$defs/Command" } + }, + "text_blocks": { + "type": "array", + "description": "Ordered array of non-command text regions, in document order.", + "items": { "$ref": "#/$defs/TextBlock" } + }, + "warnings": { + "type": "array", + "description": "Parser warnings (e.g., unclosed fences). Empty array if no warnings.", + "items": { "$ref": "#/$defs/Warning" } + } + }, + "$defs": { + "LineRange": { + "type": "object", + "description": "Zero-based physical line range (inclusive on both ends).", + "required": ["start_line", "end_line"], + "additionalProperties": false, + "properties": { + "start_line": { + "type": "integer", + "minimum": 0, + "description": "First physical line (zero-based) covered by this element." + }, + "end_line": { + "type": "integer", + "minimum": 0, + "description": "Last physical line (zero-based) covered by this element." + } + } + }, + "CommandArguments": { + "type": "object", + "description": "Parsed argument structure for a command. Content is opaque to the parser.", + "required": ["header", "mode", "fence_lang", "payload"], + "additionalProperties": false, + "properties": { + "header": { + "type": "string", + "description": "Inline argument text from the command line, before any fence opener. Serves as the dispatch/routing portion in both single-line and fence modes." + }, + "mode": { + "type": "string", + "enum": ["single-line", "fence"], + "description": "How the arguments were assembled." + }, + "fence_lang": { + "type": ["string", "null"], + "description": "Language identifier from the fence opener, or null if not in fence mode or no language was specified." + }, + "payload": { + "type": "string", + "description": "Assembled argument content. In single-line mode, identical to header. In fence mode, the verbatim content between the fence delimiters." + } + } + }, + "Command": { + "type": "object", + "description": "A single parsed slash command.", + "required": ["id", "name", "raw", "range", "arguments"], + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "pattern": "^cmd-[0-9]+$", + "description": "Sequential zero-based identifier (cmd-0, cmd-1, ...)." + }, + "name": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]*$", + "description": "Command name without the leading slash." + }, + "raw": { + "type": "string", + "description": "Exact source text from the normalized input (before line joining), including backslashes and physical newlines for joined commands, and fence delimiters for fenced commands." + }, + "range": { + "$ref": "#/$defs/LineRange", + "description": "Physical line range in the normalized input covered by this command." + }, + "arguments": { "$ref": "#/$defs/CommandArguments" } + } + }, + "TextBlock": { + "type": "object", + "description": "A contiguous region of non-command text.", + "required": ["id", "range", "content"], + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "pattern": "^text-[0-9]+$", + "description": "Sequential zero-based identifier (text-0, text-1, ...)." + }, + "range": { + "$ref": "#/$defs/LineRange", + "description": "Physical line range covered by this text block." + }, + "content": { + "type": "string", + "description": "Original text content with lines joined by newline separators." + } + } + }, + "Warning": { + "type": "object", + "description": "A parser warning indicating a non-fatal issue.", + "required": ["type"], + "additionalProperties": true, + "properties": { + "type": { + "type": "string", + "description": "Warning category identifier (e.g., 'unclosed_fence')." + }, + "start_line": { + "type": "integer", + "minimum": 0, + "description": "Physical line number where the issue was detected." + }, + "message": { + "type": "string", + "description": "Human-readable description of the warning." + } + } + } + } +} +``` + +### 8.6 Parser Engine Interface + +The parser engine exposes two entry points. Implementations may adapt the signatures to their host +language, but the semantics must be preserved. + +1. **Parse with default context.** Accepts a single UTF-8 string input. Returns a + `SlashParserResult` envelope with an empty `context` object. + +Logical signature: + + parse(input: string) → SlashParserResult + +2. **Parse with caller-supplied context.** Accepts a UTF-8 string input and a context object. The + context is passed through to the envelope unchanged. The parser does not interpret context + fields. + +Logical signature: + + parse(input: string, context: object) → SlashParserResult + +Both entry points are pure functions: no I/O, no global mutable state, no side effects. All +configuration is supplied through the input parameters. + +The return type is always `SlashParserResult` as defined in Section 8.5. Implementations must not +use exceptions, panics, or error return channels for parse outcomes. Warnings about malformed input +(such as unclosed fences) are reported inside the envelope's `warnings` array. + +## 9. Conformance + +### 9.1 No AST Requirement + +A conforming implementation must not expose or require an abstract syntax tree as a public artifact. +The only standardized observable output is the JSON envelope defined in Section 8. Implementations +may use any internal data structures, but the contract with consumers is the envelope schema alone. + +### 9.2 Incremental Emission + +A conforming implementation must support finalizing each command or text block as soon as its last +physical line has been consumed. Memory usage must be bounded by the size of the largest single +command payload or text block, not the total size of the input. + +### 9.3 Opaque Payload + +The parser is command-agnostic. It never interprets argument content, does not tokenize, and does +not apply quoting, escaping, or key-value semantics to the `header` or `payload` fields. All +argument strings are treated as opaque byte sequences. Any higher-level parsing (e.g., splitting on +spaces, parsing JSON or YAML) is the sole responsibility of the command implementation. + +### 9.4 Roundtrip Fidelity Invariant + +Let `P` be a conforming parser and `F` be a formatter that generates a canonical plaintext +representation from `P`'s JSON envelope output. The following invariant must hold: + +For any valid input `I`, parsing, formatting, and parsing again yields a structurally equivalent +result: + +``` +P(F(P(I))) ≡ P(I) +``` + +Structural equivalence means the same sequence of commands and text blocks with identical values for +`name`, `arguments.mode`, `arguments.header`, `arguments.payload`, `arguments.fence_lang`, and text +block `content`. + +The invariant is structural, not byte-for-byte: `raw` values, line numbers, and whitespace details +may differ between the two parse results due to normalization and line joining. These are allowed +lossy transformations. + +### 9.5 Determinism + +A conforming implementation must be deterministic: given identical input bytes and identical +context, the output envelope must be byte-for-byte identical across invocations and across +implementations. + +### 9.6 JSONL Envelope Semantics + +In JSONL streaming mode, each line of the output stream is exactly one `SlashParserResult` envelope +encoded as a single JSON object. One input unit (e.g., one file or one string passed to the parse +function) produces exactly one JSONL line. + +The `commands` and `text_blocks` arrays inside each envelope are ordered by document position. +Command IDs (`cmd-0`, `cmd-1`, ...) and text block IDs (`text-0`, `text-1`, ...) are scoped to their +envelope and reset for each input unit. + +Implementations that process multiple inputs in a pipeline emit one JSONL line per input, in the +order the inputs were supplied. The parser itself is not aware of the pipeline; the JSONL framing is +the responsibility of the host runtime or CLI layer. + +## Appendix A: Parsing Examples + +### A.1 Single-Line Command + +Input: + +```text +/echo hello world +``` + +Output (commands array element): + +```json +{ + "id": "cmd-0", + "name": "echo", + "raw": "/echo hello world", + "range": { "start_line": 0, "end_line": 0 }, + "arguments": { + "header": "hello world", + "mode": "single-line", + "fence_lang": null, + "payload": "hello world" + } +} +``` + +### A.2 Joined Multi-Line Command + +Input (three physical lines): + +```text +/deploy production \ + --region us-west-2 \ + --canary +``` + +After joining: `/deploy production --region us-west-2 --canary` + +Output: + +```json +{ + "id": "cmd-0", + "name": "deploy", + "raw": "/deploy production \\\n --region us-west-2 \\\n --canary", + "range": { "start_line": 0, "end_line": 2 }, + "arguments": { + "header": "production --region us-west-2 --canary", + "mode": "single-line", + "fence_lang": null, + "payload": "production --region us-west-2 --canary" + } +} +``` + +### A.3 Fenced Command with Header + +Input: + +````text +/mcp call_tool write_file ```json +{ "path": "/src/index.ts" } +``` +```` + +Output: + +````json +{ + "id": "cmd-0", + "name": "mcp", + "raw": "/mcp call_tool write_file ```json\n{ \"path\": \"/src/index.ts\" }\n```", + "range": { "start_line": 0, "end_line": 2 }, + "arguments": { + "header": "call_tool write_file", + "mode": "fence", + "fence_lang": "json", + "payload": "{ \"path\": \"/src/index.ts\" }" + } +} +```` + +### A.4 Backslash Join into Fence + +Input (four physical lines): + +````text +/mcp call_tool write_file \ +```json +{ "path": "foo" } +``` +```` + +After joining, physical lines 0 and 1 become: ``/mcp call_tool write_file ```json`` + +Output: + +````json +{ + "id": "cmd-0", + "name": "mcp", + "raw": "/mcp call_tool write_file \\\n```json\n{ \"path\": \"foo\" }\n```", + "range": { "start_line": 0, "end_line": 3 }, + "arguments": { + "header": "call_tool write_file", + "mode": "fence", + "fence_lang": "json", + "payload": "{ \"path\": \"foo\" }" + } +} +```` + +### A.5 Text Blocks and Multiple Commands + +Input: + +```text +Welcome to the deployment system. + +/deploy staging +/notify team --channel ops +Deployment complete. +``` + +Output: + +```json +{ + "version": "0.3.0", + "context": {}, + "commands": [ + { + "id": "cmd-0", + "name": "deploy", + "raw": "/deploy staging", + "range": { "start_line": 2, "end_line": 2 }, + "arguments": { + "header": "staging", + "mode": "single-line", + "fence_lang": null, + "payload": "staging" + } + }, + { + "id": "cmd-1", + "name": "notify", + "raw": "/notify team --channel ops", + "range": { "start_line": 3, "end_line": 3 }, + "arguments": { + "header": "team --channel ops", + "mode": "single-line", + "fence_lang": null, + "payload": "team --channel ops" + } + } + ], + "text_blocks": [ + { + "id": "text-0", + "range": { "start_line": 0, "end_line": 1 }, + "content": "Welcome to the deployment system.\n" + }, + { + "id": "text-1", + "range": { "start_line": 4, "end_line": 4 }, + "content": "Deployment complete." + } + ], + "warnings": [] +} +``` + +### A.6 Invalid Slash Lines + +Input: + +```text +/123 not a command +/ bare slash +/Hello capitalized +/deploy staging +``` + +Output: the first three lines form `text-0`; `/deploy staging` is `cmd-0`. + +### A.7 Unclosed Fence + +Input: + +````text +/mcp call_tool ```json +{ "incomplete": true } +```` + +Output: the command is finalized with the accumulated payload, and a warning is emitted: + +````json +{ + "version": "0.3.0", + "context": {}, + "commands": [ + { + "id": "cmd-0", + "name": "mcp", + "raw": "/mcp call_tool ```json\n{ \"incomplete\": true }", + "range": { "start_line": 0, "end_line": 1 }, + "arguments": { + "header": "call_tool", + "mode": "fence", + "fence_lang": "json", + "payload": "{ \"incomplete\": true }" + } + } + ], + "text_blocks": [], + "warnings": [ + { + "type": "unclosed_fence", + "start_line": 0, + "message": "Fenced block opened at line 0 was never closed." + } + ] +} +```` + +### A.8 Closing Fence with Trailing Backslash + +A closing fence line that ends with a trailing backslash closes the fence normally, then the +backslash triggers a line join with the next physical line. The joined content is outside the fenced +command. + +Input (six physical lines): + +````text +/mcp call_tool write_file -c \ +```json +{ "path": "foo" } +``` \ +\ +production +```` + +Physical lines: + +- 0: `/mcp call_tool write_file -c \` +- 1: `` ```json `` +- 2: `{ "path": "foo" }` +- 3: `` ``` \ `` +- 4: `\` +- 5: `production` + +Line joining (outside fences): + +- Lines 0 and 1 join: ``/mcp call_tool write_file -c ```json`` +- Lines 2 is inside the fence (verbatim, no joining). +- Line 3 is the closing fence (backticks only after trim? No: it contains `` ``` \ ``, which after + trimming whitespace is `` ```\ `` or `` ``` \ ``. The backslash means this is not solely + backticks, so it is not a valid closing fence.) + +Because line 3 is not a valid closing fence, the fence never closes. All remaining lines (2 through +5) become part of the fenced payload, and the parser emits an `unclosed_fence` warning. + +Output: + +````json +{ + "version": "0.3.0", + "context": {}, + "commands": [ + { + "id": "cmd-0", + "name": "mcp", + "raw": "/mcp call_tool write_file -c \\\n```json\n{ \"path\": \"foo\" }\n``` \\\n\\\nproduction", + "range": { "start_line": 0, "end_line": 5 }, + "arguments": { + "header": "call_tool write_file -c", + "mode": "fence", + "fence_lang": "json", + "payload": "{ \"path\": \"foo\" }\n``` \\\n\\\nproduction" + } + } + ], + "text_blocks": [], + "warnings": [ + { + "type": "unclosed_fence", + "start_line": 1, + "message": "Fenced block opened at line 1 was never closed." + } + ] +} +```` + +To avoid this, write the closing fence on its own line without a trailing backslash: + +````text +/mcp call_tool write_file -c \ +```json +{ "path": "foo" } +``` +```` + +### A.9 Proper Fence Close Followed by Additional Content + +Input (six physical lines): + +````text +/mcp call_tool write_file -c\ +```json +{ "path": "foo" } +``` +\ +production +```` + +Physical lines: + +- 0: `/mcp call_tool write_file -c\` +- 1: `` ```json `` +- 2: `{ "path": "foo" }` +- 3: `` ``` `` +- 4: `\` +- 5: `production` + +Line joining (outside fences): + +- Lines 0 and 1 join: ``/mcp call_tool write_file -c ```json`` +- Line 2 is inside the fence (verbatim). +- Line 3 is a valid closing fence (solely backticks after trim). Fence closes. Parser returns to + `idle`. +- Lines 4 and 5 join: `production` + +Output: + +````json +{ + "version": "0.3.0", + "context": {}, + "commands": [ + { + "id": "cmd-0", + "name": "mcp", + "raw": "/mcp call_tool write_file -c\\\n```json\n{ \"path\": \"foo\" }\n```", + "range": { "start_line": 0, "end_line": 3 }, + "arguments": { + "header": "call_tool write_file -c", + "mode": "fence", + "fence_lang": "json", + "payload": "{ \"path\": \"foo\" }" + } + } + ], + "text_blocks": [ + { "id": "text-0", "range": { "start_line": 4, "end_line": 5 }, "content": " production" } + ], + "warnings": [] +} +```` + +## Appendix B: Change Log + +Changes from version 0.2.0: + +- Replaced continuation markers (`" /"`) with POSIX-style backslash line joining, reducing the state + machine from three states to two. +- Added explicit treatment of invalid slash lines in idle state (Section 3.2). +- Defined fence lifetime as "until closing fence or EOF" (Section 5.2.3). +- Formalized trailing backslash at EOF behavior (Section 2.2.2). +- Added backslash-join-then-fence example (Section 5.2.6). +- Added closing-fence-with-backslash examples (Appendix A.8, A.9). +- Stated opaque payload principle as a conformance requirement (Section 9.3). +- Stated roundtrip fidelity invariant as a testable property (Section 9.4). +- Stated incremental emission as a conformance constraint (Section 9.2). +- Committed to no-AST as a conformance requirement (Section 9.1). +- Added determinism requirement (Section 9.5). +- Clarified JSONL streaming semantics with rationale (Section 8.3). +- Documented `header` as the dispatch portion available in both modes (Section 3.4). +- Introduced `warnings` array for non-fatal issues (Section 5.2.5). +- Removed unused `children` field from Command schema. +- Only backtick fences are supported; tilde fences are explicitly excluded (Section 5.2). +- Version bumped to 0.3.0. diff --git a/docs/slash-parser.md b/docs/slash-parser.md deleted file mode 100644 index 833dec6..0000000 --- a/docs/slash-parser.md +++ /dev/null @@ -1,333 +0,0 @@ -# small Rust+WASM “slash command parser” component with a JSON-in / JSON-out API, - -## style and patterns - -code snippets and patterns in this doc other than the parser semantics are only examples. The -language, build, and testing patterns override and should be followed. - -- toms-clean-code.md -- toms-clean-arch.md -- lang-rust.md -- testing.md -- testing-rust.md -- universal-rust-wasm-lib.md - -## 1. Component overview - -- Name: `slash_parser` -- Implementation language: Rust -- Compilation target: WebAssembly -- Binding strategy: `wasm-bindgen` for JS/TS environments.[^1][^2] -- Interface style: - - Input: UTF‑8 text (string). - - Output: UTF‑8 JSON string conforming to a provided JSON Schema. - -Responsibilities: - -1. Parse an input text buffer using the **slash-command semantics** described below. -2. Emit a JSON document describing: - - All detected commands and their arguments, - - Optional non-command text blocks, - - A context object. - -The WASM module must be deterministic and pure (no I/O, no global mutable state aside from internal -caches). - ---- - -## 2. Parsing semantics (Rust core) - -The Rust core must implement the exact language you’ve defined: - -### 2.1 Input - -- Input is a `&str` (UTF‑8 text). -- The parser processes it line-by-line, splitting on `\n`. -- `\r\n` must be normalized to `\n` before parsing. - -### 2.2 Command lines - -A **command line** is any line whose first non-whitespace character is `/`. - -Structure: - -- Leading whitespace (spaces/tabs) before `/` is ignored for detection but not included in `raw`. -- After the first `/`, parse `` using regex `[a-z][a-z0-9-]*`. -- The rest of the line, after one or more whitespace characters, is `arguments.header` prefix. - -Non-command lines: - -- Lines that do not start with `/` (ignoring leading whitespace) are considered **non-command text** - when the parser is in `idle` state. -- If a command is currently being accumulated, non-command lines are appended to that command’s - payload (depending on the state). - -### 2.3 States - -The parser is a state machine: - -- `Idle` – not inside any command. -- `Accumulating` – building `arguments.payload` for a command outside a fence. -- `InFence` – inside a fenced block attached to a command. - -State transitions are driven by: - -- New command detection, -- Continuation marker `" /"`, -- Fence openers / closers. - -### 2.4 Continuation with `" /"` - -Continuation is explicit and conservative: - -- A line continues the current command if and only if it ends with **space + slash** (`" /"`) - immediately before the newline. - -Semantics: - -- In `Accumulating` (or on the first header line of a command): - - If the line ends with `" /"`: - - Remove the trailing `" /"` from the line content. - - Append the remaining content plus `\n` to `arguments.payload`. - - Stay in `Accumulating`. - - If the line does not end with `" /"` and does not start a fence: - - Append the full line content plus `\n`. - - Finalize the command and return to `Idle`. - -Note: `" /"` must be checked literally; a line ending in `/` with no preceding space is **not** a -continuation marker. This avoids collisions with paths like `/var/log/`. - -### 2.5 Fenced block arguments - -Use markdown-style fenced code blocks:[^3][^4] - -#### Fence opener detection - -Fence openers can appear: - -1. **Inline on the command line**: - - Inside ``, find the first occurrence of three or more consecutive backticks: - ```… - ``` - - Everything before the backticks remains in `arguments.header`. - - The backticks and optional language identifier (e.g., `jsonl`) mark the start of fence mode. -2. **On the next line after continuation**: - - The command line ends with `" /"` and the parser enters `Accumulating`. - - The next line starts with the fence opener ```[lang]. - - The parser transitions to `InFence`. - -Fence metadata: - -- Record: - - Fence marker: number of backticks (typically 3). - - Optional `lang` following the backticks (up to first whitespace). - -#### Fence mode - -In `InFence`: - -- Every line is appended verbatim to `arguments.payload` with a trailing `\n`. -- `" /"` has no special meaning inside the fence. -- The parser looks for a closing fence: a line whose first non-whitespace characters are the same - number of backticks and nothing else but whitespace. - -On closing fence: - -- The closing fence line is not included in the payload. -- Command is finalized. -- `arguments.mode = "fence"`. -- `arguments.fence_lang` is set to the captured language or `null`. -- Parser returns to `Idle`. - -### 2.6 Multiple commands - -- In `Idle`, when a command line appears, start a new command. -- When a command is finalized, append it to the `commands` array and go back to `Idle`. -- Non-command text between commands may be recorded as `text_blocks`. - -You can treat lines that fall entirely outside commands as text blocks grouped by contiguous ranges. - ---- - -## 3. JSON output format - -The Rust core will produce a JSON value of type `SlashParseResult`: - -```ts -interface SlashParseResult { - version: string // e.g., "0.1.0" - context: { - source?: string - timestamp?: string // ISO 8601, if provided by caller - user?: string - session_id?: string - extra?: Record - [key: string]: unknown - } - commands: Command[] - text_blocks?: TextBlock[] -} - -interface CommandRange { - start_line: number // 0-based or 1-based (document in spec) - end_line: number -} - -type ArgumentMode = 'single-line' | 'continuation' | 'fence' - -interface CommandArguments { - header?: string - mode: ArgumentMode - fence_lang?: string | null - payload: string -} - -interface Command { - id: string - name: string - raw?: string - range: CommandRange - arguments: CommandArguments - children?: Command[] // reserved -} - -interface TextBlock { - id: string - range: CommandRange - content: string -} -``` - -The JSON Schema in your previous step applies; the Rust types should be defined to match that -schema, using `serde` for serialization.[^5][^6][^7] - ---- - -## 4. WASM API design - -### 4.1 Rust public API (pre-wasm) - -Define a core Rust function: - -```rust -pub struct ParserContext { - pub source: Option, - pub timestamp: Option, - pub user: Option, - pub session_id: Option, - pub extra: Option, -} - -pub fn parse_slash_commands( - input: &str, - context: ParserContext, -) -> Result; -``` - -- `ParseError` should include at least a message and optionally a line/column. - -### 4.2 wasm-bindgen exports - -Use `wasm-bindgen` to expose a JS-friendly API:[^8][^2][^1] - -- Module name: `slash_parser`. -- Exported functions: - -```rust -use serde::{Deserialize, Serialize}; -use wasm_bindgen::prelude::*; - -#[wasm_bindgen] -pub fn parse_text(input: &str) -> Result; - -#[wasm_bindgen] -pub fn parse_text_with_context(input: &str, context: &JsValue) -> Result; -``` - -Semantics: - -- `parse_text`: - - Uses a default `ParserContext { source: None, ... }`. - - Returns a `JsValue` containing a JSON object conforming to `SlashParseResult`, via - `JsValue::from_serde(&result)`. -- `parse_text_with_context`: - - `context` is optional; if provided, it is interpreted as a JS object containing any of: - - `source`, `timestamp`, `user`, `session_id`, `extra`. - - Rust side should `from_serde` the context into `ParserContext` (or build it manually from - `JsValue`).[^9] - - Returns the same JSON structure as `parse_text`. - -On error: - -- Return a `JsValue` representing a JS `Error` or a structured - `{ error: string, line?: number, column?: number }`. - -### 4.3 JS / TS usage expectations - -Consumers (e.g. a Node-based or browser-based tool) will: - -```ts -import init, { parse_text, parse_text_with_context } from 'slash_parser' - -await init() // or equivalent wasm-bindgen init - -const result = parse_text_with_context(sourceText, { source: 'file://path/to/doc.md', user: 'tom' }) - -// `result` is a JS object matching SlashParseResult -console.log(result.commands) -``` - -The bindings should be compatible with bundlers (Vite, Webpack) and Node 18+. - ---- - -## 5. Non-goals / constraints - -To keep Perplexity Computer’s scope tight: - -- No I/O: the parser does not read files or network; it only parses the provided string. -- No global config: all configuration is via parameters or context. -- No incremental parsing: first version can parse whole strings only. -- No reliance on external crates beyond: - - `serde`, `serde_json`, - - `wasm-bindgen` (and its minimal dependencies). - -[^1]: https://rustwasm.github.io/docs/wasm-bindgen/ - -[^2]: https://rustwasm.github.io/docs/wasm-bindgen/print.html - -[^3]: https://www.markdownguide.org/extended-syntax/ - -[^4]: https://python-markdown.github.io/extensions/fenced_code_blocks/ - -[^5]: https://json-schema.org/learn/json-schema-examples - -[^6]: https://json-schema.org/understanding-json-schema/basics - -[^7]: https://blog.promptlayer.com/how-json-schema-works-for-structured-outputs-and-tool-integration/ - -[^8]: https://rustwasm.github.io/docs/wasm-bindgen/contributing/design/index.html - -[^9]: https://users.rust-lang.org/t/how-do-i-sent-a-js-object-to-rust-through-wasm/80007 - -[^10]: https://www.reddit.com/r/rust/comments/15sjuyo/interfacing_complex_types_in_wasm/ - -[^11]: https://github.com/rustwasm/wasm-bindgen/discussions/2883 - -[^12]: https://stackoverflow.com/questions/65242336/js-binding-for-large-rust-object-using-wasm-bindgen - -[^13]: https://nickb.dev/blog/recommendations-when-publishing-a-wasm-library/ - -[^14]: https://www.speakeasy.com/blog/building-speakeasy-openapi-go-library - -[^15]: https://github.com/douglance/valrs - -[^16]: https://hacks.mozilla.org/2019/08/webassembly-interface-types/ - -[^17]: https://github.com/WebAssembly/design/blob/main/Rationale.md - -[^18]: https://docs.rs/valrs-wasm - -[^19]: https://gov.near.org/t/proposal-use-webassembly-interface-types-to-describe-all-standards-interfaces-and-application-contract-interface-aci/23256 - -[^20]: https://developer.mozilla.org/en-US/docs/WebAssembly/Guides/Concepts From 597eb02829e3a2c9d0122ca8d2068d3f3df39d9f Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Thu, 19 Mar 2026 05:16:49 -0600 Subject: [PATCH 04/13] feat(parser): update domain layer to parser spec 0.3.0 --- parser/src/domain/errors.rs | 7 ++++--- parser/src/domain/mod.rs | 6 ++---- parser/src/domain/types.rs | 28 ++++++---------------------- 3 files changed, 12 insertions(+), 29 deletions(-) diff --git a/parser/src/domain/errors.rs b/parser/src/domain/errors.rs index 23b1e8a..8654a6b 100644 --- a/parser/src/domain/errors.rs +++ b/parser/src/domain/errors.rs @@ -3,7 +3,8 @@ /// Warnings are collected in `ParseResult.warnings` rather than /// causing the parse to fail. The parser is intentionally permissive. #[derive(Debug, Clone, PartialEq, Eq)] -pub enum ParseWarning { - UnclosedFence { start_line: usize }, - UnclosedContinuation { start_line: usize }, +pub struct Warning { + pub wtype: String, + pub start_line: Option, + pub message: Option, } diff --git a/parser/src/domain/mod.rs b/parser/src/domain/mod.rs index ef66902..0453bad 100644 --- a/parser/src/domain/mod.rs +++ b/parser/src/domain/mod.rs @@ -1,9 +1,7 @@ mod errors; mod types; -pub use errors::ParseWarning; +pub use errors::Warning; pub use types::{ - ArgumentMode, Command, CommandArguments, LineRange, ParseResult, ParserContext, TextBlock, + ArgumentMode, Command, CommandArguments, LineRange, ParseResult, TextBlock, SPEC_VERSION, }; - -pub use types::SPEC_VERSION; diff --git a/parser/src/domain/types.rs b/parser/src/domain/types.rs index d07d448..3868589 100644 --- a/parser/src/domain/types.rs +++ b/parser/src/domain/types.rs @@ -1,6 +1,4 @@ -use std::collections::HashMap; - -use super::errors::ParseWarning; +use super::errors::Warning; /// Inclusive line range (zero-based). #[derive(Debug, Clone, PartialEq, Eq)] @@ -13,7 +11,6 @@ pub struct LineRange { #[derive(Debug, Clone, PartialEq, Eq)] pub enum ArgumentMode { SingleLine, - Continuation, Fence, } @@ -29,6 +26,7 @@ pub struct CommandArguments { /// A single parsed slash command. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Command { + pub id: String, pub name: String, pub raw: String, pub range: LineRange, @@ -38,32 +36,18 @@ pub struct Command { /// A contiguous block of non-command text. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TextBlock { + pub id: String, pub range: LineRange, pub content: String, } -/// Metadata provided by the caller, merged into the output envelope. -/// -/// `extra` holds additional key-value pairs beyond the known fields. -/// Values are strings; richer types are a serialization concern -/// handled at the application boundary. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct ParserContext { - pub source: Option, - pub timestamp: Option, - pub user: Option, - pub session_id: Option, - pub extra: HashMap, -} - /// Top-level parse result. #[derive(Debug, Clone, PartialEq)] pub struct ParseResult { pub version: String, - pub context: ParserContext, pub commands: Vec, - pub text_blocks: Vec, - pub warnings: Vec, + pub textblocks: Vec, + pub warnings: Vec, } -pub const SPEC_VERSION: &str = "v1"; +pub const SPEC_VERSION: &str = "0.3.0"; From e57d3eab4baf4f52baac2cff06166d3fb99d8c8f Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Thu, 19 Mar 2026 05:21:20 -0600 Subject: [PATCH 05/13] feat(parser): partial update app layer to parser spec 0.3.0 --- .gitignore | 1 + .../application/command_accumulate.txt | 7 + .../application/tests/mod.txt | 8 + parser/src/application/command_accumulate.rs | 406 +++++++---- parser/src/application/command_finalize.rs | 308 +++++--- parser/src/application/document_parse.rs | 680 ++++++++++-------- parser/src/application/line_classify.rs | 478 ++++++++---- parser/src/application/line_join.rs | 357 +++++++++ parser/src/application/mod.rs | 5 +- parser/src/application/normalize.rs | 127 ++++ parser/src/application/tests/mod.rs | 459 +++++++++++- parser/src/application/text_collect.rs | 240 ++++--- parser/src/domain/mod.rs | 5 +- parser/src/lib.rs | 10 +- 14 files changed, 2332 insertions(+), 759 deletions(-) create mode 100644 parser/proptest-regressions/application/command_accumulate.txt create mode 100644 parser/proptest-regressions/application/tests/mod.txt create mode 100644 parser/src/application/line_join.rs create mode 100644 parser/src/application/normalize.rs diff --git a/.gitignore b/.gitignore index ecc6fd3..7b057d7 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,4 @@ tmp/ # moon .moon/cache/ .moon/docker/ + diff --git a/parser/proptest-regressions/application/command_accumulate.txt b/parser/proptest-regressions/application/command_accumulate.txt new file mode 100644 index 0000000..ccfb93a --- /dev/null +++ b/parser/proptest-regressions/application/command_accumulate.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 1657315ea6a9fa6475f78c9454ab64e78fab99d9eef8792ed95cd52b7ecd3bdc # shrinks to name = "a", body_lines = [" "] diff --git a/parser/proptest-regressions/application/tests/mod.txt b/parser/proptest-regressions/application/tests/mod.txt new file mode 100644 index 0000000..647bf83 --- /dev/null +++ b/parser/proptest-regressions/application/tests/mod.txt @@ -0,0 +1,8 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc d1207456a51fa8d19d9c2072c9b707b0c1aac013c87cfd4ae9a65975cdf9227c # shrinks to body = "\r" +cc ca107577def8080df35b3c6887f98d087144422311d1bc59ca5f0c535b65b1c2 # shrinks to lines = ["a\r/a"] diff --git a/parser/src/application/command_accumulate.rs b/parser/src/application/command_accumulate.rs index d89f76f..bede54e 100644 --- a/parser/src/application/command_accumulate.rs +++ b/parser/src/application/command_accumulate.rs @@ -1,7 +1,7 @@ -use super::line_classify::{CommandHeader, LineKind, classify_line}; +use super::line_classify::CommandHeader; use crate::domain::ArgumentMode; -/// Result of offering a line to an in-progress command. +/// Result of offering a physical line to an in-progress command. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AcceptResult { Consumed, @@ -9,46 +9,64 @@ pub enum AcceptResult { Rejected, } -/// In-progress command being assembled from consecutive lines. +/// In-progress command being assembled from physical lines. +/// +/// v0.3.0: Continuation mode is no longer handled here. Multi-physical-line commands +/// that are not fenced are resolved entirely by line joining (§2.2) before this module runs. #[derive(Debug, Clone)] pub struct PendingCommand { + pub id: usize, pub name: String, - pub raw_header: String, pub header_text: String, pub mode: ArgumentMode, pub fence_lang: Option, + pub fence_backtick_count: usize, pub start_line: usize, pub end_line: usize, pub payload_lines: Vec, + pub raw_lines: Vec, pub is_open: bool, } -pub fn start_command(header: CommandHeader, line_index: usize) -> PendingCommand { - let initial_lines = if header.header_text.is_empty() { - vec![] - } else { - vec![header.header_text.clone()] - }; - +/// Begin accumulating a new command from its parsed header. +/// +/// §7: id is the caller-supplied sequential zero-based counter used to produce cmd-0, cmd-1, ... +/// §8.2: raw_lines is seeded with the header's raw text; physical lines are appended by accept_line. +pub fn start_command(header: CommandHeader, line_index: usize, id: usize) -> PendingCommand { + // §5.1: single-line commands are fully resolved on their logical line; is_open = false. + // §5.2.1: fence commands open immediately and stay open until a closer is found. let (payload_lines, is_open) = match &header.mode { - ArgumentMode::SingleLine => (initial_lines, false), - ArgumentMode::Continuation => (initial_lines, true), + ArgumentMode::SingleLine => { + let lines = if header.header_text.is_empty() { + vec![] + } else { + vec![header.header_text.clone()] + }; + (lines, false) + } ArgumentMode::Fence => (vec![], true), }; PendingCommand { + id, name: header.name, - raw_header: header.raw, header_text: header.header_text, mode: header.mode, fence_lang: header.fence_lang, + fence_backtick_count: header.fence_backtick_count, start_line: line_index, end_line: line_index, payload_lines, + raw_lines: vec![header.raw], is_open, } } +/// Offer one physical line to an open command. +/// +/// §5.2.2: fence body lines are appended to payload_lines verbatim. +/// §5.2.4: a physical line that satisfies is_fence_closer closes the fence. +/// §8.2: every offered line, including the closer, is appended to raw_lines. pub fn accept_line( cmd: PendingCommand, line_index: usize, @@ -59,37 +77,9 @@ pub fn accept_line( } match cmd.mode { - ArgumentMode::SingleLine => (cmd, AcceptResult::Rejected), - ArgumentMode::Continuation => accept_continuation(cmd, line_index, line), ArgumentMode::Fence => accept_fence(cmd, line_index, line), - } -} - -fn accept_continuation( - mut cmd: PendingCommand, - line_index: usize, - line: &str, -) -> (PendingCommand, AcceptResult) { - if line.trim().is_empty() { - cmd.is_open = false; - return (cmd, AcceptResult::Rejected); - } - - if matches!(classify_line(line), LineKind::Command(_)) { - cmd.is_open = false; - return (cmd, AcceptResult::Rejected); - } - - if line.trim_end().ends_with('\\') { - let content = line.trim_end().trim_end_matches('\\').trim_end(); - cmd.payload_lines.push(content.to_string()); - cmd.end_line = line_index; - (cmd, AcceptResult::Consumed) - } else { - cmd.payload_lines.push(line.to_string()); - cmd.end_line = line_index; - cmd.is_open = false; - (cmd, AcceptResult::Completed) + // §4: only idle and in-fence states exist in v0.3.0; all other modes are closed. + _ => (cmd, AcceptResult::Rejected), } } @@ -98,152 +88,326 @@ fn accept_fence( line_index: usize, line: &str, ) -> (PendingCommand, AcceptResult) { - if line.trim_start().starts_with("```") { - cmd.end_line = line_index; + // §8.2: raw accumulation happens unconditionally before the closer check. + cmd.raw_lines.push(line.to_string()); + cmd.end_line = line_index; + + if is_fence_closer(line, cmd.fence_backtick_count) { + // §5.2.4: the closing fence line is not included in the payload. cmd.is_open = false; (cmd, AcceptResult::Completed) } else { - cmd.end_line = line_index; cmd.payload_lines.push(line.to_string()); (cmd, AcceptResult::Consumed) } } +/// §5.2.4: A physical line is a closing fence if, after trimming leading and trailing +/// whitespace, it consists solely of backtick characters and the count is >= the opener's count. +fn is_fence_closer(line: &str, opener_count: usize) -> bool { + let trimmed = line.trim(); + !trimmed.is_empty() && trimmed.chars().all(|c| c == '`') && trimmed.len() >= opener_count +} + #[cfg(test)] mod tests { use super::*; + use crate::application::line_classify::{LineKind, classify_line}; fn header_from(line: &str) -> CommandHeader { match classify_line(line) { LineKind::Command(h) => h, - _ => panic!("expected command header"), + _ => panic!("expected command header from: {line:?}"), } } + // --- start_command --- + #[test] fn single_line_is_immediately_closed() { - let cmd = start_command(header_from("/help"), 0); + // §5.1: a single-line command is finalized on its logical line; no further lines consumed. + let cmd = start_command(header_from("/help"), 0, 0); assert!(!cmd.is_open); } #[test] - fn fence_stays_open_until_closing_ticks() { - let cmd = start_command(header_from("/code ```rust"), 0); + fn fence_opens_with_is_open_true() { + // §5.2.3: fence lifetime begins at the opener and extends until a valid closer or EOF. + let cmd = start_command(header_from("/code ```rust"), 0, 0); assert!(cmd.is_open); - - let (cmd, res) = accept_line(cmd, 1, "fn main() {}"); - assert_eq!(res, AcceptResult::Consumed); - assert!(cmd.is_open); - - let (cmd, res) = accept_line(cmd, 2, "```"); - assert_eq!(res, AcceptResult::Completed); - assert!(!cmd.is_open); - assert_eq!(cmd.payload_lines, vec!["fn main() {}"]); } #[test] - fn continuation_ends_on_line_without_backslash() { - let cmd = start_command(header_from("/cmd first \\"), 0); - assert!(cmd.is_open); + fn id_is_stored_on_pending_command() { + // §7: commands are assigned sequential zero-based IDs in encounter order (cmd-0, cmd-1, ...). + let cmd = start_command(header_from("/deploy production"), 3, 7); + assert_eq!(cmd.id, 7); + } - let (cmd, res) = accept_line(cmd, 1, "second \\"); - assert_eq!(res, AcceptResult::Consumed); + #[test] + fn raw_lines_initialized_with_opener() { + // §8.2: raw includes the command line itself; accumulation starts at start_command. + let cmd = start_command(header_from("/mcp call_tool ```json"), 0, 0); + assert_eq!(cmd.raw_lines, vec!["/mcp call_tool ```json"]); + } - let (cmd, res) = accept_line(cmd, 2, "last line"); - assert_eq!(res, AcceptResult::Completed); - assert!(!cmd.is_open); + #[test] + fn single_line_payload_initialized_from_header_text() { + // §5.1: in single-line mode header and payload contain the same string. + let cmd = start_command(header_from("/deploy production --region us-west-2"), 0, 0); + assert_eq!(cmd.payload_lines, vec!["production --region us-west-2"]); } #[test] - fn continuation_rejects_blank_line() { - let cmd = start_command(header_from("/cmd start \\"), 0); - let (cmd, res) = accept_line(cmd, 1, ""); - assert_eq!(res, AcceptResult::Rejected); - assert!(!cmd.is_open); + fn single_line_no_args_has_empty_payload() { + // §5.1 (implied): a command with no arguments produces an empty payload_lines. + let cmd = start_command(header_from("/ping"), 0, 0); + assert!(cmd.payload_lines.is_empty()); } #[test] - fn continuation_rejects_new_command() { - let cmd = start_command(header_from("/cmd start \\"), 0); - let (cmd, res) = accept_line(cmd, 1, "/other"); - assert_eq!(res, AcceptResult::Rejected); - assert!(!cmd.is_open); + fn fence_payload_lines_initially_empty() { + // §5.2.2: the fence body begins empty; content is appended as physical lines arrive. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + assert!(cmd.payload_lines.is_empty()); } #[test] - fn continuation_two_lines_produces_continuation_mode() { - let header = header_from("/mcp call_tool read_file \\"); - let cmd = start_command(header, 0); + fn fence_backtick_count_carried_from_header() { + // §5.2.1: the opener's backtick run length is recorded for use by the closer check (§5.2.4). + let cmd = start_command(header_from("/cmd ```rust"), 0, 0); + assert_eq!(cmd.fence_backtick_count, 3); + } + + // --- accept_line: fence body --- - assert_eq!(cmd.mode, ArgumentMode::Continuation); + #[test] + fn fence_body_line_is_consumed() { + // §5.2.2: physical lines that are not a closer are appended to the payload verbatim. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, "fn main() {}"); + assert_eq!(res, AcceptResult::Consumed); assert!(cmd.is_open); + assert_eq!(cmd.payload_lines, vec!["fn main() {}"]); } #[test] - fn continuation_header_strips_trailing_marker() { - let header = header_from("/mcp call_tool read_file \\"); - let cmd = start_command(header, 0); + fn fence_body_line_added_to_raw_lines() { + // §8.2: for fenced commands raw includes the command line, all body lines, and the closer. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "first"); + let (cmd, _) = accept_line(cmd, 2, "second"); + assert_eq!(cmd.raw_lines, vec!["/cmd ```", "first", "second"]); + } - assert_eq!(cmd.header_text, "call_tool read_file"); - assert_eq!(cmd.payload_lines, vec!["call_tool read_file"]); + #[test] + fn fence_body_preserves_lines_verbatim() { + // §5.2.2: content is appended verbatim; no parsing rules including line joining apply. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let line = r" leading spaces and trailing backslash\"; + let (cmd, _) = accept_line(cmd, 1, line); + assert_eq!(cmd.payload_lines, vec![line]); } #[test] - fn continuation_payload_preserves_newlines() { - let cmd = start_command(header_from("/mcp call_tool read_file \\"), 0); + fn blank_line_inside_fence_is_payload() { + // §5.2.3: inside a fence, blank lines are literal payload content. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, ""); + assert_eq!(res, AcceptResult::Consumed); + assert_eq!(cmd.payload_lines, vec![""]); + } + + #[test] + fn command_line_inside_fence_is_payload() { + // §5.2.3: inside a fence, command triggers are literal payload; no state change. + let cmd = start_command(header_from("/outer ```"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, "/inner arg"); + assert_eq!(res, AcceptResult::Consumed); + assert!(cmd.is_open); + } + + // --- accept_line: fence closer --- - let line1 = r#"{"path": "src/index.ts"}"#; - let (cmd, res) = accept_line(cmd, 1, line1); + #[test] + fn fence_closer_completes_command() { + // §5.2.4: a valid closer transitions the accumulator to closed and returns Completed. + let cmd = start_command(header_from("/cmd ```rust"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "fn main() {}"); + let (cmd, res) = accept_line(cmd, 2, "```"); assert_eq!(res, AcceptResult::Completed); assert!(!cmd.is_open); + } - assert_eq!( - cmd.payload_lines, - vec!["call_tool read_file".to_string(), line1.to_string()] - ); + #[test] + fn fence_closer_added_to_raw_lines() { + // §8.2: raw includes the closing fence line if present. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "content"); + let (cmd, _) = accept_line(cmd, 2, "```"); + assert_eq!(cmd.raw_lines, vec!["/cmd ```", "content", "```"]); + } - assert_eq!(cmd.start_line, 0); - assert_eq!(cmd.end_line, 1); + #[test] + fn fence_closer_not_in_payload() { + // §5.2.4: the closing fence line is not included in the payload. + let cmd = start_command(header_from("/cmd ```rust"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "fn main() {}"); + let (cmd, _) = accept_line(cmd, 2, "```"); + assert_eq!(cmd.payload_lines, vec!["fn main() {}"]); } #[test] - fn continuation_three_lines() { - let cmd = start_command(header_from("/cmd first \\"), 0); + fn fence_closer_with_more_backticks_than_opener() { + // §5.2.4: closer count >= opener count; four backticks close a triple-backtick fence. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, "````"); + assert_eq!(res, AcceptResult::Completed); + assert!(!cmd.is_open); + } - let (cmd, res) = accept_line(cmd, 1, "second \\"); + #[test] + fn fewer_backticks_than_opener_is_not_a_closer() { + // §5.2.4: closer count must be >= opener count; two backticks cannot close a triple fence. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, "``"); assert_eq!(res, AcceptResult::Consumed); - let (cmd, res) = accept_line(cmd, 2, "third"); + assert!(cmd.is_open); + } + + #[test] + fn line_with_backticks_and_trailing_text_is_not_a_closer() { + // §5.2.4: after trimming whitespace the line must consist solely of backtick characters. + // A line like "```rust" or "``` trailing" is body content, not a closer. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, "```rust"); + assert_eq!(res, AcceptResult::Consumed); + assert!(cmd.is_open); + } + + #[test] + fn fence_closer_with_leading_whitespace_is_valid() { + // §5.2.4: closer is checked after trimming leading and trailing whitespace. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, " ``` "); assert_eq!(res, AcceptResult::Completed); assert!(!cmd.is_open); + } - assert_eq!( - cmd.payload_lines, - vec![ - "first".to_string(), - "second".to_string(), - "third".to_string() - ] - ); + #[test] + fn closed_fence_rejects_subsequent_lines() { + // §5.2.3: once the closing fence is found the command is finalized; no further lines accepted. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "content"); + let (cmd, _) = accept_line(cmd, 2, "```"); + let (cmd, res) = accept_line(cmd, 3, "after"); + assert_eq!(res, AcceptResult::Rejected); + assert_eq!(cmd.payload_lines, vec!["content"]); } + // --- accept_line: single-line --- + #[test] - fn continuation_range_spans_all_lines() { - let cmd = start_command(header_from("/cmd first \\"), 0); + fn single_line_rejects_any_subsequent_line() { + // §5.1: single-line commands are finalized on their logical line; accept_line is a no-op. + let cmd = start_command(header_from("/help"), 0, 0); + let (cmd, res) = accept_line(cmd, 1, "anything"); + assert_eq!(res, AcceptResult::Rejected); + assert!(cmd.payload_lines == vec![""] || cmd.payload_lines.is_empty()); + } - let (cmd, _) = accept_line(cmd, 1, "second \\"); - let (cmd, _) = accept_line(cmd, 2, "third"); + // --- range tracking --- + #[test] + fn end_line_advances_with_each_accepted_line() { + // §2.2.1: range.endLine is the zero-based index of the last physical line consumed. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "line one"); + let (cmd, _) = accept_line(cmd, 2, "line two"); + let (cmd, _) = accept_line(cmd, 3, "```"); assert_eq!(cmd.start_line, 0); - assert_eq!(cmd.end_line, 2); + assert_eq!(cmd.end_line, 3); } - #[test] - fn slash_at_end_without_space_is_not_continuation() { - let header = header_from("/path /var/log/"); - let cmd = start_command(header, 0); + // --- Property tests --- - assert_eq!(cmd.mode, ArgumentMode::SingleLine); - assert!(!cmd.is_open); - assert_eq!(cmd.payload_lines, vec!["/var/log/"]); + use proptest::prelude::*; + + fn valid_command_name() -> impl Strategy { + "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + } + + proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn raw_lines_count_equals_lines_consumed( + // §8.2: raw accumulates opener + every body line + closer (if present). + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 0..8) + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0, 0); + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); + prop_assert_eq!(cmd.raw_lines.len(), body_lines.len() + 2); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn payload_never_contains_closer( + // §5.2.4: the closing fence line is never part of the payload. + // The check mirrors is_fence_closer: non-empty after trim, all backticks. + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 0..8) + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0, 0); + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); + let no_closer = !cmd.payload_lines.iter().any(|l| { + let t = l.trim(); + !t.is_empty() && t.chars().all(|c| c == '`') + }); + prop_assert!(no_closer); } + + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fence_closer_property_count_gte_opener( + // §5.2.4: any line of N or more backticks (N >= opener) closes the fence. + name in valid_command_name(), + extra in 0usize..5 + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0, 0); + let closer = "`".repeat(3 + extra); + let (cmd, res) = accept_line(cmd, 1, &closer); + prop_assert_eq!(res, AcceptResult::Completed); + prop_assert!(!cmd.is_open); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn id_is_preserved_through_accumulation( + // §7: the id assigned at start_command is stable throughout accumulation. + name in valid_command_name(), + id in 0usize..1000, + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 0..5) + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0, id); + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + prop_assert_eq!(cmd.id, id); + } + } } diff --git a/parser/src/application/command_finalize.rs b/parser/src/application/command_finalize.rs index 7c3ca49..a7e9288 100644 --- a/parser/src/application/command_finalize.rs +++ b/parser/src/application/command_finalize.rs @@ -1,34 +1,47 @@ use super::command_accumulate::PendingCommand; -use crate::domain::{ArgumentMode, Command, CommandArguments, LineRange, ParseWarning}; +use crate::domain::{ArgumentMode, Command, CommandArguments, LineRange, Warning}; #[derive(Debug)] pub struct FinalizedCommand { pub command: Command, - pub warnings: Vec, + pub warnings: Vec, } +/// Consume a PendingCommand and produce the finalized Command value plus any warnings. +/// +/// §7: id is formatted as cmd-{n} using the zero-based encounter index from start_command. +/// §8.2: raw is the physical lines joined with newline separators (opener + body + closer). +/// §5.2.5: an unclosed fence produces a Warning rather than a hard parse failure. pub fn finalize_command(pending: PendingCommand) -> FinalizedCommand { let mut warnings = Vec::new(); - match pending.mode { - ArgumentMode::Fence if pending.is_open => { - warnings.push(ParseWarning::UnclosedFence { - start_line: pending.start_line, - }); - } - ArgumentMode::Continuation if pending.is_open => { - warnings.push(ParseWarning::UnclosedContinuation { - start_line: pending.start_line, - }); - } - _ => {} + // §5.2.5: if the parser reached EOF without finding a closing fence, emit one warning + // and finalize with whatever payload was accumulated. + if pending.mode == ArgumentMode::Fence && pending.is_open { + warnings.push(Warning { + wtype: "unclosed-fence".to_string(), + start_line: Some(pending.start_line), + message: Some(format!( + "Fenced block opened at line {} was never closed.", + pending.start_line + )), + }); } + // §8.2: raw is built from all accumulated physical lines joined with newline separators. + // For single-line commands this is just the one header line. + // For fenced commands this includes the opener, all body lines, and the closer if present. + let raw = pending.raw_lines.join("\n"); + + // §5.1: single-line payload equals header_text (initialised at start_command). + // §5.2.2: fence payload is the verbatim body lines joined with newline separators. let payload = pending.payload_lines.join("\n"); let command = Command { + // §7: sequential zero-based id formatted as cmd-0, cmd-1, ... + id: format!("cmd-{}", pending.id), name: pending.name, - raw: pending.raw_header, + raw, range: LineRange { start_line: pending.start_line, end_line: pending.end_line, @@ -58,142 +71,217 @@ mod tests { fn header_from(line: &str) -> CommandHeader { match classify_line(line) { LineKind::Command(h) => h, - _ => panic!("expected command header"), + _ => panic!("expected command header from: {line:?}"), } } - // --- Warning tests --- + // --- id --- #[test] - fn unclosed_fence_at_eof_warns() { - let cmd = start_command(header_from("/cmd ```rust"), 0); - let (cmd, _) = accept_line(cmd, 1, "line1"); - let (cmd, _) = accept_line(cmd, 2, "line2"); + fn id_is_formatted_as_cmd_zero() { + // §7: commands are assigned sequential zero-based IDs formatted as cmd-0, cmd-1, ... + let cmd = start_command(header_from("/deploy staging"), 0, 0); + let result = finalize_command(cmd); + assert_eq!(result.command.id, "cmd-0"); + } + #[test] + fn id_reflects_encounter_index() { + // §7: the id counter is caller-supplied; cmd-5 identifies the sixth command encountered. + let cmd = start_command(header_from("/deploy staging"), 0, 5); let result = finalize_command(cmd); + assert_eq!(result.command.id, "cmd-5"); + } - assert_eq!(result.warnings.len(), 1); - assert!(matches!( - result.warnings[0], - ParseWarning::UnclosedFence { start_line: 0 } - )); + // --- raw --- + + #[test] + fn single_line_raw_is_header_line_only() { + // §8.2: for a single-line command raw contains only the original command line. + let cmd = start_command(header_from("/deploy production --region us-west-2"), 0, 0); + let result = finalize_command(cmd); + assert_eq!(result.command.raw, "/deploy production --region us-west-2"); } #[test] - fn unclosed_continuation_at_eof_warns() { - let cmd = start_command(header_from("/cmd first \\"), 0); + fn fenced_raw_includes_opener_body_and_closer() { + // §8.2: for fenced commands raw includes the command line, all body lines, + // and the closing fence line, joined with newline separators. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "line one"); + let (cmd, _) = accept_line(cmd, 2, "line two"); + let (cmd, _) = accept_line(cmd, 3, "```"); + let result = finalize_command(cmd); + assert_eq!(result.command.raw, "/cmd ```\nline one\nline two\n```"); + } + #[test] + fn unclosed_fence_raw_includes_opener_and_partial_body() { + // §8.2: if the fence is unclosed raw still contains all accumulated physical lines; + // there is simply no closer line to include. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "partial"); let result = finalize_command(cmd); + assert_eq!(result.command.raw, "/cmd ```\npartial"); + } + + // --- warnings --- + #[test] + fn unclosed_fence_emits_exactly_one_warning() { + // §5.2.5: an unclosed fence produces exactly one warning; the parser does not fail. + let cmd = start_command(header_from("/cmd ```rust"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "line1"); + let (cmd, _) = accept_line(cmd, 2, "line2"); + let result = finalize_command(cmd); assert_eq!(result.warnings.len(), 1); - assert!(matches!( - result.warnings[0], - ParseWarning::UnclosedContinuation { start_line: 0 } - )); } #[test] - fn closed_fence_has_no_warning() { - let cmd = start_command(header_from("/cmd ```"), 0); - let (cmd, _) = accept_line(cmd, 1, "content"); - let (cmd, _) = accept_line(cmd, 2, "```"); - + fn unclosed_fence_warning_wtype_is_unclosed_fence() { + // §5.2.5: the warning type string is "unclosed-fence". + let cmd = start_command(header_from("/cmd ```"), 0, 0); let result = finalize_command(cmd); + assert_eq!(result.warnings[0].wtype, "unclosed-fence"); + } - assert!(result.warnings.is_empty()); + #[test] + fn unclosed_fence_warning_start_line_is_opener_physical_line() { + // §5.2.5: start_line is set to the fence opener's physical line number. + let cmd = start_command(header_from("/cmd ```"), 4, 0); + let result = finalize_command(cmd); + assert_eq!(result.warnings[0].start_line, Some(4)); } #[test] - fn closed_continuation_has_no_warning() { - let cmd = start_command(header_from("/cmd first \\"), 0); - let (cmd, _) = accept_line(cmd, 1, "last line"); + fn unclosed_fence_warning_message_is_present() { + // §5.2.5 (implied): a human-readable message is present and non-empty. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let result = finalize_command(cmd); + assert!( + result.warnings[0] + .message + .as_deref() + .map(|m| !m.is_empty()) + .unwrap_or(false) + ); + } + #[test] + fn unclosed_fence_warning_message_references_start_line() { + // §5.2.5 (implied): the message should identify the offending line number for diagnostics. + let cmd = start_command(header_from("/cmd ```"), 7, 0); let result = finalize_command(cmd); + let msg = result.warnings[0].message.as_deref().unwrap_or(""); + assert!(msg.contains('7')); + } + #[test] + fn closed_fence_has_no_warning() { + // §5.2.3: once the closing fence is found the command is well-formed; no warning emitted. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "content"); + let (cmd, _) = accept_line(cmd, 2, "```"); + let result = finalize_command(cmd); assert!(result.warnings.is_empty()); } #[test] fn single_line_has_no_warning() { - let cmd = start_command(header_from("/help"), 0); - + // §5.1 (implied): a single-line command is always complete; no warning emitted. + let cmd = start_command(header_from("/help"), 0, 0); let result = finalize_command(cmd); - assert!(result.warnings.is_empty()); } - // --- Finalized command structure --- + // --- payload --- #[test] - fn finalized_command_has_correct_name() { - let cmd = start_command(header_from("/deploy production"), 0); - + fn single_line_payload_equals_header_text() { + // §5.1: in single-line mode header and payload contain the same string. + let cmd = start_command(header_from("/hello world"), 0, 0); let result = finalize_command(cmd); - - assert_eq!(result.command.name, "deploy"); + assert_eq!(result.command.arguments.payload, "world"); + assert_eq!(result.command.arguments.mode, ArgumentMode::SingleLine); } #[test] - fn finalized_command_has_correct_range() { - let cmd = start_command(header_from("/cmd ```"), 0); - let (cmd, _) = accept_line(cmd, 1, "body"); - let (cmd, _) = accept_line(cmd, 2, "```"); - + fn single_line_no_args_has_empty_payload() { + // §5.1 (implied): a command with no arguments has an empty header and empty payload. + let cmd = start_command(header_from("/ping"), 0, 0); let result = finalize_command(cmd); - - assert_eq!(result.command.range.start_line, 0); - assert_eq!(result.command.range.end_line, 2); + assert_eq!(result.command.arguments.payload, ""); + assert_eq!(result.command.arguments.header, ""); } #[test] - fn finalized_fence_payload_is_joined_lines() { - let cmd = start_command(header_from("/cmd ```"), 0); + fn fence_payload_is_body_lines_joined() { + // §5.2.2: fence payload is the verbatim body lines joined with newline separators. + // §5.2.4: the closing fence line is not included in the payload. + // Order matters: body accumulation (§5.2.2) must be verified against closer exclusion (§5.2.4). + let cmd = start_command(header_from("/cmd ```"), 0, 0); let (cmd, _) = accept_line(cmd, 1, "line one"); let (cmd, _) = accept_line(cmd, 2, "line two"); let (cmd, _) = accept_line(cmd, 3, "```"); - let result = finalize_command(cmd); - assert_eq!(result.command.arguments.payload, "line one\nline two"); assert_eq!(result.command.arguments.mode, ArgumentMode::Fence); } #[test] - fn finalized_continuation_payload_is_joined_lines() { - let cmd = start_command(header_from("/cmd first \\"), 0); - let (cmd, _) = accept_line(cmd, 1, "second \\"); - let (cmd, _) = accept_line(cmd, 2, "third"); - + fn empty_fence_body_has_empty_payload() { + // §5.2.2 (implied): a fence with no body lines between opener and closer has empty payload. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "```"); let result = finalize_command(cmd); - - assert_eq!(result.command.arguments.payload, "first\nsecond\nthird"); - assert_eq!(result.command.arguments.mode, ArgumentMode::Continuation); + assert_eq!(result.command.arguments.payload, ""); } - #[test] - fn finalized_single_line_payload_matches_header() { - let cmd = start_command(header_from("/hello world"), 0); + // --- range and metadata --- + #[test] + fn finalized_command_has_correct_range() { + // §2.2.1: range covers zero-based physical line numbers from opener through closer. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "body"); + let (cmd, _) = accept_line(cmd, 2, "```"); let result = finalize_command(cmd); + assert_eq!(result.command.range.start_line, 0); + assert_eq!(result.command.range.end_line, 2); + } - assert_eq!(result.command.arguments.payload, "world"); - assert_eq!(result.command.arguments.mode, ArgumentMode::SingleLine); + #[test] + fn finalized_command_name_matches_header() { + // §3.1 (implied): the command name is carried through the pipeline unchanged. + let cmd = start_command(header_from("/deploy production"), 0, 0); + let result = finalize_command(cmd); + assert_eq!(result.command.name, "deploy"); } #[test] - fn finalized_fence_captures_language() { - let cmd = start_command(header_from("/code ```rust"), 0); + fn fence_lang_preserved_in_arguments() { + // §5.2.1: fence_lang is the language identifier from the opener; null if absent. + let cmd = start_command(header_from("/code ```rust"), 0, 0); let (cmd, _) = accept_line(cmd, 1, "fn main() {}"); let (cmd, _) = accept_line(cmd, 2, "```"); - let result = finalize_command(cmd); - assert_eq!( result.command.arguments.fence_lang, Some("rust".to_string()) ); } + #[test] + fn fence_without_lang_has_none_fence_lang() { + // §5.2.1: fence_lang is None when no language identifier follows the backtick run. + let cmd = start_command(header_from("/cmd ```"), 0, 0); + let (cmd, _) = accept_line(cmd, 1, "```"); + let result = finalize_command(cmd); + assert_eq!(result.command.arguments.fence_lang, None); + } + + // --- Property tests --- + use proptest::prelude::*; fn valid_command_name() -> impl Strategy { @@ -201,29 +289,43 @@ mod tests { } proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn id_format_is_cmd_n( + // §7: the id field always matches the pattern cmd-[0-9]+. + name in valid_command_name(), + n in 0usize..1000 + ) { + let input = format!("/{name} arg"); + let cmd = start_command(header_from(&input), 0, n); + let result = finalize_command(cmd); + prop_assert_eq!(result.command.id, format!("cmd-{n}")); + } + #[test] #[cfg_attr(feature = "tdd", ignore)] fn finalized_name_matches_pending_name(name in valid_command_name()) { + // §3.1 (implied): name survives the full start_command -> finalize_command pipeline. let input = format!("/{name} arg"); - let cmd = start_command(header_from(&input), 0); + let cmd = start_command(header_from(&input), 0, 0); let result = finalize_command(cmd); prop_assert_eq!(result.command.name, name); } #[test] #[cfg_attr(feature = "tdd", ignore)] - fn closed_commands_never_warn( + fn closed_fence_never_warns( + // §5.2.3: a properly closed fence produces zero warnings regardless of body content. name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9]{1,30}", 1..8) + body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 1..8) ) { let input = format!("/{name} ```"); - let cmd = start_command(header_from(&input), 0); + let cmd = start_command(header_from(&input), 0, 0); let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { let (next, _) = accept_line(cmd, i + 1, line); next }); let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); - let result = finalize_command(cmd); prop_assert!(result.warnings.is_empty()); } @@ -231,38 +333,60 @@ mod tests { #[test] #[cfg_attr(feature = "tdd", ignore)] fn unclosed_fence_always_warns( + // §5.2.5: any fence that reaches EOF without a closer produces exactly one warning. name in valid_command_name(), body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 1..5) ) { let input = format!("/{name} ```"); - let cmd = start_command(header_from(&input), 0); + let cmd = start_command(header_from(&input), 0, 0); let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { let (next, _) = accept_line(cmd, i + 1, line); next }); - let result = finalize_command(cmd); prop_assert_eq!(result.warnings.len(), 1); - let is_unclosed_fence = matches!(result.warnings[0], ParseWarning::UnclosedFence { .. }); - prop_assert!(is_unclosed_fence); + prop_assert_eq!(&result.warnings[0].wtype, "unclosed-fence"); } #[test] #[cfg_attr(feature = "tdd", ignore)] - fn payload_is_payload_lines_joined( + fn fence_payload_equals_body_lines_joined( + // §5.2.2: payload is exactly the body lines joined with "\n", verbatim. name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 1..8) + body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,20}", 1..8) ) { let input = format!("/{name} ```"); - let cmd = start_command(header_from(&input), 0); + let cmd = start_command(header_from(&input), 0, 0); let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { let (next, _) = accept_line(cmd, i + 1, line); next }); let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); - let result = finalize_command(cmd); prop_assert_eq!(result.command.arguments.payload, body_lines.join("\n")); } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn raw_newline_count_equals_physical_lines_minus_one( + // §8.2: raw joins all physical lines so its newline count equals consumed lines - 1. + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 0..6) + ) { + let input = format!("/{name} ```"); + let cmd = start_command(header_from(&input), 0, 0); + let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { + let (next, _) = accept_line(cmd, i + 1, line); + next + }); + let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); + let result = finalize_command(cmd); + // opener + body + closer = body_lines.len() + 2 physical lines + let expected_newlines = body_lines.len() + 1; + prop_assert_eq!( + result.command.raw.chars().filter(|&c| c == '\n').count(), + expected_newlines + ); + } } } diff --git a/parser/src/application/document_parse.rs b/parser/src/application/document_parse.rs index a9da645..06e66dc 100644 --- a/parser/src/application/document_parse.rs +++ b/parser/src/application/document_parse.rs @@ -1,452 +1,520 @@ use super::{ command_accumulate::{AcceptResult, PendingCommand, accept_line, start_command}, command_finalize::finalize_command, - line_classify::{LineKind, classify_line}, + line_classify::{CommandHeader, LineKind, classify_line}, + line_join::LineJoiner, + normalize::normalize, text_collect::{PendingText, append_text, finalize_text, start_text}, }; -use crate::domain::{Command, ParseResult, ParseWarning, ParserContext, SPEC_VERSION, TextBlock}; +use crate::domain::{Command, ParseResult, SPEC_VERSION, TextBlock, Warning}; -struct ParseState { - current_cmd: Option, +// --- Types --- + +#[derive(Debug)] +enum ParserState { + Idle, + InFence(PendingCommand), +} + +#[derive(PartialEq, Eq)] +enum LoopAction { + Continue, + Break, +} + +struct ParseCtx { + state: ParserState, current_text: Option, commands: Vec, - text_blocks: Vec, - warnings: Vec, + textblocks: Vec, + warnings: Vec, + cmd_seq: usize, + text_seq: usize, } +impl ParseCtx { + fn new() -> Self { + Self { + state: ParserState::Idle, + current_text: None, + commands: Vec::new(), + textblocks: Vec::new(), + warnings: Vec::new(), + cmd_seq: 0, + text_seq: 0, + } + } + + fn into_result(self) -> ParseResult { + ParseResult { + version: SPEC_VERSION.to_owned(), + commands: self.commands, + textblocks: self.textblocks, + warnings: self.warnings, + } + } +} + +// --- Public entry point --- + pub fn parse_document(input: &str) -> ParseResult { - let state = input - .lines() - .enumerate() - .fold(empty_state(), |state, (i, line)| step(state, i, line)); + let normalized = normalize(input); + let physical_lines = split_physical_lines(&normalized); + let owned: Vec = physical_lines.iter().map(|s| s.to_string()).collect(); + let mut joiner = LineJoiner::new(owned); + let mut ctx = ParseCtx::new(); - finalize(state) + while step(&mut ctx, &mut joiner, &physical_lines) == LoopAction::Continue {} + + flush_text(&mut ctx); + ctx.into_result() } -fn empty_state() -> ParseState { - ParseState { - current_cmd: None, - current_text: None, - commands: Vec::new(), - text_blocks: Vec::new(), - warnings: Vec::new(), +// --- Pipeline --- + +fn split_physical_lines(normalized: &str) -> Vec<&str> { + let mut lines: Vec<&str> = normalized.split('\n').collect(); + if lines.last() == Some(&"") { + lines.pop(); } + lines } -fn step(mut state: ParseState, line_index: usize, line: &str) -> ParseState { - if try_feed_command(&mut state, line_index, line) { - return state; +fn step(ctx: &mut ParseCtx, joiner: &mut LineJoiner, phys: &[&str]) -> LoopAction { + let state = std::mem::replace(&mut ctx.state, ParserState::Idle); + match state { + ParserState::Idle => step_idle(ctx, joiner, phys), + ParserState::InFence(cmd) => step_in_fence(ctx, joiner, cmd), } - - classify_fresh_line(&mut state, line_index, line); - state } -fn try_feed_command(state: &mut ParseState, line_index: usize, line: &str) -> bool { - let Some(cmd) = state.current_cmd.take() else { - return false; +// --- State handlers --- + +// fn step_idle(ctx: &mut ParseCtx, joiner: &mut LineJoiner, phys: &[&str]) -> LoopAction { +// let Some(ll) = joiner.next_logical() else { +// return LoopAction::Break; +// }; +// match classify_line(&ll.text) { +// LineKind::Command(mut header) => { +// header.raw = phys[ll.first_physical..=ll.last_physical].join("\n"); +// flush_text(ctx); +// start_new_command(ctx, header, ll.first_physical); +// } +// LineKind::Text => { +// accumulate_text(ctx, ll.first_physical, ll.last_physical, phys); +// } +// } +// LoopAction::Continue +// } + +fn step_idle(ctx: &mut ParseCtx, joiner: &mut LineJoiner, phys: &[&str]) -> LoopAction { + let Some(ll) = joiner.next_logical() else { + return LoopAction::Break; }; - - if !cmd.is_open { - absorb_command(state, cmd); - return false; + match classify_line(&ll.text) { + LineKind::Command(mut header) => { + header.raw = phys[ll.first_physical..=ll.last_physical].join("\n"); + flush_text(ctx); + start_new_command(ctx, header, ll.first_physical, ll.last_physical); + } + LineKind::Text => { + accumulate_text(ctx, ll.first_physical, ll.last_physical, phys); + } } + LoopAction::Continue +} - let (updated_cmd, result) = accept_line(cmd, line_index, line); +fn step_in_fence(ctx: &mut ParseCtx, joiner: &mut LineJoiner, cmd: PendingCommand) -> LoopAction { + let Some((line_idx, line)) = joiner.next_physical() else { + absorb_command(ctx, cmd); + return LoopAction::Break; + }; + let (updated, result) = accept_line(cmd, line_idx, &line); match result { AcceptResult::Consumed => { - state.current_cmd = Some(updated_cmd); - true + ctx.state = ParserState::InFence(updated); } - AcceptResult::Completed => { - absorb_command(state, updated_cmd); - true - } - AcceptResult::Rejected => { - absorb_command(state, updated_cmd); - false + AcceptResult::Completed | AcceptResult::Rejected => { + absorb_command(ctx, updated); } } + LoopAction::Continue } -fn classify_fresh_line(state: &mut ParseState, line_index: usize, line: &str) { - match classify_line(line) { - LineKind::Command(header) => { - flush_text(state); - state.current_cmd = Some(start_command(header, line_index)); - } - LineKind::Text => { - accumulate_text(state, line_index, line); - } +// --- Context helpers --- + +// fn start_new_command(ctx: &mut ParseCtx, header: CommandHeader, first_physical: usize) { +// let cmd = start_command(header, first_physical, ctx.cmd_seq); +// ctx.cmd_seq += 1; +// if cmd.is_open { +// ctx.state = ParserState::InFence(cmd); +// } else { +// absorb_command(ctx, cmd); +// } +// } + +fn start_new_command( + ctx: &mut ParseCtx, + header: CommandHeader, + first_physical: usize, + last_physical: usize, +) { + let mut cmd = start_command(header, first_physical, ctx.cmd_seq); + cmd.end_line = last_physical; + ctx.cmd_seq += 1; + if cmd.is_open { + ctx.state = ParserState::InFence(cmd); + } else { + absorb_command(ctx, cmd); } } -fn absorb_command(state: &mut ParseState, cmd: PendingCommand) { +fn absorb_command(ctx: &mut ParseCtx, cmd: PendingCommand) { let finalized = finalize_command(cmd); - state.commands.push(finalized.command); - state.warnings.extend(finalized.warnings); + ctx.commands.push(finalized.command); + ctx.warnings.extend(finalized.warnings); } -fn flush_text(state: &mut ParseState) { - if let Some(text) = state.current_text.take() { - state.text_blocks.push(finalize_text(text)); - } +fn flush_text(ctx: &mut ParseCtx) { + let Some(text) = ctx.current_text.take() else { + return; + }; + ctx.textblocks.push(finalize_text(text, ctx.text_seq)); + ctx.text_seq += 1; } -fn accumulate_text(state: &mut ParseState, line_index: usize, line: &str) { - state.current_text = match state.current_text.take() { - Some(text) => Some(append_text(text, line_index, line)), - None => Some(start_text(line_index, line)), +fn accumulate_text(ctx: &mut ParseCtx, first: usize, last: usize, phys: &[&str]) { + let text = match ctx.current_text.take() { + Some(existing) => fold_physical_lines(existing, first, last, phys), + None => { + let started = start_text(first, phys[first]); + fold_physical_lines(started, first + 1, last, phys) + } }; + ctx.current_text = Some(text); } -fn finalize(mut state: ParseState) -> ParseResult { - if let Some(cmd) = state.current_cmd.take() { - absorb_command(&mut state, cmd); - } - - flush_text(&mut state); - - ParseResult { - commands: state.commands, - text_blocks: state.text_blocks, - warnings: state.warnings, - version: SPEC_VERSION.to_owned(), - context: ParserContext::default(), +fn fold_physical_lines( + mut text: PendingText, + from: usize, + to: usize, + phys: &[&str], +) -> PendingText { + for (idx, line) in (from..=to).zip(&phys[from..=to]) { + text = append_text(text, idx, line); } + text } #[cfg(test)] mod tests { - use proptest::prelude::*; + use super::parse_document; + use crate::domain::{ArgumentMode, SPEC_VERSION}; - use crate::{application::document_parse::parse_document, domain::ArgumentMode}; - - fn parse(input: &str) -> crate::domain::ParseResult { - parse_document(input) - } - - fn assert_single_command(input: &str) -> crate::domain::Command { - let result = parse(input); - assert_eq!(result.commands.len(), 1, "expected exactly 1 command"); - result.commands.into_iter().next().unwrap() - } + // --- Category 1: Empty / trivial input --- #[test] - fn single_line_simple_command_parses_name() { - let cmd = assert_single_command("/hello world"); - assert_eq!(cmd.name, "hello"); + fn empty_input_produces_empty_result() { + // §8.1 (implied): empty input produces envelope with empty arrays. + let result = parse_document(""); + assert!(result.commands.is_empty()); + assert!(result.textblocks.is_empty()); + assert!(result.warnings.is_empty()); } #[test] - fn trailing_newline_does_not_create_empty_text_block() { - let result = parse("/cmd arg\n"); - assert_eq!(result.text_blocks.len(), 0); + fn whitespace_only_input_is_text() { + // §6 (implied): lines that are not commands become text blocks. + let result = parse_document(" "); + assert!(result.commands.is_empty()); + assert_eq!(result.textblocks.len(), 1); } + // --- Category 2: Single-line command threading --- + #[test] - fn single_line_mode_threads_through_from_classify() { - let cmd = assert_single_command("/hello world"); - assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); + fn single_line_command_parses_name() { + // §5.1: command name is extracted and threaded to output. + let result = parse_document("/deploy production"); + assert_eq!(result.commands.len(), 1); + assert_eq!(result.commands.first().unwrap().name, "deploy"); } #[test] - fn single_line_payload_threads_through_from_classify() { - let cmd = assert_single_command("/hello world"); - assert_eq!(cmd.arguments.payload, "world"); + fn single_line_command_with_no_args_has_empty_payload() { + // §5.1 (implied): command with no arguments produces empty header and payload. + let result = parse_document("/ping"); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.arguments.header, ""); + assert_eq!(cmd.arguments.payload, ""); } #[test] - fn unclosed_continuation_produces_warning() { - let input = "/cmd start \\"; - let result = parse(input); - assert_eq!(result.commands.len(), 1); - assert_eq!(result.warnings.len(), 1); + fn single_line_command_range_is_same_line() { + // §2.2.1: range uses physical line numbers. Single logical line -> start == end. + let result = parse_document("/deploy production"); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 0); } #[test] - fn leading_whitespace_command_threads_through() { - let cmd = assert_single_command(" /hello world"); - assert_eq!(cmd.name, "hello"); + fn single_line_mode_threads_through() { + // §5.1: mode is single-line when no fence opener present. + let result = parse_document("/deploy production"); + assert_eq!( + result.commands.first().unwrap().arguments.mode, + ArgumentMode::SingleLine + ); } #[test] - fn single_line_command_with_no_args_has_empty_header() { - let cmd = assert_single_command("/ping"); - assert_eq!(cmd.arguments.header, ""); - assert_eq!(cmd.arguments.payload, ""); + fn single_line_payload_threads_through() { + // §5.1: payload equals the full arguments text. + let result = parse_document("/deploy production --region us-west-2"); + assert_eq!( + result.commands.first().unwrap().arguments.payload, + "production --region us-west-2" + ); } + // --- Category 3: Fenced command pipeline --- + #[test] - fn single_line_command_with_complex_args() { - let cmd = assert_single_command(r#"/mcp call_tool read_file {"path": "src/index.ts"}"#); - assert_eq!(cmd.name, "mcp"); + fn fenced_command_parses_through_document() { + // §5.2.2: fence body lines appended verbatim, joined with \n separators. + let result = parse_document("/cmd ```\nline one\nline two\n```"); + assert_eq!(result.commands.len(), 1); assert_eq!( - cmd.arguments.header, - r#"call_tool read_file {"path": "src/index.ts"}"# + result.commands.first().unwrap().arguments.payload, + "line one\nline two" ); } #[test] - fn single_line_command_range_is_same_line() { - let cmd = assert_single_command("/test"); + fn fenced_command_range_covers_opener_through_closer() { + // §2.2.1: range.start_line and range.end_line are physical line numbers. + let result = parse_document("/cmd ```\nbody\n```"); + let cmd = result.commands.first().unwrap(); assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 0); + assert_eq!(cmd.range.end_line, 2); } #[test] - fn parses_text_and_single_command() { - let input = "intro\n/cmd arg\noutro"; - let result = parse_document(input); - assert_eq!(result.commands.len(), 1); - assert_eq!(result.text_blocks.len(), 2); - assert_eq!(result.commands[0].name, "cmd"); - assert_eq!(result.commands[0].arguments.mode, ArgumentMode::SingleLine); + fn fence_lang_threads_through() { + // §5.2.1: fence_lang is the language identifier from the opener. + let result = parse_document("/cmd ```json\n{}\n```"); + assert_eq!( + result.commands.first().unwrap().arguments.fence_lang, + Some("json".to_string()) + ); } - #[test] - fn command_on_line_two_has_correct_range() { - let input = "text\nmore text\n/cmd arg"; - let result = parse(input); - assert_eq!(result.commands[0].range.start_line, 2); - assert_eq!(result.commands[0].range.end_line, 2); - } + // --- Category 4: Unclosed fence --- #[test] - fn two_single_line_commands_both_parse() { - let result = parse("/first a\n/second b"); - assert_eq!(result.commands.len(), 2); - assert_eq!(result.commands[0].name, "first"); - assert_eq!(result.commands[1].name, "second"); + fn unclosed_fence_produces_warning() { + // §5.2.5: unclosed fence emits warning with type "unclosed-fence". + let result = parse_document("/cmd ```\norphaned body"); + assert_eq!(result.warnings.len(), 1); + assert_eq!(result.warnings.first().unwrap().wtype, "unclosed-fence"); } #[test] - fn empty_input_produces_empty_result() { - let result = parse(""); - assert_eq!(result.commands.len(), 0); - assert_eq!(result.text_blocks.len(), 0); - assert_eq!(result.warnings.len(), 0); + fn unclosed_fence_still_produces_command() { + // §5.2.5: parser emits the command with its partial payload. + let result = parse_document("/cmd ```\npartial"); + assert_eq!(result.commands.len(), 1); + assert_eq!( + result.commands.first().unwrap().arguments.payload, + "partial" + ); } + // --- Category 5: Text block accumulation --- + #[test] fn text_only_produces_one_text_block() { - let result = parse("just some text"); - assert_eq!(result.commands.len(), 0); - assert_eq!(result.text_blocks.len(), 1); + // §6: consecutive non-command lines form a single text block. + let result = parse_document("just some text"); + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().content, "just some text"); } #[test] fn adjacent_text_lines_merge_into_one_block() { - let result = parse("line one\nline two\nline three"); - assert_eq!(result.text_blocks.len(), 1); - assert_eq!(result.text_blocks[0].range.start_line, 0); - assert_eq!(result.text_blocks[0].range.end_line, 2); + // §6: consecutive non-command lines form a single text block. + let result = parse_document("line one\nline two\nline three"); + assert_eq!(result.textblocks.len(), 1); + assert_eq!( + result.textblocks.first().unwrap().content, + "line one\nline two\nline three" + ); } #[test] - fn text_block_before_command_has_correct_content() { - let result = parse("intro line\n/cmd arg"); - assert_eq!(result.text_blocks.len(), 1); - assert_eq!(result.text_blocks[0].content, "intro line"); + fn text_block_content_uses_physical_lines() { + // ADR-NNNN: text block content preserves original physical lines. + let result = parse_document("hello\nworld"); + assert_eq!(result.textblocks.first().unwrap().content, "hello\nworld"); } #[test] - fn text_block_after_command_has_correct_range() { - let result = parse("/cmd arg\noutro line"); - assert_eq!(result.text_blocks.len(), 1); - assert_eq!(result.text_blocks[0].range.start_line, 1); + fn text_block_range_covers_physical_lines() { + // §6: text blocks use physical line numbers for their range. + let result = parse_document("line one\nline two"); + let tb = result.textblocks.first().unwrap(); + assert_eq!(tb.range.start_line, 0); + assert_eq!(tb.range.end_line, 1); } + // --- Category 6: Interleaving commands and text --- + #[test] - fn continuation_command_parses_through_document() { - let input = "/cmd first \\\nsecond \\\nthird"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Continuation); - assert_eq!(cmd.arguments.payload, "first\nsecond\nthird"); + fn text_before_command_is_captured() { + // §6: non-command lines before a command form a text block. + let result = parse_document("preamble\n/cmd arg"); + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().content, "preamble"); + assert_eq!(result.commands.len(), 1); } #[test] - fn fenced_command_parses_through_document() { - let input = "/cmd ```\nline one\nline two\n```"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); - assert_eq!(cmd.arguments.payload, "line one\nline two"); + fn text_after_command_is_captured() { + // §6: new text block begins after a command is finalized. + let result = parse_document("/cmd arg\npostamble"); + assert_eq!(result.commands.len(), 1); + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().content, "postamble"); } #[test] - fn unclosed_fence_produces_warning() { - let input = "/cmd ```\nline one\nline two"; - let result = parse(input); - assert_eq!(result.commands.len(), 1); - assert_eq!(result.warnings.len(), 1); + fn text_between_commands_is_captured() { + // §6: text between two commands forms its own block. + let result = parse_document("/cmd1 a\nmiddle text\n/cmd2 b"); + assert_eq!(result.commands.len(), 2); + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().content, "middle text"); } #[test] - fn fence_followed_by_new_command() { - let input = "/first ```\nbody\n```\n/second arg"; - let result = parse(input); + fn two_single_line_commands_both_parse() { + // §7: multiple commands assigned sequential IDs. + let result = parse_document("/cmd1 a\n/cmd2 b"); assert_eq!(result.commands.len(), 2); - assert_eq!(result.commands[0].name, "first"); - assert_eq!(result.commands[1].name, "second"); + assert_eq!(result.commands.first().unwrap().name, "cmd1"); + assert_eq!(result.commands.get(1).unwrap().name, "cmd2"); } #[test] - fn continuation_followed_by_new_command() { - let input = "/first start \\\nend\n/second arg"; - let result = parse(input); + fn fence_followed_by_new_command() { + // §4.1: after fence closes, parser returns to idle and scans for next command. + let result = parse_document("/cmd1 ```\nbody\n```\n/cmd2 arg"); assert_eq!(result.commands.len(), 2); - assert_eq!(result.commands[0].name, "first"); - assert_eq!(result.commands[1].name, "second"); + assert_eq!( + result.commands.first().unwrap().arguments.mode, + ArgumentMode::Fence + ); + assert_eq!( + result.commands.get(1).unwrap().arguments.mode, + ArgumentMode::SingleLine + ); } + // --- Category 7: ID assignment --- + #[test] - fn version_is_set() { - let result = parse(""); - assert_eq!(result.version, crate::domain::SPEC_VERSION); + fn command_ids_are_sequential_zero_based() { + // §7: commands assigned cmd-0, cmd-1, cmd-2. + let result = parse_document("/a x\n/b y\n/c z"); + assert_eq!(result.commands.first().unwrap().id, "cmd-0"); + assert_eq!(result.commands.get(1).unwrap().id, "cmd-1"); + assert_eq!(result.commands.get(2).unwrap().id, "cmd-2"); } - // --- Property tests --- + #[test] + fn text_block_ids_are_sequential_zero_based() { + // §7: text blocks assigned text-0, text-1, text-2. + let result = parse_document("aaa\n/cmd x\nbbb\n/cmd y\nccc"); + assert_eq!(result.textblocks.first().unwrap().id, "text-0"); + assert_eq!(result.textblocks.get(1).unwrap().id, "text-1"); + assert_eq!(result.textblocks.get(2).unwrap().id, "text-2"); + } - fn valid_command_name() -> impl Strategy { - "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + #[test] + fn command_and_text_ids_are_independent() { + // §7: command and text block ID sequences are independent. + let result = parse_document("prose\n/cmd arg\nmore prose"); + assert_eq!(result.commands.first().unwrap().id, "cmd-0"); + assert_eq!(result.textblocks.first().unwrap().id, "text-0"); + assert_eq!(result.textblocks.get(1).unwrap().id, "text-1"); } - fn command_line() -> impl Strategy { - (valid_command_name(), "[a-z0-9 ]{0,30}").prop_map(|(name, args)| format!("/{name} {args}")) + // --- Category 8: raw field --- + + #[test] + fn single_line_raw_is_header_line_only() { + // §8.2: single-line command raw contains only the original command line. + let result = parse_document("/echo hello world"); + assert_eq!(result.commands.first().unwrap().raw, "/echo hello world"); } - fn text_line() -> impl Strategy { - "[a-zA-Z0-9 !.,]{1,60}".prop_filter("must not start with slash", |s| { - !s.trim_start().starts_with('/') - }) + #[test] + fn joined_command_raw_includes_backslashes() { + // §8.2 + ADR-NNNN: raw contains physical lines with backslashes and \n separators. + // The physical_lines slice and join logic is exclusively in document_parse.rs. + let result = parse_document("/deploy prod \\\n --region us-west-2"); + assert_eq!( + result.commands.first().unwrap().raw, + "/deploy prod \\\n --region us-west-2" + ); } - proptest! { - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn never_panics_on_arbitrary_input(input in "[\\x00-\\x7F]{0,500}") { - let _ = parse_document(&input); - } + #[test] + fn fenced_raw_includes_opener_body_and_closer() { + // §8.2: fenced command raw includes command line, body lines, and closing fence. + let result = parse_document("/cmd ```\nbody\n```"); + assert_eq!(result.commands.first().unwrap().raw, "/cmd ```\nbody\n```"); + } - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn command_count_matches_slash_lines( - commands in prop::collection::vec(command_line(), 1..10) - ) { - let input = commands.join("\n"); - let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), commands.len()); - } + // --- Category 10: Trailing newline edge case --- - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn text_only_input_produces_zero_commands( - lines in prop::collection::vec(text_line(), 1..20) - ) { - let input = lines.join("\n"); - let result = parse_document(&input); - prop_assert!(result.commands.is_empty()); - prop_assert!(!result.text_blocks.is_empty()); - } + #[test] + fn trailing_newline_does_not_create_empty_text_block() { + // §2.1 + ADR-NNNN: split_physical_lines pops the trailing empty element. + let result = parse_document("/cmd arg\n"); + assert_eq!(result.commands.len(), 1); + assert!(result.textblocks.is_empty()); + } - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn version_is_always_spec_version(input in "[\\x20-\\x7E]{0,200}") { - let result = parse_document(&input); - prop_assert_eq!(&result.version, crate::domain::SPEC_VERSION); - } + // --- Category 12: Version --- - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn fenced_content_preserved_verbatim( - name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9 !]{1,50}", 1..5) - ) { - let body = body_lines.join("\n"); - let input = format!("/{name} ```\n{body}\n```"); - let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), 1); - prop_assert_eq!(&result.commands[0].arguments.payload, &body); - prop_assert_eq!(&result.commands[0].arguments.mode, &ArgumentMode::Fence); - } + #[test] + fn version_is_set() { + // §8.1: envelope contains version field set to SPEC_VERSION. + let result = parse_document(""); + assert_eq!(result.version, SPEC_VERSION); + } - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn interleaved_text_and_commands_both_captured( - cmd_count in 1usize..5 - ) { - let mut lines = Vec::new(); - for i in 0..cmd_count { - lines.push(format!("text line {i}")); - lines.push(format!("/cmd{i} arg")); - } - let input = lines.join("\n"); - let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), cmd_count); - prop_assert_eq!(result.text_blocks.len(), cmd_count); - } + // --- Property tests --- + // Separated from watch mode via tdd feature flag. - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn unclosed_fence_always_produces_warning( - name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9]{1,30}", 1..5) - ) { - let body = body_lines.join("\n"); - let input = format!("/{name} ```\n{body}"); - let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), 1); - prop_assert_eq!(result.warnings.len(), 1); - } + use proptest::prelude::*; + proptest! { #[test] #[cfg_attr(feature = "tdd", ignore)] - fn text_block_content_matches_non_command_lines( - lines in prop::collection::vec(text_line(), 1..10) - ) { - let input = lines.join("\n"); - let result = parse_document(&input); - prop_assert_eq!(result.text_blocks.len(), 1); - prop_assert_eq!(&result.text_blocks[0].content, &input); - } - - fn continuation_payload_matches_joined_lines( - name in valid_command_name(), - middle_lines in prop::collection::vec("[a-z0-9]{1,20}", 0..5), - final_line in "[a-z0-9]{1,20}" - ) { - let mut input_lines = vec![format!("/{name} start \\")]; - let mut expected_parts = vec!["start".to_string()]; - for line in &middle_lines { - input_lines.push(format!("{line} \\")); - expected_parts.push(line.clone()); - } - input_lines.push(final_line.clone()); - expected_parts.push(final_line); - let input = input_lines.join("\n"); - - let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), 1); - prop_assert_eq!(&result.commands[0].arguments.mode, &ArgumentMode::Continuation); - prop_assert_eq!(&result.commands[0].arguments.payload, &expected_parts.join("\n")); + fn never_panics_on_arbitrary_input(input in "\\PC{0,500}") { + // §8.1 (total function): parser always produces a valid envelope, never panics. + let _ = parse_document(&input); } #[test] #[cfg_attr(feature = "tdd", ignore)] - fn unclosed_continuation_always_produces_warning( - name in valid_command_name(), - middle_lines in prop::collection::vec("[a-z0-9]{1,20}", 0..3) - ) { - let mut input_lines = vec![format!("/{name} start \\")]; - for line in &middle_lines { - input_lines.push(format!("{line} \\")); - } - let input = input_lines.join("\n"); - + fn version_is_always_spec_version(input in "\\PC{0,200}") { + // §8.1: version field is always SPEC_VERSION regardless of input. let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), 1); - prop_assert_eq!(result.warnings.len(), 1); + prop_assert_eq!(result.version, SPEC_VERSION); } - } } diff --git a/parser/src/application/line_classify.rs b/parser/src/application/line_classify.rs index c95be82..1446be9 100644 --- a/parser/src/application/line_classify.rs +++ b/parser/src/application/line_classify.rs @@ -1,13 +1,17 @@ use crate::domain::ArgumentMode; -/// Raw classification of a single input line. +/// Raw classification of a single logical input line. #[derive(Debug, Clone, PartialEq, Eq)] pub enum LineKind { Command(CommandHeader), Text, } -/// Extracted header fields from a line that starts a slash command. +/// Extracted header fields from a line that opens a slash command. +/// +/// `fence_backtick_count` is the length of the backtick run that opened the +/// fence (e.g. 3 for ` ``` `, 4 for ` ```` `). It is 0 for single-line commands. +/// The state machine uses this to recognise the matching closer (§5.2.4). #[derive(Debug, Clone, PartialEq, Eq)] pub struct CommandHeader { pub raw: String, @@ -15,9 +19,10 @@ pub struct CommandHeader { pub header_text: String, pub mode: ArgumentMode, pub fence_lang: Option, + pub fence_backtick_count: usize, } -/// Classify a single line as either a command header or plain text. +/// Classify a single logical line as either a command header or plain text. pub fn classify_line(line: &str) -> LineKind { match try_parse_command(line) { Some(header) => LineKind::Command(header), @@ -25,6 +30,29 @@ pub fn classify_line(line: &str) -> LineKind { } } +/// Find the first run of 3 or more consecutive backtick characters in `s`. +/// +/// Returns `(start_byte_offset, backtick_count)`. +fn find_fence_opener(s: &str) -> Option<(usize, usize)> { + let bytes = s.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'`' { + let start = i; + while i < bytes.len() && bytes[i] == b'`' { + i += 1; + } + let count = i - start; + if count >= 3 { + return Some((start, count)); + } + } else { + i += 1; + } + } + None +} + fn try_parse_command(line: &str) -> Option { let trimmed = line.trim_start(); if !trimmed.starts_with('/') { @@ -33,62 +61,55 @@ fn try_parse_command(line: &str) -> Option { let without_slash = &trimmed[1..]; - // Command name is the first contiguous non-whitespace run. let mut parts = without_slash.splitn(2, char::is_whitespace); let name_raw = parts.next().filter(|n| !n.is_empty())?; - // Command must begin with a lowercase ASCII letter and may contain lowercase letters, digits, and hyphens + // §3.1: [a-z][a-z0-9-]* if !name_raw.starts_with(|c: char| c.is_ascii_lowercase()) { return None; } - if !name_raw .chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-' || c == '_') + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') { return None; } + let name = name_raw.to_string(); let rest = parts.next().unwrap_or("").trim_start(); - // Fence mode: rest starts with ``` - if let Some(stripped) = rest.strip_prefix("```") { - let after_ticks = stripped.trim(); - let lang = after_ticks - .split_whitespace() - .next() - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()); - - return Some(CommandHeader { - raw: line.to_string(), - name, - header_text: rest.to_string(), - mode: ArgumentMode::Fence, - fence_lang: lang, - }); - } - - // Continuation mode: rest ends with trailing backslash. - if rest.ends_with('\\') { - let header_text = rest.trim_end_matches('\\').trim_end().to_string(); + // §5.2.1: detect first occurrence of 3+ backticks anywhere in the args. + if let Some((fence_start, fence_count)) = find_fence_opener(rest) { + let header_text = rest[..fence_start].trim_end().to_string(); + let after_ticks = &rest[fence_start + fence_count..]; + // "trimmed of leading whitespace, if non-empty and consisting of a + // single token (no internal whitespace)" + let after_trimmed = after_ticks.trim(); + let fence_lang = + if !after_trimmed.is_empty() && !after_trimmed.contains(char::is_whitespace) { + Some(after_trimmed.to_string()) + } else { + None + }; return Some(CommandHeader { raw: line.to_string(), name, header_text, - mode: ArgumentMode::Continuation, - fence_lang: None, + mode: ArgumentMode::Fence, + fence_lang, + fence_backtick_count: fence_count, }); } - // Single-line mode (default). + // §5.1: single-line mode — no fence opener present. Some(CommandHeader { raw: line.to_string(), name, header_text: rest.to_string(), mode: ArgumentMode::SingleLine, fence_lang: None, + fence_backtick_count: 0, }) } @@ -96,107 +117,242 @@ fn try_parse_command(line: &str) -> Option { mod tests { use super::*; + // --- find_fence_opener --- + #[test] - fn plain_text_line() { - assert_eq!(classify_line("hello world"), LineKind::Text); + fn find_fence_opener_empty_returns_none() { + // Structural: no input, no opener. + assert!(find_fence_opener("").is_none()); } #[test] - fn bare_slash_is_text() { - assert_eq!(classify_line("/ "), LineKind::Text); + fn find_fence_opener_no_backticks_returns_none() { + // Structural: absence of backticks cannot produce an opener. + assert!(find_fence_opener("hello world").is_none()); } #[test] - fn simple_command() { - let result = classify_line("/cmd some args"); - match result { - LineKind::Command(h) => { - assert_eq!(h.name, "cmd"); - assert_eq!(h.header_text, "some args"); - assert_eq!(h.mode, ArgumentMode::SingleLine); - } - _ => panic!("expected Command"), - } + fn find_fence_opener_one_backtick_returns_none() { + // §5.2.1: opener requires "three or more consecutive backtick characters". + assert!(find_fence_opener("`").is_none()); } #[test] - fn fence_command_with_lang() { - let result = classify_line("/code ```rust"); - match result { - LineKind::Command(h) => { - assert_eq!(h.name, "code"); - assert_eq!(h.mode, ArgumentMode::Fence); - assert_eq!(h.fence_lang.as_deref(), Some("rust")); - } - _ => panic!("expected Command"), - } + fn find_fence_opener_two_backticks_returns_none() { + // §5.2.1: two consecutive backticks do not meet the three-or-more threshold. + assert!(find_fence_opener("``").is_none()); } #[test] - fn continuation_command() { - let result = classify_line("/cmd first part \\"); - match result { - LineKind::Command(h) => { - assert_eq!(h.name, "cmd"); - assert_eq!(h.header_text, "first part"); - assert_eq!(h.mode, ArgumentMode::Continuation); - } - _ => panic!("expected Command"), - } + fn find_fence_opener_three_backticks_at_start() { + // §5.2.1: exactly three backticks form a valid opener at offset 0 with count 3. + assert_eq!(find_fence_opener("```"), Some((0, 3))); } #[test] - fn command_with_no_args() { - let result = classify_line("/help"); - match result { - LineKind::Command(h) => { - assert_eq!(h.name, "help"); - assert_eq!(h.header_text, ""); - assert_eq!(h.mode, ArgumentMode::SingleLine); - } - _ => panic!("expected Command"), - } + fn find_fence_opener_four_backticks_counted_correctly() { + // §5.2.1: "variable-length backtick fence (three or more)" — four ticks + // produce count 4, not two separate runs of two. + assert_eq!(find_fence_opener("````"), Some((0, 4))); } + #[test] - fn slash_alone_no_space_is_text() { - assert_eq!(classify_line("/"), LineKind::Text); + fn find_fence_opener_backticks_after_prefix() { + // §5.2.1: "first occurrence" — offset must reflect the actual position in args. + assert_eq!(find_fence_opener("call_tool write_file ```"), Some((21, 3))); } #[test] - fn command_name_starting_with_uppercase_is_text() { - assert_eq!(classify_line("/Hello"), LineKind::Text); + fn find_fence_opener_returns_first_qualifying_run() { + // §5.2.1: "first occurrence" — a second run of 3+ is ignored. + assert_eq!(find_fence_opener("``` and ```"), Some((0, 3))); } #[test] - fn command_name_starting_with_digit_is_text() { - assert_eq!(classify_line("/2fast"), LineKind::Text); + fn find_fence_opener_two_runs_of_two_returns_none() { + // §5.2.1: non-adjacent pairs do not combine into a qualifying run. + assert!(find_fence_opener("`` ``").is_none()); } #[test] - fn leading_spaces_still_detected_as_command() { - match classify_line(" /cmd arg") { - LineKind::Command(h) => assert_eq!(h.name, "cmd"), - _ => panic!("expected Command"), - } + fn find_fence_opener_tilde_is_not_backtick() { + // §5.2: "Only backtick (`) fences are recognised. Tilde (~) fences are not supported." + assert!(find_fence_opener("~~~").is_none()); } + // --- try_parse_command --- + #[test] - fn fence_without_language() { - match classify_line("/cmd ```") { - LineKind::Command(h) => { - assert_eq!(h.mode, ArgumentMode::Fence); - assert_eq!(h.fence_lang, None); - } - _ => panic!("expected Command"), - } + fn no_slash_returns_none() { + // §3: "first non-whitespace character is `/`" is required for a command. + assert!(try_parse_command("hello").is_none()); } #[test] - fn raw_preserves_original_line() { - match classify_line(" /cmd arg") { - LineKind::Command(h) => assert_eq!(h.raw, " /cmd arg"), - _ => panic!("expected Command"), - } + fn bare_slash_returns_none() { + // §3.2: "a bare `/`" is an invalid slash line treated as text. + assert!(try_parse_command("/").is_none()); + } + + #[test] + fn slash_then_space_returns_none() { + // §3.2: "/ space" — space after slash means no command name follows. + assert!(try_parse_command("/ ").is_none()); + } + + #[test] + fn uppercase_name_returns_none() { + // §3.1: pattern is `[a-z][a-z0-9-]*`; uppercase start does not match. + // §3.2: "/Hello" is an example of an invalid slash line. + assert!(try_parse_command("/Hello").is_none()); + } + + #[test] + fn digit_name_returns_none() { + // §3.1: pattern requires lowercase letter start; digit start does not match. + // §3.2: "/123" is an example of an invalid slash line. + assert!(try_parse_command("/123").is_none()); + } + + #[test] + fn underscore_in_name_returns_none() { + // §3.1: pattern `[a-z][a-z0-9-]*` — hyphens allowed, underscores are not. + assert!(try_parse_command("/cmd_foo").is_none()); + } + + #[test] + fn hyphen_in_name_is_valid() { + // §3.1: "followed by zero or more lowercase ASCII letters, ASCII digits, or hyphens". + let h = try_parse_command("/call-tool args").unwrap(); + assert_eq!(h.name, "call-tool"); + } + + #[test] + fn leading_whitespace_before_slash_is_ignored() { + // §3: "first non-whitespace character is `/`" — leading spaces are stripped. + let h = try_parse_command(" /cmd arg").unwrap(); + assert_eq!(h.name, "cmd"); + } + + #[test] + fn raw_field_preserves_original_line_with_leading_whitespace() { + // §8.2: `raw` must contain "the exact source text … as it appeared in the + // normalized input". Leading whitespace is part of that text. + let h = try_parse_command(" /cmd arg").unwrap(); + assert_eq!(h.raw, " /cmd arg"); + } + + #[test] + fn no_args_produces_empty_header_single_line() { + // §3.3: "The arguments portion may be empty (command with no arguments)." + // §5.1: mode is single-line, header is the empty arguments string. + let h = try_parse_command("/help").unwrap(); + assert_eq!(h.name, "help"); + assert_eq!(h.header_text, ""); + assert_eq!(h.mode, ArgumentMode::SingleLine); + assert_eq!(h.fence_lang, None); + assert_eq!(h.fence_backtick_count, 0); + } + + #[test] + fn single_line_header_text_is_full_args() { + // §3.4: in single-line mode "header and payload contain the same string + // (the full arguments text)". + // §3.3: "The whitespace between the command name and the arguments is + // consumed as a separator and is not included in the arguments string." + let h = try_parse_command("/deploy production --region us-west-2").unwrap(); + assert_eq!(h.header_text, "production --region us-west-2"); + assert_eq!(h.mode, ArgumentMode::SingleLine); + assert_eq!(h.fence_backtick_count, 0); + } + + #[test] + fn fence_at_start_of_args_gives_empty_header() { + // §5.2.1: "Text before the backtick run (trimmed of trailing whitespace) + // becomes arguments.header." When the backtick run is first, header is empty. + let h = try_parse_command("/cmd ```json").unwrap(); + assert_eq!(h.header_text, ""); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, Some("json".to_string())); + assert_eq!(h.fence_backtick_count, 3); + } + + #[test] + fn fence_with_preceding_header_text() { + // §5.2.1: text before the backtick run, trimmed of trailing whitespace, + // becomes header. §3.4: header serves as the dispatch/routing portion. + let h = try_parse_command("/mcp call_tool write_file ```json").unwrap(); + assert_eq!(h.header_text, "call_tool write_file"); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, Some("json".to_string())); + assert_eq!(h.fence_backtick_count, 3); + } + + #[test] + fn fence_without_lang_gives_none() { + // §5.2.1: fence_lang is the optional language identifier — None when absent. + let h = try_parse_command("/cmd ```").unwrap(); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, None); + } + + #[test] + fn fence_lang_none_when_multiple_tokens_after_ticks() { + // §5.2.1: "consisting of a single token (no internal whitespace)" — + // multiple tokens disqualify the language identifier. + let h = try_parse_command("/cmd ``` foo bar").unwrap(); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, None); + } + + #[test] + fn fence_backtick_count_four() { + // §5.2.1: "variable-length backtick fence (three or more)" — count must + // match the actual run length so the closer check can match correctly (§5.2.4). + let h = try_parse_command("/cmd ````json").unwrap(); + assert_eq!(h.fence_backtick_count, 4); + assert_eq!(h.mode, ArgumentMode::Fence); + } + + #[test] + fn tilde_fence_not_recognised() { + // §5.2: "Only backtick (`) fences are recognised. Tilde (~) fences are not supported." + let h = try_parse_command("/cmd ~~~").unwrap(); + assert_eq!(h.mode, ArgumentMode::SingleLine); + assert_eq!(h.header_text, "~~~"); + } + + #[test] + fn fence_opener_anywhere_in_args_is_detected() { + // §5.2.1: "first occurrence of three or more consecutive backtick characters" + // — the opener does not need to be at the start of the arguments. + let h = try_parse_command("/mcp call_tool write_file -c```json").unwrap(); + assert_eq!(h.header_text, "call_tool write_file -c"); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, Some("json".to_string())); + } + + // --- classify_line (public API — composition only) --- + + #[test] + fn plain_text_line_produces_text_kind() { + // §3: lines whose first non-whitespace char is not `/` are not commands. + assert_eq!(classify_line("hello world"), LineKind::Text); + } + + #[test] + fn valid_command_produces_command_kind() { + // §3: valid command lines produce LineKind::Command. + assert!(matches!(classify_line("/cmd args"), LineKind::Command(_))); + } + + #[test] + fn invalid_slash_lines_produce_text_kind() { + // §3.2: invalid slash lines are treated as ordinary text in idle state. + assert_eq!(classify_line("/Hello"), LineKind::Text); + assert_eq!(classify_line("/123"), LineKind::Text); + assert_eq!(classify_line("/"), LineKind::Text); + assert_eq!(classify_line("/ "), LineKind::Text); } // --- Property tests --- @@ -204,18 +360,16 @@ mod tests { use proptest::prelude::*; fn valid_command_name() -> impl Strategy { - "[a-z][a-z0-9\\-]{0,20}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + // §3.1: [a-z][a-z0-9-]* + "[a-z][a-z0-9\\-]{0,20}".prop_map(|s| s) } fn arbitrary_line() -> impl Strategy { prop_oneof![ - // plain text "[a-zA-Z0-9 !.,]{0,80}", - // command-like valid_command_name().prop_flat_map(|name| { "[a-zA-Z0-9 ]{0,40}".prop_map(move |args| format!("/{name} {args}")) }), - // leading whitespace + command valid_command_name().prop_flat_map(|name| { (1usize..5, "[a-zA-Z0-9 ]{0,40}").prop_map(move |(spaces, args)| { format!("{}/{} {}", " ".repeat(spaces), name, args) @@ -225,57 +379,89 @@ mod tests { } proptest! { - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn classify_never_panics(line in "[\\x00-\\x7F]{0,200}") { - let _ = classify_line(&line); - } + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn classify_never_panics(line in "[\\x00-\\x7F]{0,200}") { + // §8.1 (implied): "The parser is a total function: it always produces a + // valid envelope for any input." classify_line must never panic. + let _ = classify_line(&line); + } - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn valid_name_always_produces_command(name in valid_command_name(), args in "[a-z0-9 ]{0,40}") { - let input = format!("/{name} {args}"); - match classify_line(&input) { - LineKind::Command(h) => prop_assert_eq!(h.name, name), - LineKind::Text => panic!("expected Command for input: {input}"), + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn valid_name_always_produces_command( + name in valid_command_name(), + args in "[a-z0-9 ]{0,40}" + ) { + // §3, §3.1: any line whose first non-whitespace is `/` followed by a + // name matching `[a-z][a-z0-9-]*` must produce LineKind::Command. + let input = format!("/{name} {args}"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.name, name), + LineKind::Text => panic!("expected Command for input: {input}"), + } } - } - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn text_without_slash_is_never_command(line in "[a-zA-Z0-9 !.,]{1,80}") { - prop_assert!(matches!(classify_line(&line), LineKind::Text)); - } + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_without_slash_is_never_command(line in "[a-zA-Z0-9 !.,]{1,80}") { + // §3: command detection requires the first non-whitespace char to be `/`. + prop_assert!(matches!(classify_line(&line), LineKind::Text)); + } - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn raw_field_preserves_original_input(line in arbitrary_line()) { - if let LineKind::Command(h) = classify_line(&line) { - prop_assert_eq!(h.raw, line); + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn raw_field_preserves_original_input(line in arbitrary_line()) { + // §8.2: raw must contain "the exact source text … as it appeared in + // the normalized input". + if let LineKind::Command(h) = classify_line(&line) { + prop_assert_eq!(h.raw, line); + } } - } - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn fence_mode_iff_backticks_present(name in valid_command_name(), lang in "[a-z]{0,10}") { - let input = format!("/{name} ```{lang}"); - match classify_line(&input) { - LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Fence), - _ => panic!("expected Command"), + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fence_mode_iff_three_or_more_backticks( + name in valid_command_name(), + lang in "[a-z]{0,10}" + ) { + // §5.2.1: the first occurrence of 3+ consecutive backticks in the args + // triggers fence mode regardless of position. + let input = format!("/{name} ```{lang}"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Fence), + _ => panic!("expected Command"), + } } - } - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn continuation_mode_iff_trailing_backslash( - name in valid_command_name(), - args in "[a-z0-9 ]{1,30}" - ) { - let input = format!("/{name} {args} \\"); - match classify_line(&input) { - LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Continuation), - _ => panic!("expected Command"), + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn single_line_fence_count_is_zero( + name in valid_command_name(), + args in "[a-zA-Z0-9 ]{0,40}" + ) { + // §5.1: single-line mode has no fence opener, so fence_backtick_count + // must be 0. + let input = format!("/{name} {args}"); + let LineKind::Command(h) = classify_line(&input) else { return Ok(()); }; + prop_assert_eq!(h.mode, ArgumentMode::SingleLine); + prop_assert_eq!(h.fence_backtick_count, 0); + } + + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fence_backtick_count_matches_opener_length( + name in valid_command_name(), + extra in 0usize..5 + ) { + // §5.2.1: "The backtick run length is recorded as the fence marker length." + // §5.2.4: the closer must have >= this many backticks. + let ticks = "`".repeat(3 + extra); + let input = format!("/{name} {ticks}json"); + if let LineKind::Command(h) = classify_line(&input) { + prop_assert_eq!(h.fence_backtick_count, 3 + extra); + } } } - } } diff --git a/parser/src/application/line_join.rs b/parser/src/application/line_join.rs new file mode 100644 index 0000000..a4d95b0 --- /dev/null +++ b/parser/src/application/line_join.rs @@ -0,0 +1,357 @@ +/// POSIX-style backslash line joining. +/// +/// Consumes physical lines and produces logical lines. A physical line ending +/// with `\` is joined with the next physical line, separated by a single space, +/// and the backslash is removed. Joining repeats while the accumulated line +/// still ends with `\`. At EOF, a trailing `\` is silently removed. +/// +/// Fence immunity is enforced by the caller: when the state machine enters +/// `InFence`, it calls `next_physical` directly instead of `next_logical`, +/// bypassing the joiner for those lines. +pub struct LogicalLine { + pub text: String, + pub first_physical: usize, + pub last_physical: usize, +} + +pub struct LineJoiner { + lines: Vec, + cursor: usize, +} + +/// Consume the next logical line from `lines` starting at `*cursor`, +/// joining any trailing-backslash continuations. +fn consume_logical(lines: &[String], cursor: &mut usize) -> Option { + if *cursor >= lines.len() { + return None; + } + + let first_physical = *cursor; + let mut text = lines[*cursor].clone(); + *cursor += 1; + + while text.ends_with('\\') { + text.truncate(text.len() - 1); + + if *cursor >= lines.len() { + // Trailing backslash at EOF: silently removed, line stands alone. + break; + } + + text.push(' '); + text.push_str(&lines[*cursor]); + *cursor += 1; + } + + Some(LogicalLine { + text, + first_physical, + last_physical: *cursor - 1, + }) +} + +/// Consume the next raw physical line from `lines` at `*cursor`, +/// bypassing join logic entirely. +fn consume_physical(lines: &[String], cursor: &mut usize) -> Option<(usize, String)> { + if *cursor >= lines.len() { + return None; + } + let idx = *cursor; + let line = lines[*cursor].clone(); + *cursor += 1; + Some((idx, line)) +} + +impl LineJoiner { + pub fn new(lines: Vec) -> Self { + Self { lines, cursor: 0 } + } + + pub fn next_logical(&mut self) -> Option { + consume_logical(&self.lines, &mut self.cursor) + } + + /// Used by the state machine when inside a fenced block. + pub fn next_physical(&mut self) -> Option<(usize, String)> { + consume_physical(&self.lines, &mut self.cursor) + } + + // Implemented but does not look like it's needed by the pipeline. + // TODO: remove if still dead when parser is finished + #[allow(dead_code)] + pub fn is_exhausted(&self) -> bool { + self.cursor >= self.lines.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + + fn lines(xs: &[&str]) -> Vec { + xs.iter().map(|s| s.to_string()).collect() + } + + fn joiner(xs: &[&str]) -> LineJoiner { + LineJoiner::new(lines(xs)) + } + + /// Build a string ending with a backslash without embedding `\\` in literals. + fn bsl(s: &str) -> String { + let mut r = s.to_string(); + r.push('\\'); + r + } + + // --- consume_logical (private free function) --- + + #[test] + fn consume_logical_empty_returns_none() { + // Structural: cursor must not advance and None must be returned on empty input. + let ls = lines(&[]); + let mut cursor = 0; + assert!(consume_logical(&ls, &mut cursor).is_none()); + assert_eq!(cursor, 0); + } + + #[test] + fn consume_logical_no_backslash_passes_through() { + // §2.2: "Lines that do not end with `\` are left unchanged." + let ls = lines(&["/echo hello"]); + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "/echo hello"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 0); + assert_eq!(cursor, 1); + } + + #[test] + fn consume_logical_joins_two_lines() { + // §2.2 steps 1-3: remove trailing backslash, concatenate with next physical + // line separated by a single space. + let ls = vec![bsl("a"), "b".to_string()]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "a b"); + assert_eq!(ll.last_physical, 1); + assert_eq!(cursor, 2); + } + + #[test] + fn consume_logical_trailing_backslash_at_eof_removed() { + // §2.2.2: "If the final physical line ends with a backslash and there is no + // subsequent line to join with, the trailing backslash is removed and the + // line stands alone." + let ls = vec![bsl("a")]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "a"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 0); + } + + #[test] + fn consume_logical_advances_cursor_past_consumed_lines() { + // §2.2.1: the cursor must advance past all physical lines consumed by a + // single logical line so that subsequent calls produce the next logical line. + let ls = vec![bsl("x"), "y".to_string(), "z".to_string()]; + let mut cursor = 0; + consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(cursor, 2); + let ll2 = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll2.text, "z"); + } + + // --- consume_physical (private free function) --- + + #[test] + fn consume_physical_empty_returns_none() { + // Structural: cursor must not advance and None must be returned on empty input. + let ls = lines(&[]); + let mut cursor = 0; + assert!(consume_physical(&ls, &mut cursor).is_none()); + assert_eq!(cursor, 0); + } + + #[test] + fn consume_physical_returns_raw_line_with_backslash() { + // §2.3: inside a fenced block "all physical lines are consumed verbatim", + // including trailing backslashes that would otherwise be join markers. + let ls = vec![bsl("line one"), "line two".to_string()]; + let mut cursor = 0; + let (idx, line) = consume_physical(&ls, &mut cursor).unwrap(); + assert_eq!(idx, 0); + assert!(line.ends_with('\\')); + assert_eq!(cursor, 1); + } + + #[test] + fn consume_physical_does_not_join() { + // §2.3: "A trailing `\` inside a fence is literal content, not a join marker." + let ls = vec![bsl("a"), "b".to_string()]; + let mut cursor = 0; + let (_, line) = consume_physical(&ls, &mut cursor).unwrap(); + assert!(line.ends_with('\\')); + let (_, line2) = consume_physical(&ls, &mut cursor).unwrap(); + assert_eq!(line2, "b"); + } + + // --- LineJoiner impl (delegation + composition) --- + + #[test] + fn next_logical_delegates_correctly() { + // Structural: next_logical must delegate to consume_logical and advance state. + let mut j = joiner(&["/echo hello"]); + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "/echo hello"); + assert!(j.is_exhausted()); + } + + #[test] + fn next_physical_delegates_correctly() { + // Structural: next_physical must delegate to consume_physical and advance state. + let input = vec![bsl("line one"), "line two".to_string()]; + let mut j = LineJoiner::new(input); + let (idx, line) = j.next_physical().unwrap(); + assert_eq!(idx, 0); + assert!(line.ends_with('\\')); + } + + #[test] + fn interleaving_logical_and_physical_shares_cursor() { + // §2.3: the state machine calls next_logical in idle state and next_physical + // in fence state; both must share a single cursor so no lines are skipped + // or double-consumed at the transition boundary. + let input = vec![bsl("/cmd a"), " b".to_string(), "fence body".to_string()]; + let mut j = LineJoiner::new(input); + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "/cmd a b"); + assert_eq!(ll.last_physical, 1); + let (idx, line) = j.next_physical().unwrap(); + assert_eq!(idx, 2); + assert_eq!(line, "fence body"); + assert!(j.is_exhausted()); + } + + #[test] + fn is_exhausted_false_when_lines_remain() { + // Structural: is_exhausted must reflect the current cursor position accurately. + let mut j = joiner(&["a", "b"]); + assert!(!j.is_exhausted()); + j.next_logical(); + assert!(!j.is_exhausted()); + j.next_logical(); + assert!(j.is_exhausted()); + } + + // --- Spec examples --- + + #[test] + fn spec_example_three_physical_lines_join_to_one() { + // §2.2.3 example: three physical lines collapse into one logical line. + // §2.2.1: first_physical and last_physical cover the full physical range. + let input = vec![ + bsl("/mcp call_tool read_file"), + bsl(" --path src/index.ts"), + " --format json".to_string(), + ]; + let mut j = LineJoiner::new(input); + let ll = j.next_logical().unwrap(); + assert_eq!( + ll.text, + "/mcp call_tool read_file --path src/index.ts --format json" + ); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 2); + } + + #[test] + fn spec_example_trailing_backslash_at_eof_space_preserved() { + // §2.2.2 example: "/echo hello \" -> "/echo hello " — the backslash is + // removed but the space before it is part of the content and stays. + let input = vec![bsl("/echo hello ")]; + let mut j = LineJoiner::new(input); + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "/echo hello "); + } + + #[test] + fn fence_closer_line_with_trailing_backslash_joins_normally() { + // §2.2: "The join marker is any backslash character immediately before the + // physical line terminator, regardless of what precedes it. This includes + // lines that serve other syntactic roles, such as a closing fence line + // followed by a trailing backslash." + // §5.2.4: the fence close is detected by the state machine, not the joiner; + // the joiner has no special case here. + let input = vec![bsl("```"), "next line".to_string()]; + let mut j = LineJoiner::new(input); + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "``` next line"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 1); + } + + // --- Property tests --- + + proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn logical_count_lte_physical_count( + ls in prop::collection::vec("[a-zA-Z0-9 ]{0,40}", 0..20) + ) { + // §2.2: joining only reduces or maintains line count, never increases it. + let count = ls.len(); + let mut cursor = 0; + let mut logical_count = 0; + while consume_logical(&ls, &mut cursor).is_some() { + logical_count += 1; + } + prop_assert!(logical_count <= count); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn clean_lines_pass_through_unchanged( + ls in prop::collection::vec("[a-zA-Z0-9 !.,]{1,40}", 1..10) + ) { + // §2.2: "Lines that do not end with `\` are left unchanged." + let expected = ls.clone(); + let mut cursor = 0; + for expected_text in expected { + let ll = consume_logical(&ls, &mut cursor).unwrap(); + prop_assert_eq!(ll.text, expected_text); + } + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn ranges_cover_all_physical_lines( + ls in prop::collection::vec("[a-zA-Z0-9 ]{0,40}", 1..20) + ) { + // §2.2.1: logical lines must partition the physical line sequence without + // gaps or overlaps — every physical line belongs to exactly one logical line. + let count = ls.len(); + let mut cursor = 0; + let mut next_expected = 0usize; + while let Some(ll) = consume_logical(&ls, &mut cursor) { + prop_assert_eq!(ll.first_physical, next_expected); + prop_assert!(ll.first_physical <= ll.last_physical); + next_expected = ll.last_physical + 1; + } + prop_assert_eq!(next_expected, count); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn exhausted_after_consuming_all_logical_lines( + ls in prop::collection::vec("[a-zA-Z0-9 ]{0,40}", 0..20) + ) { + // Structural: is_exhausted must be true once all lines have been consumed. + let mut j = LineJoiner::new(ls); + while j.next_logical().is_some() {} + prop_assert!(j.is_exhausted()); + } + } +} diff --git a/parser/src/application/mod.rs b/parser/src/application/mod.rs index 19c9b5b..5c99c1f 100644 --- a/parser/src/application/mod.rs +++ b/parser/src/application/mod.rs @@ -1,10 +1,13 @@ +// src/application/mod.rs mod command_accumulate; mod command_finalize; mod document_parse; mod line_classify; +mod line_join; +mod normalize; mod text_collect; pub use document_parse::parse_document; #[cfg(test)] -mod tests; +pub mod tests; diff --git a/parser/src/application/normalize.rs b/parser/src/application/normalize.rs new file mode 100644 index 0000000..17030a7 --- /dev/null +++ b/parser/src/application/normalize.rs @@ -0,0 +1,127 @@ +/// Normalize line endings in `input` to LF only. +/// +/// Step 1: replace all CRLF (`\r\n`) with LF (`\n`). +/// Step 2: replace any remaining bare CR (`\r`) with LF (`\n`). +/// +/// All other bytes, including literal `\n` escape sequences inside content, +/// are preserved verbatim. +pub fn normalize(input: &str) -> String { + input.replace("\r\n", "\n").replace('\r', "\n") +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + + // --- Unit tests --- + + #[test] + fn clean_lf_input_is_unchanged() { + // §2.1: LF is the normalized form; input already using LF needs no changes. + let input = "line one\nline two\nline three"; + assert_eq!(normalize(input), input); + } + + #[test] + fn empty_input_is_unchanged() { + // §2.1 (implied): normalize is a total function — empty input produces empty output. + assert_eq!(normalize(""), ""); + } + + #[test] + fn crlf_becomes_lf() { + // §2.1 rule 1: "Replace all \r\n (CRLF) sequences with \n (LF)." + assert_eq!(normalize("a\r\nb"), "a\nb"); + } + + #[test] + fn bare_cr_becomes_lf() { + // §2.1 rule 2: "Replace all remaining \r (bare CR) characters with \n." + assert_eq!(normalize("a\rb"), "a\nb"); + } + + #[test] + fn mixed_endings_all_become_lf() { + // §2.1 rules 1-2: CRLF replacement runs first so the CR in \r\n is not + // re-matched as a bare CR and doubled into \n\n. + assert_eq!(normalize("a\r\nb\rc\nd"), "a\nb\nc\nd"); + } + + #[test] + fn multiple_crlf_all_converted() { + // §2.1 rule 1: all CRLF occurrences are replaced, not just the first. + assert_eq!(normalize("a\r\nb\r\nc"), "a\nb\nc"); + } + + #[test] + fn crlf_at_start_and_end() { + // §2.1 rule 1: replacement applies anywhere in the input, including boundaries. + assert_eq!(normalize("\r\nhello\r\n"), "\nhello\n"); + } + + #[test] + fn only_cr_sequence() { + // §2.1 rule 2: bare CR with no following LF is still converted. + assert_eq!(normalize("\r\r\r"), "\n\n\n"); + } + + #[test] + fn content_without_any_line_endings_unchanged() { + // §2.1: content with no CR or LF characters is unaffected by normalization. + let input = "no line endings here"; + assert_eq!(normalize(input), input); + } + + #[test] + fn literal_backslash_n_in_content_is_not_a_line_terminator() { + // §2.1: "Literal escape sequences inside content (e.g., \n in a JSON string + // \"blah\nblah\") are ordinary characters, not line terminators. They are + // preserved verbatim in the output." + // Rust "\\n" = two chars: backslash + n. Not a real newline. + let input = "before\\nafter"; + assert_eq!(normalize(input), "before\\nafter"); + } + + // --- Property tests --- + + proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn output_never_contains_cr(input in "[\x00-\x7F]{0,500}") { + // §2.1: "After normalization, all line terminators are LF." No CR may remain. + let result = normalize(&input); + prop_assert!(!result.contains('\r')); + } + + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn idempotent(input in "[\x00-\x7F]{0,500}") { + // §2.1 (implied): normalizing already-normalized output is a no-op. + let once = normalize(&input); + let twice = normalize(&once); + prop_assert_eq!(once, twice); + } + + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn clean_input_is_unchanged(input in "[\x20-\x7E\n]{0,500}") { + // §2.1: input containing no CR characters requires no changes. + prop_assert_eq!(normalize(&input), input); + } + + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn lf_count_gte_original_lf_count(input in "[\x00-\x7F]{0,500}") { + // §2.1 rule 2: each bare CR becomes an LF, so LF count can only stay + // the same or increase after normalization. + let original_lf = input.chars().filter(|&c| c == '\n').count(); + let result = normalize(&input); + let result_lf = result.chars().filter(|&c| c == '\n').count(); + prop_assert!(result_lf >= original_lf); + } + } +} diff --git a/parser/src/application/tests/mod.rs b/parser/src/application/tests/mod.rs index cd2f278..0967701 100644 --- a/parser/src/application/tests/mod.rs +++ b/parser/src/application/tests/mod.rs @@ -1 +1,458 @@ -mod proptest; +// Layer 2 integration tests: cross-module composition within the application layer. +// +// These tests exercise combinations of modules that no single file's tests can cover. +// They use only the public API of each module. + +use super::document_parse::parse_document; +use super::line_classify::{LineKind, classify_line}; +use super::line_join::LineJoiner; +use super::normalize::normalize; +use crate::domain::ArgumentMode; + +// --- normalize + line_join --- + +#[test] +fn crlf_continuation_joins_same_as_lf() { + // §2.1 rules 1-2: CRLF and bare CR both normalize to LF before any other processing. + // §2.2: line joining runs after normalization, so a backslash before a CRLF boundary + // must produce the same logical line as the equivalent LF-only input. + let crlf = "/deploy production\\\r\n --region us-west-2"; + let lf = "/deploy production\\\n --region us-west-2"; + + let crlf_lines: Vec = normalize(crlf).split('\n').map(|s| s.to_string()).collect(); + let lf_lines: Vec = normalize(lf).split('\n').map(|s| s.to_string()).collect(); + + let crlf_ll = LineJoiner::new(crlf_lines).next_logical().unwrap(); + let lf_ll = LineJoiner::new(lf_lines).next_logical().unwrap(); + + assert_eq!(crlf_ll.text, lf_ll.text); + assert_eq!(crlf_ll.first_physical, lf_ll.first_physical); + assert_eq!(crlf_ll.last_physical, lf_ll.last_physical); +} + +#[test] +fn bare_cr_continuation_joins_same_as_lf() { + // §2.1 rule 2: remaining bare CR characters are replaced with LF after CRLF removal. + // §2.2: joining is agnostic to the original line-ending style. + let cr = "/deploy production\\\r --region us-west-2"; + let lf = "/deploy production\\\n --region us-west-2"; + + let cr_lines: Vec = normalize(cr).split('\n').map(|s| s.to_string()).collect(); + let lf_lines: Vec = normalize(lf).split('\n').map(|s| s.to_string()).collect(); + + let cr_ll = LineJoiner::new(cr_lines).next_logical().unwrap(); + let lf_ll = LineJoiner::new(lf_lines).next_logical().unwrap(); + + assert_eq!(cr_ll.text, lf_ll.text); +} + +#[test] +fn mixed_crlf_multi_line_join_matches_lf() { + // §2.1: normalization applies to all line endings uniformly. + // §2.2 step 4: joining repeats while the accumulated line still ends with `\`, + // so three physical lines collapse into one logical line regardless of ending style. + // §2.2.1: last_physical must be the zero-based index of the last consumed physical line. + let crlf = "/mcp call_tool read_file\\\r\n --path src/index.ts\\\r\n --format json"; + let lf = "/mcp call_tool read_file\\\n --path src/index.ts\\\n --format json"; + + let crlf_lines: Vec = normalize(crlf).split('\n').map(|s| s.to_string()).collect(); + let lf_lines: Vec = normalize(lf).split('\n').map(|s| s.to_string()).collect(); + + let crlf_ll = LineJoiner::new(crlf_lines).next_logical().unwrap(); + let lf_ll = LineJoiner::new(lf_lines).next_logical().unwrap(); + + assert_eq!(crlf_ll.text, lf_ll.text); + assert_eq!(crlf_ll.first_physical, 0); + assert_eq!(crlf_ll.last_physical, 2); +} + +// --- normalize + line_join + line_classify --- + +#[test] +fn joining_into_fence_opener_spec_5_2_6() { + // §5.2.6: "When backslash joining merges a command line with a line containing + // a fence opener, the fence is detected in the resulting logical line." + // §2.2.1: the logical line maps back to physical lines 0-1. + // §5.2.1: text before the backtick run becomes header; text after becomes fence_lang. + // §5.2.1: fence_backtick_count records the run length for the closer check. + // + // Input (two physical lines): + // /mcp call_tool write_file \ + // ```json + // + // After join: "/mcp call_tool write_file ```json" + // (two spaces: one trailing space before `\` + one joiner separator) + let input = normalize("/mcp call_tool write_file \\\n```json"); + let lines: Vec = input.split('\n').map(|s| s.to_string()).collect(); + let mut joiner = LineJoiner::new(lines); + + let ll = joiner.next_logical().unwrap(); + assert_eq!(ll.text, "/mcp call_tool write_file ```json"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 1); + assert!(joiner.is_exhausted()); + + match classify_line(&ll.text) { + LineKind::Command(h) => { + assert_eq!(h.name, "mcp"); + assert_eq!(h.header_text, "call_tool write_file"); + assert_eq!(h.fence_lang, Some("json".to_string())); + assert_eq!(h.fence_backtick_count, 3); + } + LineKind::Text => panic!("expected fence command, got text"), + } +} + +#[test] +fn joined_logical_line_classifies_as_single_line_command() { + // §2.2: line joining produces logical lines; the state machine then classifies them. + // §5.1: a joined result with no fence opener in the args is a single-line command. + let input = normalize("/deploy production\\\n --region us-west-2"); + let lines: Vec = input.split('\n').map(|s| s.to_string()).collect(); + let ll = LineJoiner::new(lines).next_logical().unwrap(); + + match classify_line(&ll.text) { + LineKind::Command(h) => { + assert_eq!(h.name, "deploy"); + assert_eq!(h.header_text, "production --region us-west-2"); + assert_eq!(h.mode, ArgumentMode::SingleLine); + assert_eq!(h.fence_backtick_count, 0); + } + LineKind::Text => panic!("expected single-line command, got text"), + } +} + +#[test] +fn joined_invalid_slash_line_classifies_as_text() { + // §3.2: a line whose slash is not followed by a valid name is treated as text. + // §2.2: this check runs on the logical line after joining, not on the raw physical lines. + let input = normalize("/Hello world\\\n more args"); + let lines: Vec = input.split('\n').map(|s| s.to_string()).collect(); + let ll = LineJoiner::new(lines).next_logical().unwrap(); + + assert_eq!(classify_line(&ll.text), LineKind::Text); +} + +// --- Full pipeline: document_parse --- +// These tests exercise parse_document end-to-end across multiple modules. +// They verify cross-module composition that no single file's Layer 1 tests can cover. + +// --- Backslash joining through full pipeline (§2.2 + §2.3) --- + +#[test] +fn joined_multi_line_command_through_pipeline() { + // §2.2: backslash continuation joins physical lines into one logical line. + // §5.1: the joined result with no fence opener is single-line mode. + let result = parse_document("/deploy production\\\n --region us-west-2"); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.name, "deploy"); + assert_eq!(cmd.arguments.header, "production --region us-west-2"); + assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); +} + +#[test] +fn trailing_backslash_at_eof_removed_through_pipeline() { + // §2.2.2: trailing backslash at EOF is silently removed. + // The command parses as single-line with the backslash stripped. + let result = parse_document("/echo hello\\"); + assert_eq!(result.commands.len(), 1); + assert_eq!(result.commands.first().unwrap().arguments.payload, "hello"); + assert!(result.warnings.is_empty()); +} + +#[test] +fn fence_immunity_backslash_inside_fence_is_literal() { + // §2.3: trailing backslash inside fence is literal content, not a join marker. + let result = parse_document("/cmd ```\nline one\\\nline two\n```"); + assert_eq!(result.commands.len(), 1); + assert_eq!( + result.commands.first().unwrap().arguments.payload, + "line one\\\nline two" + ); +} + +#[test] +fn joining_into_fence_opener_through_pipeline() { + // §5.2.6: backslash joining merges command line with fence opener line. + // Physical lines 0-1 join into the command header; lines 2-3 are fence body + closer. + let result = parse_document("/mcp call_tool write_file \\\n```json\n{\"path\": \"foo\"}\n```"); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.arguments.header, "call_tool write_file"); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.arguments.payload, "{\"path\": \"foo\"}"); +} + +// --- Text block content with continuation lines (ADR-NNNN) --- + +#[test] +fn text_block_with_continuation_preserves_backslash() { + // ADR-NNNN: text block content stores physical lines. Backslashes are retained + // because text blocks capture pre-join content for round-trip fidelity. + let result = parse_document("hello \\\nworld"); + assert_eq!(result.textblocks.len(), 1); + assert_eq!( + result.textblocks.first().unwrap().content, + "hello \\\nworld" + ); +} + +#[test] +fn text_block_continuation_range_covers_all_physical_lines() { + // ADR-NNNN + §2.2.1: range spans all physical lines consumed by a joined logical line. + let result = parse_document("hello \\\nworld"); + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().range.start_line, 0); + assert_eq!(result.textblocks.first().unwrap().range.end_line, 1); +} + +// --- Appendix A spec examples --- +// Full pipeline assertions matching spec appendix examples exactly. +// These are the Layer 2 "acceptance test" equivalents. + +#[test] +fn spec_a1_single_line_command() { + // Appendix A.1: "/echo hello world" + let result = parse_document("/echo hello world"); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "echo"); + assert_eq!(cmd.raw, "/echo hello world"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 0); + assert_eq!(cmd.arguments.header, "hello world"); + assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); + assert_eq!(cmd.arguments.fence_lang, None); + assert_eq!(cmd.arguments.payload, "hello world"); +} + +#[test] +fn spec_a2_joined_multi_line_command() { + // Appendix A.2: three physical lines joined into one command. + let input = "/deploy production \\\n --region us-west-2 \\\n --canary"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "deploy"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 2); + assert_eq!( + cmd.arguments.header, + "production --region us-west-2 --canary" + ); + assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); + assert_eq!( + cmd.arguments.payload, + "production --region us-west-2 --canary" + ); + // §8.2: raw contains physical lines with backslashes and \n separators. + assert_eq!( + cmd.raw, + "/deploy production \\\n --region us-west-2 \\\n --canary" + ); +} + +#[test] +fn spec_a3_fenced_command_with_header() { + // Appendix A.3: "/mcp call_tool write_file ```json\n{...}\n```" + let input = "/mcp call_tool write_file ```json\n{\"path\": \"src/index.ts\"}\n```"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 2); + assert_eq!(cmd.arguments.header, "call_tool write_file"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.arguments.payload, "{\"path\": \"src/index.ts\"}"); +} + +#[test] +fn spec_a4_backslash_join_into_fence() { + // Appendix A.4: four physical lines, join merges 0+1 into command+fence opener. + let input = "/mcp call_tool write_file \\\n```json\n{\"path\": \"foo\"}\n```"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 3); + assert_eq!(cmd.arguments.header, "call_tool write_file"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.arguments.payload, "{\"path\": \"foo\"}"); + // §8.2: raw includes all physical lines from opener through closer. + assert_eq!( + cmd.raw, + "/mcp call_tool write_file \\\n```json\n{\"path\": \"foo\"}\n```" + ); +} + +#[test] +fn spec_a5_text_blocks_and_multiple_commands() { + // Appendix A.5: text, two commands, trailing text. + let input = "Welcome to the deployment system.\n\n/deploy staging\n/notify team --channel ops\nDeployment complete."; + let result = parse_document(input); + + // Two commands in order. + assert_eq!(result.commands.len(), 2); + assert_eq!(result.commands.first().unwrap().id, "cmd-0"); + assert_eq!(result.commands.first().unwrap().name, "deploy"); + assert_eq!( + result.commands.first().unwrap().arguments.payload, + "staging" + ); + assert_eq!(result.commands.first().unwrap().range.start_line, 2); + assert_eq!(result.commands.get(1).unwrap().id, "cmd-1"); + assert_eq!(result.commands.get(1).unwrap().name, "notify"); + assert_eq!( + result.commands.get(1).unwrap().arguments.payload, + "team --channel ops" + ); + assert_eq!(result.commands.get(1).unwrap().range.start_line, 3); + + // Two text blocks in order. + assert_eq!(result.textblocks.len(), 2); + assert_eq!(result.textblocks.first().unwrap().id, "text-0"); + assert_eq!( + result.textblocks.first().unwrap().content, + "Welcome to the deployment system.\n" + ); + assert_eq!(result.textblocks.first().unwrap().range.start_line, 0); + assert_eq!(result.textblocks.first().unwrap().range.end_line, 1); + assert_eq!(result.textblocks.get(1).unwrap().id, "text-1"); + assert_eq!( + result.textblocks.get(1).unwrap().content, + "Deployment complete." + ); + assert_eq!(result.textblocks.get(1).unwrap().range.start_line, 4); + assert_eq!(result.textblocks.get(1).unwrap().range.end_line, 4); +} + +#[test] +fn spec_a7_unclosed_fence() { + // Appendix A.7: fence never closed, command emitted with partial payload + warning. + let input = "/mcp call_tool ```json\n{\"incomplete\": true}"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.arguments.header, "call_tool"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.arguments.payload, "{\"incomplete\": true}"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 1); + + assert_eq!(result.warnings.len(), 1); + assert_eq!(result.warnings.first().unwrap().wtype, "unclosed-fence"); + assert_eq!(result.warnings.first().unwrap().start_line, Some(0)); +} + +#[test] +fn spec_a8_closing_fence_with_trailing_backslash() { + // Appendix A.8: "```\" is NOT a valid closer (not solely backticks after trim). + // The fence never closes; all remaining lines become payload. Warning emitted. + let input = + "/mcp call_tool write_file -c \\\n```json\n{\"path\": \"foo\"}\n```\\\n\\\nproduction"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + // Fence never closes: lines 2-5 are all payload. + assert!(cmd.arguments.payload.contains("{\"path\": \"foo\"}")); + + assert_eq!(result.warnings.len(), 1); + assert_eq!(result.warnings.first().unwrap().wtype, "unclosed-fence"); +} + +#[test] +fn spec_a9_proper_fence_close_followed_by_content() { + // Appendix A.9: fence closes on line 3, lines 4-5 join into text block. + let input = + "/mcp call_tool write_file -c \\\n```json\n{\"path\": \"foo\"}\n```\n\\\nproduction"; + let result = parse_document(input); + + // One command with closed fence. + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.arguments.header, "call_tool write_file -c"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.arguments.payload, "{\"path\": \"foo\"}"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 3); + + // Lines 4-5 join into "production" text block. + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().content, "\\\nproduction"); + assert_eq!(result.textblocks.first().unwrap().range.start_line, 4); + assert_eq!(result.textblocks.first().unwrap().range.end_line, 5); +} + +// --- Property tests (Layer 2) --- +// Cross-module invariants over arbitrary input. + +use proptest::prelude::*; + +proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn never_panics_on_arbitrary_input(input in "\\PC{0,500}") { + // §8.1 (total function): parser always produces a valid envelope, never panics. + let _ = parse_document(&input); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn version_is_always_spec_version(input in "\\PC{0,200}") { + // §8.1: version field is always SPEC_VERSION regardless of input. + let result = parse_document(&input); + prop_assert_eq!(&result.version, crate::domain::SPEC_VERSION); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_only_input_produces_zero_commands( + lines in prop::collection::vec("[^/\\n\\r][^\\n\\r]{0,40}", 1..10) + ) { + // §6 + §7: input with no slash-prefixed lines produces no commands. + // Lines are guaranteed not to start with '/' so none can be commands. + let input = lines.join("\n"); + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), 0); + prop_assert!(!result.textblocks.is_empty()); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fenced_content_preserved_verbatim(body in "[^`\\n\\r]{1,80}") { + // §5.2.2: fence body is verbatim, no joining, no escaping. + let input = format!("/cmd ```\n{}\n```", body); + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), 1); + prop_assert_eq!(&result.commands.first().unwrap().arguments.payload, &body); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn unclosed_fence_always_produces_warning(body in "[^`]{0,80}") { + // §5.2.5: any fence reaching EOF without closer produces exactly one warning. + let input = format!("/cmd ```\n{}", body); + let result = parse_document(&input); + prop_assert_eq!(result.commands.len(), 1); + prop_assert_eq!(result.warnings.len(), 1); + prop_assert_eq!(&result.warnings.first().unwrap().wtype, "unclosed-fence"); + } +} diff --git a/parser/src/application/text_collect.rs b/parser/src/application/text_collect.rs index 21a212b..38856a5 100644 --- a/parser/src/application/text_collect.rs +++ b/parser/src/application/text_collect.rs @@ -1,6 +1,6 @@ use crate::domain::{LineRange, TextBlock}; -/// In-progress accumulation of contiguous non-command lines. +/// Accumulated state for a text block being built line by line. #[derive(Debug, Clone)] pub struct PendingText { pub start_line: usize, @@ -8,6 +8,7 @@ pub struct PendingText { pub lines: Vec, } +/// Start a new pending text block at the given physical line index. pub fn start_text(line_index: usize, line: &str) -> PendingText { PendingText { start_line: line_index, @@ -16,15 +17,22 @@ pub fn start_text(line_index: usize, line: &str) -> PendingText { } } +/// Append one more physical line to an in-progress text block. pub fn append_text(mut text: PendingText, line_index: usize, line: &str) -> PendingText { text.end_line = line_index; text.lines.push(line.to_string()); text } -pub fn finalize_text(text: PendingText) -> TextBlock { +/// Finalize a pending text block, assigning it the given sequential id. +/// +/// The caller supplies a zero-based counter; this function formats it as +/// `text-{id}` per §7. The caller is responsible for incrementing the counter +/// after each call so that IDs are unique and sequential within an envelope. +pub fn finalize_text(text: PendingText, id: usize) -> TextBlock { let content = text.lines.join("\n"); TextBlock { + id: format!("text-{id}"), range: LineRange { start_line: text.start_line, end_line: text.end_line, @@ -36,135 +44,191 @@ pub fn finalize_text(text: PendingText) -> TextBlock { #[cfg(test)] mod tests { use super::*; + use proptest::prelude::*; + + // --- start_text --- #[test] - fn single_line_text_block() { - let pending = start_text(0, "hello"); - let block = finalize_text(pending); - assert_eq!(block.content, "hello"); - assert_eq!(block.range.start_line, 0); - assert_eq!(block.range.end_line, 0); + fn start_text_single_line_range() { + // §6: "Text blocks use physical line numbers for their range." + // A freshly started block spans exactly one physical line. + let pt = start_text(3, "hello"); + assert_eq!(pt.start_line, 3); + assert_eq!(pt.end_line, 3); + assert_eq!(pt.lines, vec!["hello"]); } #[test] - fn multi_line_text_block() { - let text = start_text(2, "first"); - let text = append_text(text, 3, "second"); - let text = append_text(text, 4, "third"); - let block = finalize_text(text); - assert_eq!(block.content, "first\nsecond\nthird"); - assert_eq!(block.range.start_line, 2); - assert_eq!(block.range.end_line, 4); + fn start_text_preserves_content_verbatim() { + // §6: "Text block content preserves the original lines." Leading/trailing + // whitespace and punctuation are not modified. + let pt = start_text(0, " indented line! "); + assert_eq!(pt.lines[0], " indented line! "); } + // --- append_text --- + #[test] - fn empty_line_preserved_in_content() { - let text = start_text(0, "before"); - let text = append_text(text, 1, ""); - let text = append_text(text, 2, "after"); - let block = finalize_text(text); - assert_eq!(block.content, "before\n\nafter"); + fn append_text_updates_end_line_only() { + // §6: "Text blocks use physical line numbers for their range." Appending + // advances end_line but must never change start_line. + let pt = start_text(2, "first"); + let pt = append_text(pt, 3, "second"); + assert_eq!(pt.start_line, 2); + assert_eq!(pt.end_line, 3); } #[test] - fn whitespace_only_line_preserved() { - let pending = start_text(5, " "); - let block = finalize_text(pending); - assert_eq!(block.content, " "); - assert_eq!(block.range.start_line, 5); + fn append_text_accumulates_lines_in_order() { + // §6: "Consecutive non-command lines form a single text block." Lines must + // appear in document order. + let pt = start_text(0, "a"); + let pt = append_text(pt, 1, "b"); + let pt = append_text(pt, 2, "c"); + assert_eq!(pt.lines, vec!["a", "b", "c"]); } #[test] - fn append_updates_end_line_only() { - let text = start_text(3, "first"); - let text = append_text(text, 4, "second"); - assert_eq!(text.start_line, 3); - assert_eq!(text.end_line, 4); + fn append_text_preserves_blank_lines() { + // §6: "Blank lines that are part of a text region are included in the + // text block content." + let pt = start_text(0, "before"); + let pt = append_text(pt, 1, ""); + let pt = append_text(pt, 2, "after"); + assert_eq!(pt.lines, vec!["before", "", "after"]); } - // --- Property tests --- + #[test] + fn append_text_preserves_whitespace_only_lines() { + // §6: blank lines (including whitespace-only lines) are included verbatim. + let pt = start_text(0, "text"); + let pt = append_text(pt, 1, " "); + assert_eq!(pt.lines[1], " "); + } - use proptest::prelude::*; + // --- finalize_text --- - fn valid_command_name() -> impl Strategy { - "[a-z][a-z0-9\\-]{0,20}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + #[test] + fn finalize_text_id_format() { + // §7: "Text blocks are assigned IDs independently in the same manner: + // text-0, text-1, text-2, and so on." + let block = finalize_text(start_text(0, "x"), 0); + assert_eq!(block.id, "text-0"); + + let block = finalize_text(start_text(0, "x"), 7); + assert_eq!(block.id, "text-7"); } - fn arbitrary_line() -> impl Strategy { - prop_oneof![ - // plain text - "[a-zA-Z0-9 !.,]{0,80}", - // command-like - valid_command_name().prop_flat_map(|name| { - "[a-zA-Z0-9 ]{0,40}".prop_map(move |args| format!("/{name} {args}")) - }), - // leading whitespace + command - valid_command_name().prop_flat_map(|name| { - (1usize..5, "[a-zA-Z0-9 ]{0,40}").prop_map(move |(spaces, args)| { - format!("{}/{} {}", " ".repeat(spaces), name, args) - }) - }), - ] + #[test] + fn finalize_text_content_lines_joined_with_newline() { + // §6: "Text block content preserves the original lines joined with `\n` + // separators." + let pt = start_text(0, "line one"); + let pt = append_text(pt, 1, "line two"); + let pt = append_text(pt, 2, "line three"); + let block = finalize_text(pt, 0); + assert_eq!(block.content, "line one\nline two\nline three"); } - use crate::{ - application::line_classify::{LineKind, classify_line}, - domain::ArgumentMode, - }; - proptest! { - #[test] - #[cfg_attr(feature = "tdd", ignore)] + #[test] + fn finalize_text_single_line_no_trailing_newline() { + // §6 (implied): a single-line block has no separator — content equals + // the line itself with no added newline. + let block = finalize_text(start_text(0, "hello"), 0); + assert_eq!(block.content, "hello"); + } + #[test] + fn finalize_text_range_covers_physical_lines() { + // §6: "Text blocks use physical line numbers for their range." The range + // must span from the first to the last physical line consumed. + let pt = start_text(4, "a"); + let pt = append_text(pt, 5, "b"); + let block = finalize_text(pt, 1); + assert_eq!(block.range.start_line, 4); + assert_eq!(block.range.end_line, 5); + } - fn classify_never_panics(line in "[\\x00-\\x7F]{0,200}") { - let _ = classify_line(&line); - } + #[test] + fn finalize_text_blank_line_in_content() { + // §6: "Blank lines that are part of a text region are included in the + // text block content." + let pt = start_text(0, "before"); + let pt = append_text(pt, 1, ""); + let pt = append_text(pt, 2, "after"); + let block = finalize_text(pt, 0); + assert_eq!(block.content, "before\n\nafter"); + } - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn valid_name_always_produces_command(name in valid_command_name(), args in "[a-z0-9 ]{0,40}") { - let input = format!("/{name} {args}"); - match classify_line(&input) { - LineKind::Command(h) => prop_assert_eq!(h.name, name), - LineKind::Text => panic!("expected Command for input: {input}"), - } - } + #[test] + fn finalize_text_incremental_emission_possible() { + // §9.2: "A conforming implementation must support finalizing each … text + // block as soon as its last physical line has been consumed." finalize_text + // requires only the accumulated PendingText — no global state. + let block = finalize_text(start_text(10, "standalone"), 3); + assert_eq!(block.id, "text-3"); + assert_eq!(block.range.start_line, 10); + assert_eq!(block.content, "standalone"); + } + + // --- Property tests --- + proptest! { #[test] #[cfg_attr(feature = "tdd", ignore)] - fn text_without_slash_is_never_command(line in "[a-zA-Z0-9 !.,]{1,80}") { - prop_assert!(matches!(classify_line(&line), LineKind::Text)); + fn id_matches_text_n_pattern(id in 0usize..1000) { + // §7: IDs are "text-0, text-1, text-2, and so on" — the pattern is + // always `text-` followed by the decimal zero-based index. + let block = finalize_text(start_text(0, "x"), id); + prop_assert_eq!(block.id, format!("text-{id}")); } #[test] #[cfg_attr(feature = "tdd", ignore)] - fn raw_field_preserves_original_input(line in arbitrary_line()) { - if let LineKind::Command(h) = classify_line(&line) { - prop_assert_eq!(h.raw, line); - } + fn content_equals_lines_joined_with_newline( + lines in prop::collection::vec("[a-zA-Z0-9 !.,]{0,60}", 1..20) + ) { + // §6: "Text block content preserves the original lines joined with + // `\n` separators." + let expected = lines.join("\n"); + let pt = lines.iter().enumerate().fold( + start_text(0, &lines[0]), + |acc, (i, line)| if i == 0 { acc } else { append_text(acc, i, line) }, + ); + let block = finalize_text(pt, 0); + prop_assert_eq!(block.content, expected); } #[test] #[cfg_attr(feature = "tdd", ignore)] - fn fence_mode_iff_backticks_present(name in valid_command_name(), lang in "[a-z]{0,10}") { - let input = format!("/{name} ```{lang}"); - match classify_line(&input) { - LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Fence), - _ => panic!("expected Command"), + fn range_covers_exactly_the_lines_provided( + start in 0usize..100, + extra in 0usize..20 + ) { + // §6: "Text blocks use physical line numbers for their range." The range + // must be [start, start + extra] with no gaps or extensions. + let mut pt = start_text(start, "first"); + for i in 1..=extra { + pt = append_text(pt, start + i, "line"); } + let block = finalize_text(pt, 0); + prop_assert_eq!(block.range.start_line, start); + prop_assert_eq!(block.range.end_line, start + extra); } #[test] #[cfg_attr(feature = "tdd", ignore)] - fn continuation_mode_iff_trailing_backslash( - name in valid_command_name(), - args in "[a-z0-9 ]{1,30}" + fn append_never_changes_start_line( + start in 0usize..100, + lines in prop::collection::vec("[a-zA-Z]{1,20}", 1..10) ) { - let input = format!("/{name} {args} \\"); - match classify_line(&input) { - LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Continuation), - _ => panic!("expected Command"), + // §6: start_line is fixed at the first physical line of the block; + // subsequent appends must not modify it. + let mut pt = start_text(start, "first"); + for (i, line) in lines.iter().enumerate() { + pt = append_text(pt, start + i + 1, line); } + prop_assert_eq!(pt.start_line, start); } } } diff --git a/parser/src/domain/mod.rs b/parser/src/domain/mod.rs index 0453bad..1796666 100644 --- a/parser/src/domain/mod.rs +++ b/parser/src/domain/mod.rs @@ -1,7 +1,8 @@ mod errors; mod types; -pub use errors::Warning; pub use types::{ - ArgumentMode, Command, CommandArguments, LineRange, ParseResult, TextBlock, SPEC_VERSION, + ArgumentMode, Command, CommandArguments, LineRange, ParseResult, SPEC_VERSION, TextBlock, }; + +pub use errors::Warning; diff --git a/parser/src/lib.rs b/parser/src/lib.rs index 0001c4b..3cd24c2 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -1,5 +1,11 @@ mod application; -pub mod domain; +mod domain; -// Public API re-export +// Domain types +pub use domain::{ + ArgumentMode, Command, CommandArguments, LineRange, ParseResult, SPEC_VERSION, TextBlock, + Warning, +}; + +// Engine entry point pub use application::parse_document; From a7c47fa1afc3e68a329d98923a474e7834e5d16a Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Thu, 19 Mar 2026 20:11:01 -0600 Subject: [PATCH 06/13] docs(parser): spec 0.3.0 and things --- .gitignore | 8 +- README.md | 128 +++++ ...sitory-structure-and-project-boundaries.md | 116 ++++ ...-build-tooling-and-toolchain-management.md | 84 +++ ...lt-output-with-pretty-json-array-option.md | 109 ++++ docs/adr parse pipeline.md | 267 +++++++++ docs/parser-architecture.md | 530 ++++++++++++++++++ docs/slash-parser CLI Specification.md | 2 +- 8 files changed, 1240 insertions(+), 4 deletions(-) create mode 100644 README.md create mode 100644 docs/0006-repository-structure-and-project-boundaries.md create mode 100644 docs/0007-build-tooling-and-toolchain-management.md create mode 100644 docs/0009-jsonl-default-output-with-pretty-json-array-option.md create mode 100644 docs/adr parse pipeline.md create mode 100644 docs/parser-architecture.md diff --git a/.gitignore b/.gitignore index 7b057d7..854b856 100644 --- a/.gitignore +++ b/.gitignore @@ -18,10 +18,12 @@ Thumbs.db *.swo # Fuzz corpus artifacts (keep corpus seeds in VCS) -/fuzz/artifacts/ -/fuzz/hfuzz_* -/fuzz/fuzz_* +fuzz/artifacts/ +fuzz/hfuzz_* +fuzz/fuzz_* +# no fuzz in root +fuzz # Environment & Secrets diff --git a/README.md b/README.md new file mode 100644 index 0000000..9bce23e --- /dev/null +++ b/README.md @@ -0,0 +1,128 @@ +# slash-parser + +A Rust + WebAssembly slash command parser. Extracts `/commands` from text with support for +single-line arguments, multi-line continuation (`/`), and fenced code blocks. + +## Quick start + +Managed by [Moon](https://moonrepo.dev). All tool versions are pinned via `.prototools`. + +```bash +# Run all checks on affected projects +moon run :lint :fmt-check :test + +# Build CLI +moon run slash-parse:build + +# Build WASM for JS +moon run slash-parser-js:wasm-build +``` + +Or with cargo directly: + +```bash +cargo test --workspace +cargo clippy --workspace -- -D warnings +cargo fmt --all --check +``` + +## CLI usage + +```bash +# Single file +slash-parse ./prompt.md + +# Multiple files (streaming JSONL) +slash-parse ./a.md ./b.md + +# Stdin +echo "/help me" | slash-parse + +# With context injection +slash-parse -c user=tom -c ./config.toml ./prompt.md + +# Pretty output (JSON array) +slash-parse -p -c '{"user":"tom"}' ./prompt.md +``` + +### Context (-c flag) + +The `-c` flag is repeatable. Each value is detected as: + +1. **File path** — `.json`, `.toml`, `.env`, or plain key=value files +2. **Inline JSON** — starts with `{` +3. **Inline key=value** — contains `=` + +Values merge left to right. File source always overrides `-c source=...`. + +### Exit codes + +| Code | Meaning | +| ---- | --------------------------- | +| 0 | Success | +| 1 | File/context error | +| 2 | Usage error (bad arguments) | + +## Input format + +````text +/command single-line argument + +/command multi-line / +continuation here + +/command fenced block ```json +{"key": "value"} +```` + +```` +## Output + +JSON conforming to the `SlashParseResult` schema: + +```json +{ + "version": "0.1.0", + "context": { "source": "file.md", "user": "tom" }, + "commands": [ + { + "id": "cmd-0", + "name": "command", + "raw": "/command argument", + "range": { "start_line": 0, "end_line": 0 }, + "arguments": { + "header": "argument", + "mode": "single-line", + "payload": "argument" + }, + "children": [] + } + ] +} +```` + +## JS/TS usage + +```typescript +import init, { parseText, parseTextWithContext, version } from '@slash-parser/js' + +await init() + +const result = parseTextWithContext(sourceText, { source: 'file://path/to/doc.md', user: 'tom' }) + +console.log(result.commands) +``` + +TypeScript types are provided in `types/index.d.ts`. + +## Testing + +- **Unit tests**: 63 tests covering all argument modes, edge cases, CRLF normalization +- **Integration tests**: 17 CLI tests covering file/stdin/mixed input, context injection, pretty + output, error codes +- **Property tests**: proptest strategies for no-panic guarantees and invariant verification +- **Fuzz harness**: `cargo fuzz run fuzz_parse` for arbitrary input crash testing + +## License + +MIT diff --git a/docs/0006-repository-structure-and-project-boundaries.md b/docs/0006-repository-structure-and-project-boundaries.md new file mode 100644 index 0000000..f5f7bf4 --- /dev/null +++ b/docs/0006-repository-structure-and-project-boundaries.md @@ -0,0 +1,116 @@ +--- +number: 6 +title: Repository Structure and Project Boundaries +date: 2026-03-14 +status: accepted +--- + +# 6. Repository Structure and Project Boundaries + +Date: 2026-03-14 + +## Status + +Accepted + +## Context + +The slash project encompasses a Rust parser core, WASM build targets, language-specific SDKs, a CLI +tool (riff), distribution packaging, a documentation website, and a formal spec. + +Deep directory nesting (e.g., `core/parser/`, `sdk/slash-python/`, `cli/riff/`) adds navigation +friction. A flat layout with a strict naming convention communicates category membership and +dependency tiers without requiring a nested filesystem hierarchy. + +Naming: "slash" is the format and project name (80s rock reference). "riff" is the CLI tool (guitar +riff; it reads stdin, writes stdout, stays simple). All SDK packages are prefixed `slash-` for +discoverability in package registries. All CLI packaging projects are prefixed `riff-`. + +Because this is a polyglot monorepo, we need strict architectural constraints to ensure SDKs don't +accidentally import the CLI, and that WASM builds don't depend on downstream consumers. The +mechanisms for enforcing these constraints (e.g., build system tooling) will be handled in a +separate tooling ADR. + +## Decision + +The repository uses a flat, single-level project layout. Logical grouping and dependency enforcement +are handled via project metadata and tags, not directory depth. + +### Project Layout + +```text +wit/ core WIT interface definitions (single source of truth) +parser/ core Rust library, internal to repo, not published +wasm-javascript/ wasm wasm-bindgen module for JS/TS runtimes +wasm-wasi/ wasm WASI component for polyglot SDK consumption +slash-web/ sdk Browser runtime SDK (depends on wasm-javascript) +slash-javascript/ sdk ESM server-side SDK (depends on wasm-javascript) +slash-python/ sdk Python SDK (depends on wasm-wasi) +slash-ruby/ sdk Ruby SDK (depends on wasm-wasi) +slash-php/ sdk PHP SDK (depends on wasm-wasi) +slash-elixir/ sdk Elixir SDK (depends on wasm-wasi) +slash-ocaml/ sdk OCaml SDK (depends on wasm-wasi) +slash-haskell/ sdk Haskell SDK (depends on wasm-wasi) +slash-dart/ sdk Dart SDK (depends on wasm-wasi) +slash-java/ sdk Java SDK (depends on wasm-wasi) +slash-go/ sdk Go SDK (depends on wasm-wasi) +slash-zig/ sdk Zig SDK (native FFI or wasm-wasi) +slash-rust/ sdk Rust SDK, thin published crate wrapping parser +riff-cli/ cli CLI binary (depends on slash-rust) +riff-deb/ pkg Debian package +riff-rpm/ pkg RPM package +riff-oci/ pkg OCI container image +riff-proto/ pkg Proto toolchain plugin +website/ docs Static site for documentation and promotion +docs/ docs Formal specifications +docs/adrs/ docs Architecture Decision Records +``` + +### Module Boundaries + +Projects are grouped into six distinct tags/tiers to define architectural boundaries. Build tooling +must enforce that dependencies only flow downward through these layers: + +- `core`: the internal parser library and WIT definitions. Depends on nothing. +- `wasm`: WASM build targets that compile `parser`. Can only depend on `core`. +- `sdk`: language-specific packages wrapping a WASM module (or native FFI). Can only depend on + `wasm` and `core`. +- `cli`: the riff binary. Can only depend on `sdk` and `core`. +- `pkg`: distribution packaging for riff. Can only depend on `cli`. +- `docs`: website, specifications, and ADRs. Can only depend on `docs`. + +### Dependency Flow and The WIT Contract + +```text +wit -> parser +parser -> wasm-javascript -> slash-javascript, slash-web +parser -> wasm-wasi -> slash-python, slash-ruby, slash-php, ... +parser -> slash-rust -> riff-cli -> riff-deb, riff-rpm, riff-oci, riff-proto +docs -> website +``` + +The `wit/` directory contains the WebAssembly Interface Types definitions, acting as the single +source of truth for the parser's API surface. Language SDKs generate types and thin wrappers based +on this contract. + +### Internal vs Published Code + +`parser/` is an internal Rust crate. It is consumed only by `wasm-javascript`, `wasm-wasi`, and +`slash-rust` within this repo. External Rust consumers use the `slash-rust` crate, which provides +the stable public API. + +To keep the repository root clean, all fuzzing, profiling, and testing infrastructure for the core +engine must live entirely within the `parser/` directory, avoiding root-level artifact clutter. + +## Consequences + +- Every project is one `cd` from root. No navigating through intermediate grouping directories. +- Prefix conventions (`slash-*`, `riff-*`, `wasm-*`) make filesystem navigation and glob-based + targeting self-documenting. +- The `parser` crate stays internal, giving freedom to change its API without semver concerns. Only + `slash-rust` carries the public contract. +- Adding a new language SDK requires only a single new root directory; no restructuring is needed. +- Clear structural boundaries physically prevent architectural regressions (e.g., an SDK cannot + accidentally depend on the CLI). +- The specification and ADRs live in `docs/` alongside the code. The website project pulls from + `docs/` at build time, ensuring documentation is version-controlled with the codebase. diff --git a/docs/0007-build-tooling-and-toolchain-management.md b/docs/0007-build-tooling-and-toolchain-management.md new file mode 100644 index 0000000..a9d58e6 --- /dev/null +++ b/docs/0007-build-tooling-and-toolchain-management.md @@ -0,0 +1,84 @@ +--- +number: 7 +title: Build Tooling and Toolchain Management +date: 2026-03-14 +status: accepted +--- + +# 7. Build Tooling and Toolchain Management + +Date: 2026-03-14 + +## Status + +Accepted + +## Context + +The slash project is a polyglot monorepo containing Rust (core parser), WebAssembly components, and +multiple language-specific SDKs. Managing this requires a build system that can orchestrate a +directed acyclic graph (DAG) of tasks across different language ecosystems. We also need tooling to +physically enforce the architectural boundaries established in ADR 6, both between projects (macro) +and within projects (micro). + +Furthermore, we must prevent "works on my machine" issues by guaranteeing that all contributors and +CI runners use the exact same toolchain versions, while preventing our monorepo orchestrator from +getting into a tug-of-war with workspace-aware tools like Cargo and pnpm. + +## Decision + +We will use Moon as our primary task orchestrator and monorepo manager, alongside its companion tool +`proto` for toolchain management. Moon will wrap, rather than replace, the native tools for each +language ecosystem. + +### Boundary Enforcement (Macro vs Micro) + +Tooling will explicitly enforce the architectural constraints defined in ADR 6 at two levels: + +1. **Macro-Boundaries (Project Level):** We use Moon's `tagRelationships` in `.moon/workspace.yml`. + Projects are tagged (`core`, `wasm`, `sdk`, `cli`, `pkg`, `docs`). If an `sdk` project attempts + to depend on a `cli` project, Moon will fail the build graph generation. This physically prevents + architectural regressions across the monorepo. + +2. **Micro-Boundaries (File Level):** Within individual projects (like the `parser/` crate), we use + ecosystem-specific tools configured as Moon lint tasks to enforce Clean Architecture: + - **Rust:** We use `clippy.toml` restrictions (`disallowed-methods`, `disallowed-types`) to + prevent Infrastructure IO and WASM types from leaking into the `src/domain/` and + `src/application/` modules. We may adopt `tach` in the future for stricter folder-to-folder DAG + enforcement. + - **TypeScript:** We use `eslint-plugin-boundaries` to prevent illegal cross-folder imports. + +### Toolchain Pinning (Proto) + +All toolchain versions are strictly pinned at the repository root: + +- `.prototools` acts as the single source of truth for exact version numbers (`rust = "1.85.0"`). +- `.moon/toolchains.yml` handles configuration behavior (e.g., auto-installing the `wasm32-wasip1` + target) but explicitly omits version numbers to prevent configuration drift. + +### Virtual Workspaces and Task Scoping + +To support shared lockfiles and efficient caching, we use root-level virtual workspaces (a root +`Cargo.toml` and `pnpm-workspace.yaml`). + +**The Task Scoping Rule:** To prevent Cargo and pnpm from attempting to build the entire monorepo +when invoked inside a specific project, all inherited Moon tasks must use scope flags injected with +Moon's `$project` variable (e.g., `cargo build -p $project` instead of bare `cargo build`). + +### Editor Configuration & Validation + +All Moon configuration files will utilize remote JSON `$schema` headers. To prevent the VS Code YAML +extension from reporting false-positive validation errors, contributors must use the +`moon: Append YAML schemas configuration to settings` command provided by the official Moon VS Code +extension. + +## Consequences + +- Complete reproducibility: anyone cloning the repo will automatically download the correct + compilers and toolchains via `proto`. +- Clean Architecture is structurally enforced: Moon tags prevent illegal cross-project imports, and + Clippy/ESLint prevent illegal intra-project imports. +- By using `$project` variables with scope flags (`-p`), Moon and Cargo cooperate peacefully rather + than fighting over workspace control. +- The use of `$schema` headers combined with the VS Code extension provides immediate IDE validation + and prevents silent configuration errors. diff --git a/docs/0009-jsonl-default-output-with-pretty-json-array-option.md b/docs/0009-jsonl-default-output-with-pretty-json-array-option.md new file mode 100644 index 0000000..5ef2fa7 --- /dev/null +++ b/docs/0009-jsonl-default-output-with-pretty-json-array-option.md @@ -0,0 +1,109 @@ +--- +number: 9 +title: JSONL Default Output with Pretty JSON Array Option +date: 2026-03-13 +status: accepted +--- + +# 9. JSONL Default Output with Pretty JSON Array Option + +Date: 2026-03-13 + +## Status + +Accepted + +## Context + +The parser needs a serialization format for its output that is both machine-readable for downstream +tooling and human-inspectable for debugging. The output must represent the full result of a parse +run, including metadata, commands with their argument payloads, and interleaved text blocks. Two +common JSON-based approaches exist: a single JSON document (easy to pretty-print, harder to stream) +and JSONL (one object per line, streamable, harder to read). + +## Decision + +The parser supports two output formats, selectable via CLI flag (see ADR-0007): + +- JSONL (default): one JSON object per line, suitable for streaming and piping. +- Pretty JSON array: a single formatted JSON document, suitable for human inspection. + +### Output envelope + +A parser run produces a JSON object with the following top-level structure: + +```json +{ + "version": "0.1.0", + "context": { + "source": "string", + "timestamp": "2026-03-13T10:40:00Z", + "user": "string", + "session_id": "string", + "extra": {} + }, + "commands": [], + "text_blocks": [] +} +``` + +- `version`: semantic version of the output schema. +- `context`: metadata about the parse run. Fields `source`, `timestamp`, `user`, and `session_id` + are populated from CLI flags (ADR-0007) or defaults. `extra` is a free-form object for additional + metadata. +- `commands`: ordered array of Command objects. +- `text_blocks`: ordered array of TextBlock objects. + +### Command schema + +Each element in `commands` is: + +````json +{ + "id": "cmd-0", + "name": "mcp", + "raw": "/mcp call_tool write_file ```jsonl\n...\n```", + "range": { "start_line": 10, "end_line": 20 }, + "arguments": { + "header": "call_tool write_file", + "mode": "fence", + "fence_lang": "jsonl", + "payload": "{\n \"path\": \"...\"\n}" + }, + "children": [] +} +```` + +- `id`: unique identifier per ADR-0008 (`cmd-0`, `cmd-1`, etc.). +- `name`: command name without the leading `/`. +- `raw`: exact source slice for the command (header + all argument lines). +- `range`: inclusive line range the command covers (`start_line`, `end_line`). +- `arguments`: argument payload object per ADR-0005 with `header`, `mode`, `fence_lang`, and + `payload` fields. +- `children`: reserved for future hierarchical structures, currently always empty. + +### TextBlock schema + +Each element in `text_blocks` is: + +```json +{ "id": "text-0", "range": { "start_line": 0, "end_line": 9 }, "content": "arbitrary text\n..." } +``` + +- `id`: unique identifier per ADR-0008 (`text-0`, `text-1`, etc.). +- `range`: inclusive line range. +- `content`: exact text for the block with internal newline separators from the normalized input. + +## Consequences + +- JSONL as default enables streaming pipelines where consumers process commands as they are emitted + without waiting for the full document. +- Pretty JSON mode provides a single-document view for debugging and manual inspection. +- The envelope schema captures both parse results and run metadata, making output self-describing + and reproducible. +- The `raw` field on commands preserves the original source text for round-trip inspection or error + reporting. +- The `children` field is a forward-compatible extension point that does not affect current + consumers. +- Text blocks between commands are explicitly captured rather than discarded, ensuring no input + content is silently lost. diff --git a/docs/adr parse pipeline.md b/docs/adr parse pipeline.md new file mode 100644 index 0000000..ff00250 --- /dev/null +++ b/docs/adr parse pipeline.md @@ -0,0 +1,267 @@ +--- +status: accepted +date: 2026-03-19 +decision-makers: + - Tom +consulted: [] +informed: [] +--- + +# Document Parse Pipeline: Pull-Model LineJoiner with Two-State Loop + +## Context and Problem Statement + +The `document_parse.rs` module is the top-level orchestrator of the slash-parser v0.3.0 engine. It +must wire together normalization (§2.1), POSIX-style backslash line joining (§2.2), command +detection and classification (§3), a two-state machine (§4), fenced payload accumulation (§5.2), +text block collection (§6), sequential ID assignment (§7), and correct `raw` field construction +(§8.2), all within a single forward pass (§1.1) that supports incremental emission (§9.2). + +The fundamental tension is that the spec describes line joining as a "pre-pass" (§2.2) but mandates +fence immunity (§2.3): lines inside a fenced block must never be subject to joining. Fence +boundaries are only discoverable during command parsing, so a naive eager pre-pass is impossible +without either a second pass or buffering the entire input. + +How should `document_parse.rs` orchestrate the pipeline to satisfy all spec requirements in a single +forward pass? + +## Decision Drivers + +- Spec §2.3 (fence immunity) and §2.2 (line joining) create a circular dependency: joining must + happen before parsing, but fence boundaries are only known during parsing. +- Spec §9.2 requires incremental emission: each command or text block must be finalizable as soon as + its last physical line is consumed. Memory must be bounded by the largest single element, not + total input size. +- Spec §1.1 requires a single forward pass after normalization. No backtracking. +- Spec §8.2 requires `raw` to contain the exact normalized source text before line joining, + including backslash characters and `\n` separators between physical lines. +- Spec §6 and §9.4 (round-trip fidelity) require text block content and ranges to use physical line + numbers and physical line content so a formatter can reconstruct the original input. +- Spec §7 requires independent zero-based ID sequences for commands (`cmd-0`, `cmd-1`) and text + blocks (`text-0`, `text-1`). +- Architecture doc §9.2 recommends an explicit loop over iterator chains when the source of data + changes mid-stream. +- Architecture doc §9.1 notes that inputs are small (chatops messages, git comments), favoring + clarity over allocation avoidance. + +## Considered Options + +1. Pull-model LineJoiner with explicit two-state `while` loop +2. Pure iterator chain with `scan`/`flat_map` adapters +3. Two-pass approach: locate fence boundaries first, then join outside them +4. Zero-copy slice tracking instead of allocated `LogicalLine` strings +5. Mutable `String` builder for `raw` field construction +6. Single shared counter for command and text block IDs +7. Text block content from joined logical lines (post-join text) +8. Text block content from raw physical lines (pre-join text) + +## Decision Outcome + +Chosen options: + +- **Option 1** for the overall pipeline architecture. +- **Option 8** for text block content representation. + +Combined, these produce a pipeline where: + +1. `normalize(input)` converts all line endings to LF (§2.1). +2. The normalized string is split on `\n` into `physical_lines: Vec<&str>`, retained for `raw` + reconstruction and text block content. +3. An owned copy is passed to `LineJoiner::new(owned)`. +4. A `while` loop drives a `ParserState` enum (`Idle` | `InFence(PendingCommand)`). +5. In `Idle`, `joiner.next_logical()` produces joined `LogicalLine` values. Each is classified. + Commands start accumulation. Text lines feed physical line content into `PendingText` via + `start_text`/`append_text`. +6. In `InFence`, `joiner.next_physical()` produces raw lines. Each is fed to `accept_line`. On close + or EOF, the command is finalized and the state returns to `Idle`. +7. Before starting a command, `header.raw` is overwritten with + `physical_lines[first..=last].join("\n")` to satisfy §8.2. +8. Independent `cmd_seq: usize` and `text_seq: usize` counters are maintained and passed to + `start_command` and `finalize_text` respectively. +9. At EOF, any pending text block is finalized. The result is returned with no `context` field (SDKs + inject that). + +### Consequences + +**Good:** + +- Fence immunity is achieved without a second pass. The shared cursor inside `LineJoiner` means + calling `next_physical()` in `InFence` bypasses joining for exactly the lines inside the fence, + then `next_logical()` resumes joining when the state returns to `Idle`. +- Incremental emission is preserved. Each command and text block is finalized as soon as its last + line is consumed. No global buffering beyond the current pending element. +- The fence-opener edge case (§5.2.6) resolves implicitly. When backslash joining merges a command + line with a line containing a fence opener, `next_logical()` produces the merged text and + `classify_line` detects the fence naturally. No special-casing in the orchestrator. +- `raw` is always correct. Slicing from the retained `physical_lines` vector guarantees the exact + normalized source text including backslashes. +- Text block content stores physical lines, making round-trip reconstruction (§9.4) straightforward. + A formatter can use `content` directly without needing to reverse the joining process. +- Two independent ID counters match the spec's independent sequences exactly. + +**Neutral:** + +- The `physical_lines: Vec<&str>` borrows from the normalized string, which must live for the + duration of the parse. This is trivially satisfied since `parse_document` owns the normalized + string on the stack. +- An owned `Vec` copy is created for the `LineJoiner`. This doubles memory for the line + storage, but inputs are small and the clarity benefit is significant. + +**Bad:** + +- Text block content containing backslash-continued lines will include the trailing backslash on + each physical line. A consumer expecting "clean" joined text from text blocks would need to + re-join. This is the correct behavior for round-trip fidelity but may surprise consumers who + expect text blocks to contain joined content. This tradeoff is accepted because ranges already use + physical line numbers, and consistency between range and content is more important than consumer + convenience. +- The `LineJoiner` allocates a new `String` for every logical line produced by `next_logical()`. For + inputs with many non-continued lines, this is one allocation per line. Acceptable for + chatops-scale inputs. + +## Pros and Cons of the Options + +### Option 1: Pull-model LineJoiner with explicit two-state `while` loop + +The `LineJoiner` exposes `next_logical()` (joins) and `next_physical()` (raw) on a shared cursor. +The orchestrator drives it from a `while` loop matching on `ParserState`. + +- Good, because the state machine directly controls which consumption mode is active, making fence + immunity explicit and auditable. +- Good, because the shared cursor guarantees no lines are skipped or double-consumed at the + Idle/InFence transition boundary. +- Good, because an explicit loop is more readable than iterator chains when the data source changes + mid-stream (architecture doc §9.2). +- Good, because incremental emission is trivially satisfied since each element is finalized inline. +- Neutral, because it requires the orchestrator to own the loop and match arms, which is more code + than a fold but more explicit. + +### Option 2: Pure iterator chain with `scan`/`flat_map` adapters + +Express the entire pipeline as a chain of iterator adapters, using `scan` to carry `ParserState` and +`flat_map` to switch between logical and physical line sources. + +- Good, because it would be concise and leverage Rust's iterator optimizations (lazy evaluation, + potential vectorization). +- Bad, because switching the source of lines (logical vs physical) mid-stream requires either + duplicating the joiner or using interior mutability and `Peekable` with complex lookahead. +- Bad, because lifetime and borrowing constraints make it difficult to hold a mutable reference to + the joiner inside a `scan` closure while also producing items. +- Bad, because the control flow for fence immunity (switching consumption modes) is obscured inside + adapter closures, making the code harder to audit against the spec. + +### Option 3: Two-pass approach (locate fence boundaries first, then join) + +First pass: scan all physical lines to identify fence-open/close pairs and their line ranges. Second +pass: apply line joining only to regions outside fences, then run the state machine on the mixed +result. + +- Good, because fence immunity is guaranteed by construction. The joiner only ever sees non-fence + lines. +- Bad, because it violates §9.2 (incremental emission). The first pass must buffer all fence + boundary positions before any command can be emitted. +- Bad, because identifying fence boundaries requires knowing which lines are command lines (to find + fence openers), which means the first pass duplicates much of the classification logic. +- Bad, because it requires two iterations over the input, violating the single-forward-pass + principle (§1.1). + +### Option 4: Zero-copy slice tracking + +Represent joined logical lines as `Vec<(start_byte, end_byte)>` ranges into the normalized input +string, avoiding `String` allocation entirely. + +- Good, because it eliminates allocation for joined lines, potentially improving throughput on large + inputs. +- Bad, because backslash removal creates non-contiguous content. A joined line spanning physical + lines `"a\"` and `"b"` maps to bytes `[0..1]` + space + `[3..4]`, requiring a non-contiguous slice + assembly that complicates all downstream consumers. +- Bad, because `classify_line` and other downstream functions expect `&str`, requiring either + on-the-fly assembly or API changes throughout the pipeline. +- Neutral, because inputs are small (chatops messages, git comments). Profiling has not shown + allocation as a bottleneck. Documented as a future optimization path if profiling reveals a need. + +### Option 5: Mutable `String` builder for `raw` field + +Maintain a mutable `String` that accumulates physical lines (with `\n` separators) as they are +consumed, becoming the `raw` field when the command is finalized. + +- Good, because it avoids the post-hoc slice-and-join from `physical_lines`. +- Bad, because the builder must be reset for each command and coordinated with the transition + between Idle and InFence states. For fenced commands, `accept_line` already accumulates + `raw_lines` on `PendingCommand`, so the builder would duplicate that responsibility. +- Bad, because it obscures the source of truth. The spec defines `raw` as the exact normalized + source text, which is more clearly expressed by slicing the immutable `physical_lines` vector. + +### Option 6: Single shared counter for command and text block IDs + +Use one global counter and assign IDs like `cmd-0`, `text-1`, `cmd-2` in encounter order. + +- Good, because it simplifies bookkeeping to a single `usize`. +- Bad, because it violates §7, which specifies that command IDs and text block IDs are independent + zero-based sequences. The spec examples show `cmd-0` and `text-0` coexisting, not `cmd-0` and + `text-1`. + +### Option 7: Text block content from joined logical lines + +Store the `LogicalLine.text` (post-join, backslash removed) as text block content. + +- Good, because consumers see "clean" text without trailing backslashes or continuation artifacts. +- Bad, because the range already uses physical line numbers. A text block with + `range: {start_line: 0, end_line: 1}` but content of one joined line creates an inconsistency: the + range says two physical lines but the content has no `\n` separator. +- Bad, because round-trip reconstruction (§9.4) becomes lossy. A formatter cannot distinguish + between a two-physical-line text region that was joined (backslash removed) and a single physical + line, because both would produce the same content string. +- Bad, because it creates an asymmetry with commands: `command.raw` stores physical lines (pre-join) + but `textblock.content` would store logical lines (post-join). + +### Option 8: Text block content from raw physical lines + +Feed `physical_lines[idx]` (pre-join, backslashes retained) into `start_text`/`append_text` for each +physical line covered by a logical line. + +- Good, because content and range are consistent. A range spanning physical lines 0-1 will always + have content with exactly one `\n` separator. +- Good, because round-trip reconstruction is lossless. The formatter can reproduce the exact + normalized input from the content and range of each text block and command. +- Good, because it is symmetric with `command.raw`, which also stores physical lines. +- Neutral, because text block content for continuation lines will contain trailing backslashes. + Consumers expecting "clean" text must handle this, but the spec does not promise that text blocks + strip join markers. + +## More Information + +### Spec references + +- §1.1 Design Principles (single forward pass, incremental emission, deterministic, total) +- §2.1 Line Ending Normalization +- §2.2 Line Joining (Backslash Continuation) +- §2.2.1 Physical Line Tracking +- §2.2.2 Trailing Backslash at EOF +- §2.3 Fence Immunity +- §3 Command Detection +- §4 Parser States (Idle, InFence) +- §5.1 Single-Line Mode +- §5.2 Fence Mode (opener, body, lifetime, closer, unclosed, joining around fences) +- §6 Text Blocks +- §7 Multiple Commands and Ordering (independent ID sequences) +- §8.2 The `raw` Field +- §9.2 Incremental Emission +- §9.4 Roundtrip Fidelity Invariant + +### Architecture references + +- parser-architecture.md §3.2 `documentparse.rs` pipeline description +- parser-architecture.md §5 State Machine (Idle, InFence) +- parser-architecture.md §9.1 Logical Line Intermediate Type (allocated vs zero-copy) +- parser-architecture.md §9.2 Iterator vs Imperative Loop + +### Design council findings + +Three independent model analyses were conducted. All three converged on Option 1 (pull-model with +explicit loop) and confirmed that Option 3 (two-pass) violates incremental emission. The key +divergence was around text block content: one model identified a potential bug where physical lines +with trailing backslashes would appear in text block content, which after analysis was determined to +be the correct behavior for round-trip fidelity (Option 8). Another model independently confirmed +that the fence-opener edge case (§5.2.6) resolves implicitly through separation of concerns between +the joiner and classifier, requiring no special handling in the orchestrator. diff --git a/docs/parser-architecture.md b/docs/parser-architecture.md new file mode 100644 index 0000000..4c829d1 --- /dev/null +++ b/docs/parser-architecture.md @@ -0,0 +1,530 @@ +# Slash Command Parser (`//parser`) Architecture + +Spec: Slash Command Parser Specification v0.3.0 ADR: 0006 – Repository Structure and Project +Boundaries Patterns: Tom's Clean Code, Tom's Clean Architecture, Rust Patterns, Universal Rust/WASM +Library Pattern + +## 1. Purpose + +`//parser` is the internal Rust library that implements the Slash Command Parser v0.3.0 engine. It +is the single source of truth for all parsing behavior. No parsing logic exists outside this crate. + +It accepts a UTF-8 string and returns a structured Rust result representing all detected slash +commands, interleaved text blocks, and any parser warnings. The parser is deterministic, pure, and +total: it always produces a valid result for any input. + +## 2. Boundaries + +`//parser` is consumed by exactly three projects within the monorepo: + +- `//wasm-javascript` (wasm-bindgen adapter for JS/TS runtimes) +- `//wasm-wasi` (WASI P2 component for polyglot SDK consumption) +- `//slash-rust` (thin published Rust crate wrapping `//parser`) + +External Rust consumers use `//slash-rust`, not `//parser` directly. This keeps `//parser` internal +and free to evolve its API without semver concerns. + +### What `//parser` Does + +- Normalize line endings (CRLF/CR to LF). +- Perform POSIX-style backslash line joining with physical line tracking. +- Detect commands, classify argument modes (single-line or fence). +- Accumulate fenced payloads verbatim. +- Collect non-command lines into text blocks. +- Emit warnings for malformed constructs (e.g., unclosed fences). +- Provide a minimal infrastructure layer for JSON serialization of the result. + +### What `//parser` Does Not Do + +- JSONL framing (responsibility of CLI and SDKs). +- Context propagation (the spec's `context` object is injected by SDKs). +- Argument interpretation (all argument content is opaque). +- IO, logging, configuration, or environment access. +- WASM/WASI binding (handled by `//wasm-javascript` and `//wasm-wasi`). + +## 3. Layered Structure + +Following Tom's Clean Architecture and Rust Patterns, `//parser` is organized into three layers with +strict inward-pointing dependencies. + +``` +parser/ + src/ + lib.rs # Public API re-exports + domain/ + mod.rs + types.rs # Pure domain types (no serde) + application/ + mod.rs + normalize.rs # Line ending normalization + line_join.rs # POSIX backslash line joining + line_classify.rs # Command vs text classification + fence_collect.rs # Fence state accumulation + document_parse.rs # Top-level pipeline orchestration + tests/ + mod.rs + proptest.rs # Layer 2 cross-module property tests + infrastructure/ + mod.rs + json.rs # Serde DTOs and JSON serialization +``` + +### 3.1 Domain Layer (`domain/`) + +Pure types with no external dependencies. No serde, no framework crates. + +```rust +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ArgumentMode { + SingleLine, + Fence, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LineRange { + pub start_line: usize, + pub end_line: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommandArguments { + pub header: String, + pub mode: ArgumentMode, + pub fence_lang: Option, + pub payload: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Command { + pub id: String, + pub name: String, + pub raw: String, + pub range: LineRange, + pub arguments: CommandArguments, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TextBlock { + pub id: String, + pub range: LineRange, + pub content: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Warning { + pub wtype: String, + pub start_line: Option, + pub message: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParseResult { + pub version: String, + pub commands: Vec, + pub textblocks: Vec, + pub warnings: Vec, +} + +pub const SPEC_VERSION: &str = "0.3.0"; +``` + +### 3.2 Application Layer (`application/`) + +Orchestrates the parsing pipeline using domain types. Contains all parsing logic. No IO, no serde, +no framework imports. + +#### `normalize.rs` + +Pure function. Replaces CRLF with LF, then remaining bare CR with LF. + +```rust +pub fn normalize(input: &str) -> String +``` + +#### `line_join.rs` + +Implements POSIX-style backslash line joining (spec Section 2.2). + +For each physical line ending with `\`: + +1. Remove the trailing backslash. +2. Join with the next physical line, separated by a single space. +3. Repeat if the joined result still ends with `\`. +4. At EOF, a trailing backslash is silently removed. + +The joiner produces an iterator of `LogicalLine` values that carry the joined text and the physical +line range they originated from. + +```rust +pub struct LogicalLine { + pub text: String, + pub first_physical: usize, + pub last_physical: usize, +} +``` + +The joiner must not join lines that are inside a fenced block (spec Section 2.3: Fence Immunity). +Because fence detection happens during command parsing, the joiner and the state machine must +cooperate. The implementation uses a "pull" model: the state machine drives the joiner, and when +entering `InFence` state it switches to consuming raw physical lines directly until the fence +closes. + +**Alternative: Zero-Copy with Slice Indices** + +Instead of allocating `String`s for joined lines, track `(start_byte, end_byte)` ranges into the +normalized input. Joined lines would be represented as a `Vec` of non-contiguous slices. This avoids +allocation but complicates fence detection and command parsing because the backslash and newline are +physically removed during joining, requiring non-contiguous slice assembly. Given that inputs are +small (chatops messages, git comments), the allocation approach is preferred for clarity. The +zero-copy alternative is documented here for future reference if profiling reveals a need. + +#### `line_classify.rs` + +Classifies a single logical line as either a command or text. + +```rust +pub enum LineKind { + Command(CommandHeader), + Text, +} + +pub struct CommandHeader { + pub raw: String, + pub name: String, + pub header_text: String, + pub mode: ArgumentMode, + pub fence_lang: Option, + pub fence_backtick_count: Option, +} + +pub fn classify_line(line: &str) -> LineKind +``` + +The classifier detects: + +- Valid command names (`[a-z][a-z0-9-]*`). +- Fence openers (three or more backticks in the arguments portion). +- Single-line mode (everything else after the command name). +- Invalid slash lines (bare `/`, uppercase, digits) are classified as `Text`. + +#### `fence_collect.rs` + +Manages the `InFence` state. Consumes raw physical lines until a valid closing fence or EOF. + +```rust +pub struct FenceCollector { + pub backtick_count: usize, + pub payload_lines: Vec, + pub start_line: usize, + pub end_line: usize, +} +``` + +A closing fence is a physical line that, after trimming leading and trailing whitespace, consists +solely of backtick characters with a count >= the opener's count. + +At EOF without a closing fence, the collector signals that the fence is unclosed. The caller emits +an `unclosedfence` warning. + +#### `document_parse.rs` + +Top-level orchestration. This is the only module that composes the full pipeline. + +Pipeline: + +1. `normalize(input)` produces a normalized string. +2. Split into physical lines with indices. +3. Drive a two-state machine (`Idle` / `InFence`) over the physical lines: + - In `Idle`: consume joined logical lines via the line joiner. + - Classify each logical line. + - Command without fence: finalize as single-line, append to results. + - Command with fence: record header/metadata, switch to `InFence`. + - Text: append to current pending text block. + - In `InFence`: consume raw physical lines via the fence collector. + - On fence close: finalize fenced command, return to `Idle`. + - On EOF: finalize with warning, return to `Idle`. +4. At EOF: finalize any pending text block. +5. Assign sequential IDs (`cmd-0`, `cmd-1`, `text-0`, `text-1`). +6. Return `ParseResult`. + +```rust +pub fn parse_document(input: &str) -> ParseResult +``` + +The `raw` field on each command captures the exact source text from the normalized input (before +line joining), including backslashes, physical newlines, and fence delimiters. + +### 3.3 Infrastructure Layer (`infrastructure/`) + +Minimal. Exists solely to support serialization for downstream WASM wrappers and the Rust SDK. + +#### `json.rs` + +Serde-derived DTOs that map 1:1 to the JSON schema in spec Section 8.5. Domain types remain free of +serde derives. + +```rust +use serde::Serialize; + +#[derive(Serialize)] +pub struct ParseResultDto { + pub version: String, + pub commands: Vec, + pub textblocks: Vec, + pub warnings: Vec, +} + +#[derive(Serialize)] +pub struct CommandDto { + pub id: String, + pub name: String, + pub raw: String, + pub range: LineRangeDto, + pub arguments: CommandArgumentsDto, +} + +#[derive(Serialize)] +pub struct CommandArgumentsDto { + pub header: String, + pub mode: String, // "single-line" or "fence" + #[serde(rename = "fencelang")] + pub fence_lang: Option, + pub payload: String, +} + +#[derive(Serialize)] +pub struct LineRangeDto { + #[serde(rename = "startline")] + pub start_line: usize, + #[serde(rename = "endline")] + pub end_line: usize, +} + +#[derive(Serialize)] +pub struct TextBlockDto { + pub id: String, + pub range: LineRangeDto, + pub content: String, +} + +#[derive(Serialize)] +pub struct WarningDto { + #[serde(rename = "type")] + pub wtype: String, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "startline")] + pub start_line: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub message: Option, +} +``` + +Conversion from domain types: + +```rust +impl From<&ParseResult> for ParseResultDto { ... } +impl From<&Command> for CommandDto { ... } +// etc. +``` + +Public serialization functions: + +```rust +pub fn to_json_string(result: &ParseResult) -> String { + let dto = ParseResultDto::from(result); + serde_json::to_string(&dto).expect("serialization is infallible for valid domain types") +} +``` + +Note: `to_json_value` and `to_value` helpers for `JsValue` or `serde_json::Value` are not provided +here. Those belong in `//wasm-javascript` and `//wasm-wasi` respectively. + +## 4. Public API + +`lib.rs` re-exports the public surface: + +```rust +mod application; +mod domain; +mod infrastructure; + +// Domain types +pub use domain::types::{ + ArgumentMode, Command, CommandArguments, LineRange, ParseResult, SPEC_VERSION, TextBlock, + Warning, +}; + +// Engine entry point +pub use application::document_parse::parse_document; + +// JSON serialization (infrastructure) +pub use infrastructure::json::to_json_string; +``` + +The primary entry point: + +```rust +/// Total function. Always returns a valid ParseResult for any input. +/// Never panics. Malformed constructs produce warnings, not errors. +pub fn parse_document(input: &str) -> ParseResult +``` + +There is no `Result` wrapper. The spec requires a total function. All malformed input is reflected +via the `warnings` vec, never via control flow. SDKs that want strict mode can inspect `warnings` +and reject results containing them. + +There is no `context` parameter. The spec's `context` object is injected by SDKs when they build the +final JSON envelope for their consumers. + +## 5. State Machine + +Two states, matching spec Section 4. + +``` +┌──────┐ command + fence opener ┌─────────┐ +│ Idle │ ─────────────────────────▶│ InFence │ +│ │◀───────────────────────── │ │ +└──────┘ closing fence or EOF └─────────┘ +``` + +### Idle + +- Consumes logical lines (joined). +- Classifies each line. +- Single-line commands are finalized immediately. +- Fence-opening commands transition to `InFence`. +- Non-command lines accumulate into the current text block. +- On command or EOF, any pending text block is finalized. + +### InFence + +- Consumes raw physical lines (no joining). +- Appends each line to the fence payload verbatim. +- Checks each line for a valid closing fence. +- On close: finalizes the fenced command, returns to `Idle`. +- On EOF without close: finalizes with partial payload, emits `unclosedfence` warning, returns to + `Idle`. + +## 6. Dependencies + +```toml +[package] +name = "parser" +edition = "2021" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[dev-dependencies] +proptest = "1" +proptest-derive = "0.5" + +[features] +tdd = [] +``` + +`serde` and `serde_json` are used only in `infrastructure/json.rs`. Domain and application layers +have zero external dependencies beyond `std`. + +## 7. Testing Strategy + +### 7.1 Layer 1: In-File Unit Tests + +Each application module has `#[cfg(test)] mod tests` at the bottom. + +- `normalize.rs`: CRLF, bare CR, mixed endings, no-op on clean input. +- `line_join.rs`: single join, multi-join chains, trailing backslash at EOF, lines without backslash + pass through, fence immunity integration. +- `line_classify.rs`: valid commands, invalid slash lines, fence opener detection, fence language + extraction, edge cases (bare `/`, uppercase, digits). +- `fence_collect.rs`: closing fence detection, variable-length backticks, unclosed fence at EOF, + payload verbatim preservation. +- `document_parse.rs`: all Appendix A examples from the spec, text block accumulation, interleaved + commands and text, empty input, text-only input, multiple commands, warning emission. +- `infrastructure/json.rs`: DTO conversion correctness, field naming matches spec schema. + +### 7.2 Layer 2: Application Property Tests + +`parser/src/application/tests/proptest.rs` covers cross-module composition. + +Key properties: + +- Never panics on arbitrary ASCII/UTF-8 input (total function). +- Command count equals the number of logical lines starting with a valid slash command. +- Text-only input produces zero commands and non-empty text blocks. +- Fenced content is preserved verbatim (no joining, no escaping). +- Fence immunity: backslashes inside fences are literal, never trigger joining. +- Physical line ranges are always valid (start <= end, within input bounds). +- Version is always `SPEC_VERSION`. +- Unclosed fences always produce exactly one warning. +- Backslash joining around fences: joined lines before a fence opener produce correct header; lines + after a fence closer join correctly as text. + +All property tests use `#[cfg_attr(feature = "tdd", ignore)]` to stay out of watch mode. + +### 7.3 Layer 3: Integration Tests + +`tests/` directory at the crate root. + +- `parse_examples.rs`: exact output assertions for every example in Appendix A (A.1 through A.9). +- `json_output.rs`: end-to-end `parse_document` -> `to_json_string` -> deserialize and assert + against spec schema. +- Future: roundtrip fidelity (`P(F(P(I))) == P(I)`) once a formatter exists. + +## 8. Error Philosophy + +`//parser` has no error type at its public API boundary. The `parse_document` function is total. All +malformed input produces a valid `ParseResult` with appropriate `Warning` entries. This matches the +spec's requirement that the parser never fails. + +SDKs are responsible for deciding whether warnings constitute errors for their consumers. A strict +SDK might reject any result with warnings. A lenient SDK might pass them through as metadata. + +## 9. Decisions and Alternatives + +### 9.1 Logical Line Intermediate Type + +Decision: use `LogicalLine { text: String, first_physical: usize, last_physical: usize }` as the +intermediate between the line joiner and the state machine. + +Alternative: zero-copy slice tracking. Represent joined lines as indices into the normalized input +rather than allocated strings. This avoids allocation but complicates the implementation because +backslash removal creates non-contiguous content. Given inputs are small (chatops, git comments), +allocation is preferred for clarity. Revisit if profiling shows a need. + +### 9.2 Iterator vs Imperative Loop for Pipeline + +Decision: prefer functional iterator chains where the pipeline is clear and readable. Use an +explicit loop when fence immunity requires switching between logical-line and physical-line +consumption mid-stream. + +The top-level `document_parse` will likely use a `while let` loop driving a `ParserState` enum, +because the `Idle` -> `InFence` transition changes the _source_ of lines (logical vs physical). This +is one of the cases where a loop is more readable and maintainable than a pure iterator chain, per +Rust Patterns: "Use for loops when the iterator chain becomes unclear due to lifetimes, borrowing, +or complex control flow." + +Individual stages (normalize, classify, fence check) remain pure functions composed via iterators +where appropriate. + +### 9.3 Context Handling + +Decision: `ParseResult` has no context field. SDKs inject context when building the final JSON +envelope. This keeps `//parser` focused on parsing. + +### 9.4 JSONL Streaming + +Decision: not a concern of `//parser`. The engine emits one `ParseResult` per call. JSONL framing is +the responsibility of the CLI (`//riff-cli`) and language SDKs. + +### 9.5 Return Type + +Decision: `pub fn parse_document(&str) -> ParseResult` with no `Result` wrapper. The function is +total. Warnings are data, not control flow. + +### 9.6 Serde Location + +Decision: serde derives live only in `infrastructure/json.rs` on DTO types. Domain types in +`domain/types.rs` have zero serde annotations. Conversion happens via `From` impls at the +infrastructure boundary. diff --git a/docs/slash-parser CLI Specification.md b/docs/slash-parser CLI Specification.md index 4ff3199..404c9a4 100644 --- a/docs/slash-parser CLI Specification.md +++ b/docs/slash-parser CLI Specification.md @@ -227,4 +227,4 @@ riff -p -c '{"user":"tom","run_id":"abc-123"}' ./prompt.md # Context from multiple sources merged together riff -c ./defaults.json -c ./overrides.env -c debug=true ./prompt.md -``` \ No newline at end of file +``` From 872b0314bded3fbf2cc57d44da47a5e35ccbf14f Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Sat, 21 Mar 2026 01:31:09 -0600 Subject: [PATCH 07/13] refac: let go of first iteration of the parser engine --- parser-core/Cargo.toml | 17 - .../parser/tests/proptest.txt | 8 - parser-core/src/domain.rs | 81 --- parser-core/src/lib.rs | 10 - parser-core/src/parser/mod.rs | 292 ---------- parser-core/src/parser/tests/mod.rs | 517 ------------------ parser-core/src/parser/tests/proptest.rs | 52 -- parser-core/src/serialize.rs | 192 ------- parser-core/src/to_plaintext.rs | 64 --- 9 files changed, 1233 deletions(-) delete mode 100644 parser-core/Cargo.toml delete mode 100644 parser-core/proptest-regressions/parser/tests/proptest.txt delete mode 100644 parser-core/src/domain.rs delete mode 100644 parser-core/src/lib.rs delete mode 100644 parser-core/src/parser/mod.rs delete mode 100644 parser-core/src/parser/tests/mod.rs delete mode 100644 parser-core/src/parser/tests/proptest.rs delete mode 100644 parser-core/src/serialize.rs delete mode 100644 parser-core/src/to_plaintext.rs diff --git a/parser-core/Cargo.toml b/parser-core/Cargo.toml deleted file mode 100644 index 720a75b..0000000 --- a/parser-core/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "parser-core" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -description = "Slash command parser core — pure Rust, no WASM dependencies" - -[dependencies] -serde = { workspace = true } -serde_json = { workspace = true } -thiserror = { workspace = true } - -[dev-dependencies] -proptest = "1.4" -proptest-derive = "0.8" -rand = "^0.10" diff --git a/parser-core/proptest-regressions/parser/tests/proptest.txt b/parser-core/proptest-regressions/parser/tests/proptest.txt deleted file mode 100644 index 98d8634..0000000 --- a/parser-core/proptest-regressions/parser/tests/proptest.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc f63df0fa27263b54e17ee70d4ddf4417a859ce19da05c2b1c07e209de45d8f8f # shrinks to mut original = SlashParseResult { version: "", context: ParserContext { source: None, timestamp: None, user: None, session_id: None, extra: None }, commands: [Command { id: "", name: "0", raw: "", range: CommandRange { start_line: 0, end_line: 0 }, arguments: CommandArguments { header: "", mode: SingleLine, fence_lang: None, payload: "" } }], text_blocks: [] } -cc 0a6b314692b125a45a0f7396925be969a4c10f8fe14fb1aadd3fb6a573243ce5 # shrinks to mut original = SlashParseResult { version: "", context: ParserContext { source: None, timestamp: None, user: None, session_id: None, extra: None }, commands: [Command { id: "", name: "a", raw: "", range: CommandRange { start_line: 0, end_line: 0 }, arguments: CommandArguments { header: "", mode: Continuation, fence_lang: None, payload: "/a" } }], text_blocks: [] } diff --git a/parser-core/src/domain.rs b/parser-core/src/domain.rs deleted file mode 100644 index 8fc0cab..0000000 --- a/parser-core/src/domain.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! Domain types for the slash command parser. -//! -//! These types are pure data — no serde derives. Serialization -//! happens in the `serialize` module via DTO conversion. - -/// Inclusive line range (0-based). -#[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(proptest_derive::Arbitrary))] -pub struct CommandRange { - pub start_line: usize, - pub end_line: usize, -} - -/// How the argument payload was assembled. -#[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(proptest_derive::Arbitrary))] -pub enum ArgumentMode { - SingleLine, - Continuation, - Fence, -} - -/// Parsed arguments for a single command. -#[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(proptest_derive::Arbitrary))] -pub struct CommandArguments { - pub header: String, - pub mode: ArgumentMode, - pub fence_lang: Option, - pub payload: String, -} - -/// A single parsed slash command. -#[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(proptest_derive::Arbitrary))] -pub struct Command { - pub id: String, - #[cfg_attr(test, proptest(regex = "[a-z][a-z0-9-]*"))] - pub name: String, - pub raw: String, - pub range: CommandRange, - pub arguments: CommandArguments, -} - -/// A contiguous block of non-command text. -#[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(test, derive(proptest_derive::Arbitrary))] -pub struct TextBlock { - pub id: String, - pub range: CommandRange, - pub content: String, -} - -/// Optional context passed by the caller. -#[derive(Debug, Clone, Default)] -#[cfg_attr(test, derive(proptest_derive::Arbitrary))] -pub struct ParserContext { - pub source: Option, - pub timestamp: Option, - pub user: Option, - pub session_id: Option, - #[cfg_attr(test, proptest(value = "None"))] - pub extra: Option, -} - -/// Top-level parse result. -#[derive(Debug, Clone)] -#[cfg_attr(test, derive(proptest_derive::Arbitrary))] -pub struct SlashParseResult { - pub version: String, - pub context: ParserContext, - pub commands: Vec, - pub text_blocks: Vec, -} - -/// Typed parse errors. -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -pub enum ParseError { - #[error("serialization failed: {message}")] - SerializationError { message: String }, -} diff --git a/parser-core/src/lib.rs b/parser-core/src/lib.rs deleted file mode 100644 index a87f478..0000000 --- a/parser-core/src/lib.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub mod domain; -pub mod parser; -pub mod serialize; -pub mod to_plaintext; - -pub use domain::{ - ArgumentMode, Command, CommandArguments, CommandRange, ParseError, ParserContext, SlashParseResult, - TextBlock, -}; -pub use parser::{parse_slash_commands, parse_to_domain}; diff --git a/parser-core/src/parser/mod.rs b/parser-core/src/parser/mod.rs deleted file mode 100644 index 1f05e29..0000000 --- a/parser-core/src/parser/mod.rs +++ /dev/null @@ -1,292 +0,0 @@ -use crate::{ - domain::{ - ArgumentMode, Command, CommandArguments, CommandRange, ParseError, ParserContext, SlashParseResult, - TextBlock, - }, - serialize, -}; - -/// Parser states — flat, no recursion. -#[derive(Debug, Clone, PartialEq, Eq)] -enum State { - Idle, - Accumulating, - InFence { marker_len: usize }, -} - -/// In-progress command being built. -struct CommandBuilder { - name: String, - header: String, - raw_lines: Vec, - payload: String, - mode: ArgumentMode, - fence_lang: Option, - start_line: usize, -} - -impl CommandBuilder { - fn new(name: String, header: String, start_line: usize) -> Self { - Self { - name, - header, - raw_lines: Vec::new(), - payload: String::new(), - mode: ArgumentMode::SingleLine, - fence_lang: None, - start_line, - } - } - - fn finalize(self, end_line: usize, command_index: usize) -> Command { - Command { - id: format!("cmd-{command_index}"), - name: self.name, - raw: self.raw_lines.join("\n"), - range: CommandRange { start_line: self.start_line, end_line }, - arguments: CommandArguments { - header: self.header, - mode: self.mode, - fence_lang: self.fence_lang, - payload: self.payload, - }, - } - } -} - -/// Parse a command name from the text after the leading `/`. -/// Returns `(name, rest_of_line)` or `None` if no valid name. -fn parse_command_name(after_slash: &str) -> Option<(&str, &str)> { - let bytes = after_slash.as_bytes(); - if bytes.is_empty() || !bytes[0].is_ascii_lowercase() { - return None; - } - let end = bytes - .iter() - .position(|&b| !(b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')) - .unwrap_or(bytes.len()); - - // Name must start with lowercase letter — already checked above. - let name = &after_slash[..end]; - let rest = &after_slash[end..]; - Some((name, rest)) -} - -/// Check if a line ends with the continuation marker ` /`. -fn has_continuation(line: &str) -> bool { - line.ends_with(" /") -} - -/// Strip the trailing ` /` continuation marker. -fn strip_continuation(line: &str) -> &str { - if line.len() < 2 { line } else { &line[..line.len() - 2] } -} - -/// Detect a fence opener (three or more backticks) in text. -/// Returns `(before_fence, marker_len, lang)` or `None`. -fn detect_fence_opener(text: &str) -> Option<(&str, usize, Option<&str>)> { - let start = text.find("```")?; - let before = &text[..start]; - let backtick_region = &text[start..]; - let marker_len = backtick_region.bytes().take_while(|&b| b == b'`').count(); - let after_backticks = &backtick_region[marker_len..]; - - // Language identifier: up to first whitespace or end of string. - let lang = after_backticks.split_whitespace().next(); - let lang = lang.filter(|l| !l.is_empty()); - - Some((before, marker_len, lang)) -} - -/// Check if a line is a closing fence for the given marker length. -fn is_closing_fence(line: &str, marker_len: usize) -> bool { - let trimmed = line.trim(); - if trimmed.len() < marker_len { - return false; - } - let backtick_count = trimmed.bytes().take_while(|&b| b == b'`').count(); - backtick_count >= marker_len && trimmed.len() == backtick_count -} - -/// Detect if a line is a command line (first non-whitespace is `/`). -/// Returns the content after leading whitespace if it starts with `/`. -fn detect_command_line(line: &str) -> Option<&str> { - let trimmed = line.trim_start(); - if trimmed.starts_with('/') { Some(trimmed) } else { None } -} - -/// Parse slash commands from input text and return a JSON string. -/// -/// Input: UTF-8 text. `\r\n` is normalized to `\n`. -/// Output: `Result` where the string is JSON -/// conforming to the `SlashParseResult` schema. -pub fn parse_slash_commands(input: &str, context: ParserContext) -> Result { - let result = parse_to_domain(input, context); - serialize::to_json(&result).map_err(|err| ParseError::SerializationError { message: err.to_string() }) -} - -/// Parse slash commands into domain types. -/// -/// Returns structured domain types rather than a JSON string. Used by -/// consumers that want to work with typed data (e.g. the CLI). -pub fn parse_to_domain(input: &str, context: ParserContext) -> SlashParseResult { - let normalized = input.replace("\r\n", "\n").replace("\r", "\n"); - let lines: Vec<&str> = normalized.split('\n').collect(); - - let mut state = State::Idle; - let mut commands: Vec = Vec::new(); - let mut text_blocks: Vec = Vec::new(); - let mut current: Option = None; - - // Track contiguous non-command text. - let mut text_start: Option = None; - let mut text_content = String::new(); - - let mut line_index = 0; - while line_index < lines.len() { - let line = lines[line_index]; - - match &state { - State::Idle => { - if let Some(trimmed) = detect_command_line(line) { - // Flush any pending text block. - if let Some(start) = text_start.take() { - // Remove trailing newline from text content. - if text_content.ends_with('\n') { - text_content.pop(); - } - text_blocks.push(TextBlock { - id: format!("text-{}", text_blocks.len()), - range: CommandRange { start_line: start, end_line: line_index.saturating_sub(1) }, - content: std::mem::take(&mut text_content), - }); - } - - // Parse command name from after the `/`. - let after_slash = &trimmed[1..]; - if let Some((name, rest)) = parse_command_name(after_slash) { - let header = if rest.starts_with(char::is_whitespace) { - rest.trim_start().to_string() - } else { - String::new() - }; - - let mut builder = CommandBuilder::new(name.to_string(), header.clone(), line_index); - builder.raw_lines.push(trimmed.to_string()); - - // Check for inline fence opener in the header. - if let Some((before, marker_len, lang)) = detect_fence_opener(&header) { - builder.header = before.trim_end().to_string(); - builder.fence_lang = lang.map(ToString::to_string); - builder.mode = ArgumentMode::Fence; - current = Some(builder); - state = State::InFence { marker_len }; - } else if has_continuation(trimmed) { - // Full trimmed line has continuation marker; enter continuation mode. - // Compute the portion after `/name` as the first payload line. - let after_name = &trimmed[1 + name.len()..]; - let body_with_marker = after_name.trim_start(); - let body = strip_continuation(body_with_marker); - builder.payload.push_str(body); - builder.payload.push('\n'); - builder.mode = ArgumentMode::Continuation; - current = Some(builder); - state = State::Accumulating; - } else { - // Single-line command. - builder.payload = header; - builder.mode = ArgumentMode::SingleLine; - commands.push(builder.finalize(line_index, commands.len())); - } - } else { - // `/` followed by invalid name — treat as text. - if text_start.is_none() { - text_start = Some(line_index); - } - text_content.push_str(line); - text_content.push('\n'); - } - } else { - // Non-command line in idle — accumulate as text. - if text_start.is_none() { - text_start = Some(line_index); - } - text_content.push_str(line); - text_content.push('\n'); - } - } - - State::Accumulating => { - if let Some(builder) = current.as_mut() { - builder.raw_lines.push(line.to_string()); - - // Check if this line is a fence opener. - if let Some((_, marker_len, lang)) = detect_fence_opener(line) { - builder.fence_lang = lang.map(ToString::to_string); - builder.mode = ArgumentMode::Fence; - state = State::InFence { marker_len }; - } else if has_continuation(line) { - // Continuation marker line contributes a (possibly empty) payload line. - let stripped = strip_continuation(line); - builder.payload.push_str(stripped); - builder.payload.push('\n'); - // Stay in Accumulating. - } else if line.is_empty() { - // True blank line ends continuation without being added to payload. - let cmd = - current.take().unwrap().finalize(line_index.saturating_sub(1), commands.len()); - commands.push(cmd); - state = State::Idle; - } else { - // Normal continuation line — append and keep accumulating; EOF will finalize. - builder.payload.push_str(line); - builder.payload.push('\n'); - } - } - } - - State::InFence { marker_len } => { - let marker_len = *marker_len; - if let Some(builder) = current.as_mut() { - builder.raw_lines.push(line.to_string()); - - if is_closing_fence(line, marker_len) { - // Closing fence — finalize command. - let cmd = current.take().unwrap().finalize(line_index, commands.len()); - commands.push(cmd); - state = State::Idle; - } else { - // Inside fence — append verbatim. - builder.payload.push_str(line); - builder.payload.push('\n'); - } - } - } - } - - line_index += 1; - } - - // Finalize any in-progress command (unclosed fence or continuation at EOF). - if let Some(builder) = current.take() { - let end = lines.len().saturating_sub(1); - commands.push(builder.finalize(end, commands.len())); - } - - // Flush any trailing text block. - if let Some(start) = text_start.take() { - if text_content.ends_with('\n') { - text_content.pop(); - } - text_blocks.push(TextBlock { - id: format!("text-{}", text_blocks.len()), - range: CommandRange { start_line: start, end_line: lines.len().saturating_sub(1) }, - content: text_content, - }); - } - - SlashParseResult { version: "0.1.0".to_string(), context, commands, text_blocks } -} - -#[cfg(test)] -mod tests; diff --git a/parser-core/src/parser/tests/mod.rs b/parser-core/src/parser/tests/mod.rs deleted file mode 100644 index 0b71bdf..0000000 --- a/parser-core/src/parser/tests/mod.rs +++ /dev/null @@ -1,517 +0,0 @@ -use super::*; -use crate::domain::{ArgumentMode, ParserContext}; - -mod proptest; - -// --- Test helpers --- - -fn default_context() -> ParserContext { - ParserContext::default() -} - -fn parse(input: &str) -> SlashParseResult { - parse_to_domain(input, default_context()) -} - -fn assert_single_command(input: &str) -> Command { - let result = parse(input); - assert_eq!(result.commands.len(), 1, "expected exactly 1 command"); - result.commands.into_iter().next().unwrap() -} - -fn assert_no_commands(input: &str) { - let result = parse(input); - assert!(result.commands.is_empty(), "expected no commands, got {}", result.commands.len()); -} - -// --- Single-line commands --- - -#[test] -fn single_line_simple_command_parses_name() { - let cmd = assert_single_command("/hello world"); - assert_eq!(cmd.name, "hello"); -} - -#[test] -fn single_line_simple_command_parses_header() { - let cmd = assert_single_command("/hello world"); - assert_eq!(cmd.arguments.header, "world"); -} - -#[test] -fn single_line_simple_command_has_single_line_mode() { - let cmd = assert_single_command("/hello world"); - assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); -} - -#[test] -fn single_line_simple_command_payload_matches_header() { - let cmd = assert_single_command("/hello world"); - assert_eq!(cmd.arguments.payload, "world"); -} - -#[test] -fn single_line_command_with_no_args_has_empty_header() { - let cmd = assert_single_command("/ping"); - assert_eq!(cmd.arguments.header, ""); - assert_eq!(cmd.arguments.payload, ""); -} - -#[test] -fn single_line_command_with_complex_args() { - let cmd = assert_single_command("/mcp call_tool read_file {\"path\": \"src/index.ts\"}"); - assert_eq!(cmd.name, "mcp"); - assert_eq!(cmd.arguments.header, "call_tool read_file {\"path\": \"src/index.ts\"}"); -} - -#[test] -fn single_line_command_id_is_cmd_0() { - let cmd = assert_single_command("/test"); - assert_eq!(cmd.id, "cmd-0"); -} - -#[test] -fn single_line_command_range_is_same_line() { - let cmd = assert_single_command("/test"); - assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 0); -} - -// --- Command name validation --- - -#[test] -fn command_name_with_hyphens_parses() { - let cmd = assert_single_command("/my-command arg"); - assert_eq!(cmd.name, "my-command"); -} - -#[test] -fn command_name_with_digits_parses() { - let cmd = assert_single_command("/cmd2 arg"); - assert_eq!(cmd.name, "cmd2"); -} - -#[test] -fn command_starting_with_digit_is_not_command() { - assert_no_commands("/2fast"); -} - -#[test] -fn command_starting_with_uppercase_is_not_command() { - assert_no_commands("/Hello"); -} - -#[test] -fn slash_alone_is_not_command() { - assert_no_commands("/"); -} - -// --- Leading whitespace --- - -#[test] -fn leading_spaces_before_slash_are_ignored_for_detection() { - let cmd = assert_single_command(" /hello world"); - assert_eq!(cmd.name, "hello"); -} - -#[test] -fn leading_tabs_before_slash_are_ignored_for_detection() { - let cmd = assert_single_command("\t/hello world"); - assert_eq!(cmd.name, "hello"); -} - -// --- Continuation --- - -#[test] -fn continuation_two_lines_produces_continuation_mode() { - let input = "/mcp call_tool read_file /\n{\"path\": \"src/index.ts\"}"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Continuation); -} - -#[test] -fn continuation_header_strips_trailing_marker() { - let input = "/mcp call_tool read_file /\n{\"path\": \"src/index.ts\"}"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.header, "call_tool read_file /"); -} - -#[test] -fn continuation_payload_preserves_newlines() { - let input = "/mcp call_tool read_file /\n{\"path\": \"src/index.ts\"}"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.payload, "call_tool read_file\n{\"path\": \"src/index.ts\"}\n"); -} - -#[test] -fn continuation_three_lines() { - let input = "/cmd first /\nsecond /\nthird"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.payload, "first\nsecond\nthird\n"); -} - -#[test] -fn continuation_range_spans_all_lines() { - let input = "/cmd first /\nsecond /\nthird"; - let cmd = assert_single_command(input); - assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 2); -} - -#[test] -fn slash_at_end_without_space_is_not_continuation() { - let input = "/path /var/log/"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); - assert_eq!(cmd.arguments.payload, "/var/log/"); -} - -// --- Fenced blocks --- - -#[test] -fn closing_fence_can_be_longer_than_opener() { - let input = "/cmd ```\ncontent\n````"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); - assert_eq!(cmd.arguments.payload, "content\n"); -} - -#[test] -fn inline_fence_mode_is_fence() { - let input = "/mcp call_tool write_file ```jsonl\n{\"a\":1}\n```"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); -} - -#[test] -fn inline_fence_captures_language() { - let input = "/mcp call_tool write_file ```jsonl\n{\"a\":1}\n```"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.fence_lang, Some("jsonl".to_string())); -} - -#[test] -fn inline_fence_header_is_before_backticks() { - let input = "/mcp call_tool write_file ```jsonl\n{\"a\":1}\n```"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.header, "call_tool write_file"); -} - -#[test] -fn inline_fence_payload_is_fence_content() { - let input = "/mcp call_tool write_file ```jsonl\n{\"a\":1}\n{\"b\":2}\n```"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.payload, "{\"a\":1}\n{\"b\":2}\n"); -} - -#[test] -fn inline_fence_without_language() { - let input = "/cmd header ```\nsome content\n```"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.fence_lang, None); -} - -#[test] -fn fence_after_continuation() { - let input = "/cmd header /\n```json\n{\"key\": \"value\"}\n```"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); - assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); - // The continuation strips " /" from "header /", appends "header\n", - // then the fence content follows. - assert_eq!(cmd.arguments.payload, "header\n{\"key\": \"value\"}\n"); -} - -#[test] -fn fence_continuation_marker_inside_fence_is_literal() { - let input = "/cmd ```\nline with /\nnormal line\n```"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.payload, "line with /\nnormal line\n"); -} - -#[test] -fn fence_closing_must_match_opener_length() { - let input = "/cmd ````\nline\n```\nstill inside\n````"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.payload, "line\n```\nstill inside\n"); -} - -#[test] -fn fence_line_with_extra_chars_is_not_closing() { - let input = "/cmd ```\nline\n``` not-closing\n```"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); - assert_eq!(cmd.arguments.payload, "line\n``` not-closing\n"); -} - -#[test] -fn fence_range_includes_closing_fence() { - let input = "/cmd ```\nline\n```"; - let cmd = assert_single_command(input); - assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 2); -} - -// --- Multiple commands --- - -#[test] -fn two_single_line_commands_parsed() { - let input = "/first arg1\n/second arg2"; - let result = parse(input); - assert_eq!(result.commands.len(), 2); -} - -#[test] -fn multiple_commands_have_sequential_ids() { - let input = "/first arg1\n/second arg2\n/third arg3"; - let result = parse(input); - assert_eq!(result.commands[0].id, "cmd-0"); - assert_eq!(result.commands[1].id, "cmd-1"); - assert_eq!(result.commands[2].id, "cmd-2"); -} - -#[test] -fn commands_interspersed_with_text() { - let input = "some text\n/first arg\nmore text\n/second arg"; - let result = parse(input); - assert_eq!(result.commands.len(), 2); - assert_eq!(result.text_blocks.len(), 2); -} - -// --- Text blocks --- - -#[test] -fn consecutive_blank_lines_are_preserved_in_text_blocks() { - let input = "hello\n\n\nworld\n/cmd arg"; - let result = parse(input); - assert_eq!(result.text_blocks.len(), 1); - assert_eq!(result.text_blocks[0].content, "hello\n\n\nworld"); - assert_eq!(result.text_blocks[0].range.start_line, 0); - assert_eq!(result.text_blocks[0].range.end_line, 3); -} - -#[test] -fn text_before_command_captured() { - let input = "hello world\n/cmd arg"; - let result = parse(input); - assert_eq!(result.text_blocks.len(), 1); - assert_eq!(result.text_blocks[0].content, "hello world"); - assert_eq!(result.text_blocks[0].id, "text-0"); -} - -#[test] -fn text_after_command_captured() { - let input = "/cmd arg\nsome trailing text"; - let result = parse(input); - assert_eq!(result.text_blocks.len(), 1); - assert_eq!(result.text_blocks[0].content, "some trailing text"); -} - -#[test] -fn text_block_range_is_correct() { - let input = "line0\nline1\n/cmd arg"; - let result = parse(input); - assert_eq!(result.text_blocks[0].range.start_line, 0); - assert_eq!(result.text_blocks[0].range.end_line, 1); -} - -#[test] -fn only_text_produces_no_commands() { - let input = "just some text\nno commands here"; - let result = parse(input); - assert!(result.commands.is_empty()); - assert_eq!(result.text_blocks.len(), 1); -} - -#[test] -fn empty_input_produces_empty_result() { - let result = parse(""); - assert!(result.commands.is_empty()); - assert_eq!(result.text_blocks.len(), 1); - assert_eq!(result.text_blocks[0].content, ""); -} - -// --- CRLF normalization --- - -#[test] -fn continuation_blank_line_via_marker_is_payload() { - let input = "/cmd header /\nline1\n /\nline2"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Continuation); - assert_eq!(cmd.arguments.payload, "header\nline1\n\nline2\n"); -} - -#[test] -fn continuation_ends_on_true_blank_line() { - let input = "/cmd header /\nline1\n\ntrailing"; - let result = parse(input); - assert_eq!(result.commands.len(), 1); - assert_eq!(result.commands[0].arguments.mode, ArgumentMode::Continuation); - assert_eq!(result.commands[0].arguments.payload, "header\nline1\n"); -} - -#[test] -fn bare_slash_is_treated_as_payload_in_continuation() { - let input = "/echo /\nooga booga \n/\ntesting 123"; - let result = parse(input); - let cmd = &result.commands[0]; - assert_eq!(cmd.arguments.mode, ArgumentMode::Continuation); - assert!(cmd.arguments.payload.contains("ooga booga \n")); - assert!(cmd.arguments.payload.contains("/\n")); - assert!(cmd.arguments.payload.contains("testing 123\n")); -} - -// --- CRLF normalization --- - -#[test] -fn carriage_returns_are_normalized_to_newlines() { - let input = "/cmd first /\r\nsecond\nthird\r\nfourth"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Continuation); - assert_eq!(cmd.arguments.payload, "first\nsecond\nthird\nfourth\n"); -} - -#[test] -fn crlf_normalized_to_lf() { - let input = "/cmd arg1\r\n/cmd2 arg2"; - let result = parse(input); - assert_eq!(result.commands.len(), 2); -} - -#[test] -fn crlf_continuation_works() { - let input = "/cmd first /\r\nsecond"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Continuation); - assert_eq!(cmd.arguments.payload, "first\nsecond\n"); -} - -// --- Raw field --- - -#[test] -fn raw_single_line_is_trimmed_line() { - let cmd = assert_single_command(" /hello world"); - assert_eq!(cmd.raw, "/hello world"); -} - -#[test] -fn raw_continuation_includes_all_lines() { - let input = "/cmd first /\nsecond"; - let cmd = assert_single_command(input); - assert_eq!(cmd.raw, "/cmd first /\nsecond"); -} - -#[test] -fn raw_fence_includes_opener_and_closer() { - let input = "/cmd ```\ncontent\n```"; - let cmd = assert_single_command(input); - assert_eq!(cmd.raw, "/cmd ```\ncontent\n```"); -} - -// --- Version and context --- - -#[test] -fn result_version_is_0_1_0() { - let result = parse("/test"); - assert_eq!(result.version, "0.1.0"); -} - -#[test] -fn context_fields_pass_through() { - let ctx = ParserContext { - source: Some("test.md".to_string()), - user: Some("tom".to_string()), - ..Default::default() - }; - let result = parse_to_domain("/test", ctx); - assert_eq!(result.context.source, Some("test.md".to_string())); - assert_eq!(result.context.user, Some("tom".to_string())); -} - -// --- JSON output --- - -#[test] -fn parse_slash_commands_returns_valid_json() { - let json = parse_slash_commands("/test arg", default_context()).unwrap(); - let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed["version"], "0.1.0"); - assert_eq!(parsed["commands"][0]["name"], "test"); -} - -#[test] -fn json_mode_field_is_single_line_string() { - let json = parse_slash_commands("/test arg", default_context()).unwrap(); - let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed["commands"][0]["arguments"]["mode"], "single-line"); -} - -#[test] -fn json_mode_field_is_continuation_string() { - let json = parse_slash_commands("/test arg /\nmore", default_context()).unwrap(); - let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed["commands"][0]["arguments"]["mode"], "continuation"); -} - -#[test] -fn json_mode_field_is_fence_string() { - let json = parse_slash_commands("/test ```\ncontent\n```", default_context()).unwrap(); - let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed["commands"][0]["arguments"]["mode"], "fence"); -} - -// --- Edge cases --- - -#[test] -fn unclosed_fence_at_eof_finalizes_command() { - let input = "/cmd ```\nline1\nline2"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); - assert_eq!(cmd.arguments.payload, "line1\nline2\n"); -} - -#[test] -fn continuation_at_eof_finalizes_command() { - let input = "/cmd first /"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Continuation); -} - -#[test] -fn multiple_spaces_before_slash_in_continuation() { - let input = "/cmd header /\n /next-cmd arg"; - let result = parse(input); - assert_eq!(result.commands.len(), 1); - assert!(result.commands[0].arguments.payload.contains("/next-cmd")); -} - -#[test] -fn fence_with_indented_closing() { - let input = "/cmd ```\ncontent\n ```"; - let cmd = assert_single_command(input); - assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); - assert_eq!(cmd.arguments.payload, "content\n"); -} - -// hfuzz found cases -#[test] -fn fuzz_roundtrip_mode_mismatch() { - use crate::{domain::ParserContext, parser::parse_to_domain, to_plaintext::to_plaintext}; - - let input = "\t/ru4\x0b/"; - - let ast1 = parse_to_domain(input, ParserContext::default()); - let plaintext = to_plaintext(&ast1); - - eprintln!("input bytes: {:?}", input.as_bytes()); - eprintln!("plaintext bytes: {:?}", plaintext.as_bytes()); - eprintln!("ast1 commands: {:#?}", ast1.commands); - - let ast2 = parse_to_domain(&plaintext, ParserContext::default()); - eprintln!("ast2 commands: {:#?}", ast2.commands); - - for (a, b) in ast1.commands.iter().zip(ast2.commands.iter()) { - assert_eq!(a.name, b.name, "Command name mismatch"); - assert_eq!(a.arguments.mode, b.arguments.mode, "Mode mismatch"); - assert_eq!(a.arguments.payload, b.arguments.payload, "Payload mismatch"); - } -} diff --git a/parser-core/src/parser/tests/proptest.rs b/parser-core/src/parser/tests/proptest.rs deleted file mode 100644 index 06d2b63..0000000 --- a/parser-core/src/parser/tests/proptest.rs +++ /dev/null @@ -1,52 +0,0 @@ -use proptest::prelude::*; - -use crate::{ - domain::{ParserContext, SlashParseResult}, - parser::parse_to_domain, - serialize::to_json, - to_plaintext::to_plaintext, -}; - -proptest! { - #[test] - fn serialization_never_panics(original in any::()) { - // Arbitrary ASTs should always serialize without panicking - let _json = to_json(&original) - .expect("Valid domain object should always serialize"); - } - - #[test] - fn parse_to_plaintext_roundtrip(input in "[\\s\\S]{0,200}") { - // The real roundtrip property: - // parse(input) -> to_plaintext -> parse again -> same structure - let ctx1 = ParserContext::default(); - let ast1 = parse_to_domain(&input, ctx1); - - let plaintext = to_plaintext(&ast1); - - let ctx2 = ParserContext::default(); - let ast2 = parse_to_domain(&plaintext, ctx2); - - prop_assert_eq!( - ast1.commands.len(), - ast2.commands.len(), - "Command count mismatch.\nInput: {:?}\nPlaintext: {:?}", - input, - plaintext - ); - - for (a, b) in ast1.commands.iter().zip(ast2.commands.iter()) { - prop_assert_eq!(&a.name, &b.name, "Command name mismatch"); - prop_assert_eq!(&a.arguments.mode, &b.arguments.mode, "Argument mode mismatch"); - prop_assert_eq!(&a.arguments.payload, &b.arguments.payload, "Payload mismatch"); - } - - prop_assert_eq!( - ast1.text_blocks.len(), - ast2.text_blocks.len(), - "Text block count mismatch.\nInput: {:?}\nPlaintext: {:?}", - input, - plaintext - ); - } -} diff --git a/parser-core/src/serialize.rs b/parser-core/src/serialize.rs deleted file mode 100644 index 5adfdfe..0000000 --- a/parser-core/src/serialize.rs +++ /dev/null @@ -1,192 +0,0 @@ -/// Serialization module — converts domain types to JSON via DTOs. -/// -/// Serde derives live here, not on domain types. This module is the -/// boundary between the pure domain and the JSON output contract. -use serde::Serialize; - -use crate::domain::{ - ArgumentMode, Command, CommandArguments, CommandRange, ParserContext, SlashParseResult, TextBlock, -}; - -#[derive(Serialize)] -struct SlashParseResultDto { - version: String, - context: ContextDto, - commands: Vec, - #[serde(skip_serializing_if = "Vec::is_empty")] - text_blocks: Vec, -} - -#[derive(Serialize)] -struct ContextDto { - #[serde(skip_serializing_if = "Option::is_none")] - source: Option, - #[serde(skip_serializing_if = "Option::is_none")] - timestamp: Option, - #[serde(skip_serializing_if = "Option::is_none")] - user: Option, - #[serde(skip_serializing_if = "Option::is_none")] - session_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - extra: Option, -} - -#[derive(Serialize)] -struct CommandDto { - id: String, - name: String, - raw: String, - range: RangeDto, - arguments: ArgumentsDto, - children: Vec, -} - -#[derive(Serialize)] -struct RangeDto { - start_line: usize, - end_line: usize, -} - -#[derive(Serialize)] -struct ArgumentsDto { - #[serde(skip_serializing_if = "String::is_empty")] - header: String, - mode: String, - #[serde(skip_serializing_if = "Option::is_none")] - fence_lang: Option, - payload: String, -} - -#[derive(Serialize)] -struct TextBlockDto { - id: String, - range: RangeDto, - content: String, -} - -// --- From impls: domain → DTO --- - -impl From<&CommandRange> for RangeDto { - fn from(range: &CommandRange) -> Self { - Self { start_line: range.start_line, end_line: range.end_line } - } -} - -impl From<&ArgumentMode> for &'static str { - fn from(mode: &ArgumentMode) -> Self { - match mode { - ArgumentMode::SingleLine => "single-line", - ArgumentMode::Continuation => "continuation", - ArgumentMode::Fence => "fence", - } - } -} - -impl From<&CommandArguments> for ArgumentsDto { - fn from(args: &CommandArguments) -> Self { - Self { - header: args.header.clone(), - mode: <&str>::from(&args.mode).to_string(), - fence_lang: args.fence_lang.clone(), - payload: args.payload.clone(), - } - } -} - -impl From<&Command> for CommandDto { - fn from(cmd: &Command) -> Self { - Self { - id: cmd.id.clone(), - name: cmd.name.clone(), - raw: cmd.raw.clone(), - range: RangeDto::from(&cmd.range), - arguments: ArgumentsDto::from(&cmd.arguments), - children: Vec::new(), - } - } -} - -impl From<&TextBlock> for TextBlockDto { - fn from(block: &TextBlock) -> Self { - Self { id: block.id.clone(), range: RangeDto::from(&block.range), content: block.content.clone() } - } -} - -impl From<&ParserContext> for ContextDto { - fn from(ctx: &ParserContext) -> Self { - Self { - source: ctx.source.clone(), - timestamp: ctx.timestamp.clone(), - user: ctx.user.clone(), - session_id: ctx.session_id.clone(), - extra: ctx.extra.clone(), - } - } -} - -impl From<&SlashParseResult> for SlashParseResultDto { - fn from(result: &SlashParseResult) -> Self { - Self { - version: result.version.clone(), - context: ContextDto::from(&result.context), - commands: result.commands.iter().map(CommandDto::from).collect(), - text_blocks: result.text_blocks.iter().map(TextBlockDto::from).collect(), - } - } -} - -/// Serialize a `SlashParseResult` to a JSON string. -pub fn to_json(result: &SlashParseResult) -> Result { - let dto = SlashParseResultDto::from(result); - serde_json::to_string(&dto) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::domain::ParserContext; - - #[test] - fn to_json_empty_result_produces_valid_json() { - let result = SlashParseResult { - version: "0.1.0".to_string(), - context: ParserContext::default(), - commands: Vec::new(), - text_blocks: Vec::new(), - }; - let json = to_json(&result).unwrap(); - let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed["version"], "0.1.0"); - assert!(parsed["commands"].as_array().unwrap().is_empty()); - // text_blocks should be omitted when empty - assert!(parsed.get("text_blocks").is_none()); - } - - #[test] - fn to_json_serializes_populated_context_correctly() { - let extra_json = serde_json::json!({ "feature_flag": true }); - let context = ParserContext { - source: Some("API".to_string()), - timestamp: Some("2026-03-13T00:00:00Z".to_string()), - user: Some("tom".to_string()), - session_id: Some("session-123".to_string()), - extra: Some(extra_json), - }; - - let result = SlashParseResult { - version: "0.1.0".to_string(), - context, - commands: Vec::new(), - text_blocks: Vec::new(), - }; - - let json = to_json(&result).unwrap(); - let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - - assert_eq!(parsed["context"]["source"], "API"); - assert_eq!(parsed["context"]["timestamp"], "2026-03-13T00:00:00Z"); - assert_eq!(parsed["context"]["user"], "tom"); - assert_eq!(parsed["context"]["session_id"], "session-123"); - assert_eq!(parsed["context"]["extra"]["feature_flag"], true); - } -} diff --git a/parser-core/src/to_plaintext.rs b/parser-core/src/to_plaintext.rs deleted file mode 100644 index 43940c3..0000000 --- a/parser-core/src/to_plaintext.rs +++ /dev/null @@ -1,64 +0,0 @@ -use crate::domain::{ArgumentMode, SlashParseResult}; - -pub fn to_plaintext(result: &SlashParseResult) -> String { - #[derive(Debug)] - enum Item<'a> { - Command(&'a crate::domain::Command), - Text(&'a crate::domain::TextBlock), - } - - let mut items: Vec = Vec::new(); - for cmd in &result.commands { - items.push(Item::Command(cmd)); - } - for txt in &result.text_blocks { - items.push(Item::Text(txt)); - } - - items.sort_by_key(|item| match item { - Item::Command(c) => c.range.start_line, - Item::Text(t) => t.range.start_line, - }); - - let mut output = String::new(); - - for (i, item) in items.iter().enumerate() { - match item { - Item::Command(c) => match c.arguments.mode { - ArgumentMode::SingleLine => { - if c.arguments.payload.is_empty() { - output.push_str(&format!("/{} ", c.name)); - } else { - output.push_str(&format!("/{} {}", c.name, c.arguments.payload)); - } - } - ArgumentMode::Continuation => { - output.push_str(&format!( - "/{} \\ -{}", - c.name, c.arguments.payload - )); - } - ArgumentMode::Fence => { - let lang = c.arguments.fence_lang.as_deref().unwrap_or(""); - output.push_str(&format!( - "/{} -```{} -{} -```", - c.name, lang, c.arguments.payload - )); - } - }, - Item::Text(t) => { - output.push_str(&t.content); - } - } - - if i < items.len() - 1 { - output.push('\n'); - } - } - - output -} From e06f3f0c94fc0f29cbab117900076e960e6d252d Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Sat, 21 Mar 2026 01:34:47 -0600 Subject: [PATCH 08/13] refac: let go of first iteration of the parser engine --- parser/tests/from-old-parser-core.txt | 51 --------------------------- 1 file changed, 51 deletions(-) delete mode 100644 parser/tests/from-old-parser-core.txt diff --git a/parser/tests/from-old-parser-core.txt b/parser/tests/from-old-parser-core.txt deleted file mode 100644 index cd5a5c5..0000000 --- a/parser/tests/from-old-parser-core.txt +++ /dev/null @@ -1,51 +0,0 @@ -3) Tests that cross application ⇄ infrastructure boundaries - -These belong in parser/src/test/ (or an integration-test style place) because they talk about JSON serialization or full result shape across layers: - - Version and context plumbing: - - result_version_is_0_1_0 - - context_fields_pass_through - - JSON output shape: - - parse_slash_commands_returns_valid_json - - json_mode_field_is_single_line_string - - json_mode_field_is_continuation_string - - json_mode_field_is_fence_string - -These will be driven by your new infrastructure adapter (whatever replaces parse_slash_commands). - - - - - - - - - -Scope for parser/test/ (crossing into infrastructure) - -Tests that involve JSON output or full result shape across layers: - - Version and context: - - result_version_is_0_1_0 - - context_fields_pass_through - - JSON shape / mode field: - - parse_slash_commands_returns_valid_json - - json_mode_field_is_single_line_string - - json_mode_field_is_continuation_string - - json_mode_field_is_fence_string - -These will move to parser/tests/ and exercise your new “parse → JSON” adapter, not the application module directly. \ No newline at end of file From 20394c7760ec6b1e82e2bc3e9b2dae0280aee3e8 Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Sat, 21 Mar 2026 01:36:08 -0600 Subject: [PATCH 09/13] refac: let go of first iteration of the parser engine --- .../tests/from-old-parser-core.txt | 157 ------------------ 1 file changed, 157 deletions(-) delete mode 100644 parser/src/application/tests/from-old-parser-core.txt diff --git a/parser/src/application/tests/from-old-parser-core.txt b/parser/src/application/tests/from-old-parser-core.txt deleted file mode 100644 index 96d973c..0000000 --- a/parser/src/application/tests/from-old-parser-core.txt +++ /dev/null @@ -1,157 +0,0 @@ -2) Tests that belong in parser/src/application/test/ - -These involve more than just a single command accumulator, but still live entirely in the application layer (no JSON IO, no external boundaries). They should call parse_document and assert on ParseResult: - - Single-line command structure: - - single_line_simple_command_parses_name - - single_line_simple_command_parses_header - - single_line_simple_command_has_single_line_mode - - single_line_simple_command_payload_matches_header - - single_line_command_with_no_args_has_empty_header - - single_line_command_with_complex_args - - Command name validation (ties into line_classify and overall parsing): - - command_name_with_hyphens_parses - - command_name_with_digits_parses - - command_starting_with_digit_is_not_command - - command_starting_with_uppercase_is_not_command - - slash_alone_is_not_command - - Leading whitespace and CRLF normalization: - - leading_spaces_before_slash_are_ignored_for_detection - - leading_tabs_before_slash_are_ignored_for_detection - - crlf_normalized_to_lf - - Multiple commands and IDs (needs ID generation + state machine): - - two_single_line_commands_parsed - - multiple_commands_have_sequential_ids - - Commands interspersed with text and text block ranges: - - commands_interspersed_with_text - - text_before_command_captured - - text_after_command_captured - - text_block_range_is_correct - - only_text_produces_no_commands - - empty_input_produces_empty_result - - Raw field on Command: - - raw_single_line_is_trimmed_line - - raw_continuation_includes_all_lines - - raw_fence_includes_opener_and_closer - -These need the full DocumentParser behavior, so they’re best as application-layer tests, not unit tests for a single helper. - - -Move these ones to higher layers instead of command_accumulate.rs: - - crlf_continuation_works → document-level parsing test. - - multiple_spaces_before_slash_in_continuation → document-level parsing test. - - EOF “finalizes command” aspects → command_finalize tests (for warnings and final payload), not PendingCommand. - -Do you want me to sketch one or two co - - - - - - - - - -Scope for application/test/ (parse_document-level) - -Tests that use the application layer only (no JSON, no transport) and assert on ParseResult: - - Single-line commands: - - single_line_simple_command_parses_name - - single_line_simple_command_parses_header - - single_line_simple_command_has_single_line_mode - - single_line_simple_command_payload_matches_header - - single_line_command_with_no_args_has_empty_header - - single_line_command_with_complex_args - - Command name validation: - - command_name_with_hyphens_parses - - command_name_with_digits_parses - - command_starting_with_digit_is_not_command - - command_starting_with_uppercase_is_not_command - - slash_alone_is_not_command - - Leading whitespace and CRLF normalization: - - leading_spaces_before_slash_are_ignored_for_detection - - leading_tabs_before_slash_are_ignored_for_detection - - crlf_normalized_to_lf - - crlf_continuation_works (as an overall behavior of parse_document) - - Multiple commands and IDs: - - two_single_line_commands_parsed - - multiple_commands_have_sequential_ids - (you’ll adapt this to whatever your new ID scheme is) - - Text blocks: - - commands_interspersed_with_text - - text_before_command_captured - - text_after_command_captured - - text_block_range_is_correct - - only_text_produces_no_commands - - empty_input_produces_empty_result - - Raw field if you decide raw is assembled at the application layer: - - raw_single_line_is_trimmed_line - - raw_continuation_includes_all_lines - - raw_fence_includes_opener_and_closer - -These belong under something like parser/src/application/test/mod.rs calling parse_document. \ No newline at end of file From 4fcaad11375d73028428a4f09fa73a83388e6414 Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Sun, 22 Mar 2026 22:56:27 -0600 Subject: [PATCH 10/13] feat(engine): next iteration, sig overhaul w/ tests and updated specs. --- docs/slash-command-syntax-rfc-v1.1.0.txt | 931 ++++++++++++++ docs/slash-parser-engine-spec-v0.5.0.md | 560 +++++++++ docs/slash-parser-sdk-spec-v0.2.0.md | 517 ++++++++ docs/slash-parser-spec-v0.3.0.md | 1096 ----------------- parser/Cargo.toml | 2 +- parser/direction.md | 122 -- parser/moon.yml | 4 - .../application/command_accumulate.txt | 7 - .../application/tests/mod.txt | 8 - parser/src/application/command_accumulate.rs | 413 ------- parser/src/application/command_finalize.rs | 392 ------ parser/src/application/document_parse.rs | 520 -------- parser/src/application/line_classify.rs | 467 ------- parser/src/application/line_join.rs | 357 ------ parser/src/application/mod.rs | 13 - parser/src/application/normalize.rs | 127 -- parser/src/application/text_collect.rs | 234 ---- parser/src/classify.rs | 501 ++++++++ parser/src/domain/errors.rs | 10 - parser/src/domain/mod.rs | 8 - parser/src/fence.rs | 606 +++++++++ .../tests => integration_tests}/mod.rs | 78 +- .../tests => integration_tests}/proptest.rs | 39 +- parser/src/join.rs | 432 +++++++ parser/src/lib.rs | 24 +- parser/src/mod.rs | 12 + parser/src/normalize.rs | 189 +++ parser/src/parse.rs | 544 ++++++++ parser/src/single_line.rs | 138 +++ parser/src/text.rs | 243 ++++ parser/src/{domain => }/types.rs | 14 +- parser/tests/mod.rs | 21 - parser/tests/orchestration_tests.rs | 322 +++++ 33 files changed, 5065 insertions(+), 3886 deletions(-) create mode 100644 docs/slash-command-syntax-rfc-v1.1.0.txt create mode 100644 docs/slash-parser-engine-spec-v0.5.0.md create mode 100644 docs/slash-parser-sdk-spec-v0.2.0.md delete mode 100644 docs/slash-parser-spec-v0.3.0.md delete mode 100644 parser/direction.md delete mode 100644 parser/proptest-regressions/application/command_accumulate.txt delete mode 100644 parser/proptest-regressions/application/tests/mod.txt delete mode 100644 parser/src/application/command_accumulate.rs delete mode 100644 parser/src/application/command_finalize.rs delete mode 100644 parser/src/application/document_parse.rs delete mode 100644 parser/src/application/line_classify.rs delete mode 100644 parser/src/application/line_join.rs delete mode 100644 parser/src/application/mod.rs delete mode 100644 parser/src/application/normalize.rs delete mode 100644 parser/src/application/text_collect.rs create mode 100644 parser/src/classify.rs delete mode 100644 parser/src/domain/errors.rs delete mode 100644 parser/src/domain/mod.rs create mode 100644 parser/src/fence.rs rename parser/src/{application/tests => integration_tests}/mod.rs (90%) rename parser/src/{application/tests => integration_tests}/proptest.rs (53%) create mode 100644 parser/src/join.rs create mode 100644 parser/src/mod.rs create mode 100644 parser/src/normalize.rs create mode 100644 parser/src/parse.rs create mode 100644 parser/src/single_line.rs create mode 100644 parser/src/text.rs rename parser/src/{domain => }/types.rs (77%) delete mode 100644 parser/tests/mod.rs create mode 100644 parser/tests/orchestration_tests.rs diff --git a/docs/slash-command-syntax-rfc-v1.1.0.txt b/docs/slash-command-syntax-rfc-v1.1.0.txt new file mode 100644 index 0000000..3a53344 --- /dev/null +++ b/docs/slash-command-syntax-rfc-v1.1.0.txt @@ -0,0 +1,931 @@ + + + + +Davidson [Page 1] + +Slash Command Syntax March 2026 + + + Slash Command Syntax 1.1.0 + +Abstract + + This document defines the syntax for slash commands embedded in + UTF-8 text. It specifies how a conforming parser partitions an + input string into an ordered sequence of commands and text blocks. + + The specification is format-agnostic. It defines observable + input-to-output behavior without prescribing internal architecture, + serialization format, or host binding conventions. + +Status of This Memo + + This document is an independent submission and is not the product of + any standards body. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (c) 2026 Tom D. Davidson. All rights reserved. + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . 2 + 2. Conventions and Definitions . . . . . . . . . . . . . . . . 3 + 3. Input Model . . . . . . . . . . . . . . . . . . . . . . . . 3 + 3.1. Line Ending Normalization . . . . . . . . . . . . . . . 3 + 3.2. Line Joining . . . . . . . . . . . . . . . . . . . . . 4 + 3.3. Logical Lines . . . . . . . . . . . . . . . . . . . . . 5 + 4. Syntax Elements . . . . . . . . . . . . . . . . . . . . . . 6 + 4.1. Command Name . . . . . . . . . . . . . . . . . . . . . 6 + 4.2. Command Line . . . . . . . . . . . . . . . . . . . . . 6 + 4.3. Arguments . . . . . . . . . . . . . . . . . . . . . . . 7 + 4.4. The Header . . . . . . . . . . . . . . . . . . . . . . 7 + 4.5. Invalid Slash Lines . . . . . . . . . . . . . . . . . . 7 + 5. Argument Modes . . . . . . . . . . . . . . . . . . . . . . 8 + 5.1. Single-Line Mode . . . . . . . . . . . . . . . . . . . 8 + 5.2. Fence Mode . . . . . . . . . . . . . . . . . . . . . . 8 + 5.2.1. Fence Opener . . . . . . . . . . . . . . . . . . . 8 + 5.2.2. Fence Body . . . . . . . . . . . . . . . . . . . . 9 + 5.2.3. Fence Closer . . . . . . . . . . . . . . . . . . . 9 + 5.2.4. Unclosed Fence . . . . . . . . . . . . . . . . . . 10 + 6. Document Partitioning . . . . . . . . . . . . . . . . . . . 10 + 6.1. Fence Body Lines . . . . . . . . . . . . . . . . . . . 10 + 6.2. Command Lines . . . . . . . . . . . . . . . . . . . . . 10 + 6.3. Text Lines . . . . . . . . . . . . . . . . . . . . . . 10 + 6.4. Text Blocks . . . . . . . . . . . . . . . . . . . . . . 11 + 6.5. Ordering and Identification . . . . . . . . . . . . . . 11 + 7. Output Requirements . . . . . . . . . . . . . . . . . . . . 11 + 7.1. Per-Command Output . . . . . . . . . . . . . . . . . . 11 + 7.2. Per-Text-Block Output . . . . . . . . . . . . . . . . . 12 + 7.3. Warnings . . . . . . . . . . . . . . . . . . . . . . . 12 + 7.4. Warning Types . . . . . . . . . . . . . . . . . . . . . 12 + 8. Conformance . . . . . . . . . . . . . . . . . . . . . . . . 13 + 8.1. Conformance Targets . . . . . . . . . . . . . . . . . . 13 + 8.2. Parser Conformance . . . . . . . . . . . . . . . . . . 13 + 8.3. Roundtrip Fidelity Invariant . . . . . . . . . . . . . 13 + 8.4. Determinism . . . . . . . . . . . . . . . . . . . . . . 14 + 9. Security Considerations . . . . . . . . . . . . . . . . . . 14 + 10. IANA Considerations . . . . . . . . . . . . . . . . . . . . 14 + Appendix A. Formal Syntax . . . . . . . . . . . . . . . . . . 14 + Appendix B. Parsing Examples . . . . . . . . . . . . . . . . . 16 + Appendix C. Reference State Machine . . . . . . . . . . . . . 19 + Appendix D. Change Log . . . . . . . . . . . . . . . . . . . . 20 + Normative References . . . . . . . . . . . . . . . . . . . . . 20 + Author's Address . . . . . . . . . . . . . . . . . . . . . . . 21 + + +1. Introduction + + Slash commands are a lightweight convention for embedding executable + directives in plain text. A line beginning with "/" followed by a + command name is treated as a command; all other lines are text. + + This convention is used in chat applications, developer tooling, + and AI agent interfaces. Despite its widespread adoption, no formal + syntax specification exists. This document provides one. + + The specification defines: + + o How input text is normalized and optionally joined across + physical lines. + + o How command lines are distinguished from text lines. + + o How command arguments are delimited, including an inline + single-line mode and a fenced multi-line mode using + backtick-delimited blocks. + + o How the input is partitioned into an ordered sequence of + commands and text blocks. + + o The minimum information a conforming parser must produce for + each element. + + The specification does NOT define: + + o A serialization format for parser output. Companion + specifications (such as a JSON schema) define concrete + output formats. + + o Internal parser architecture, state machines, memory + strategies, or pipeline stages. Any implementation that + produces conforming output for all inputs is valid. + + o Interpretation of command arguments. The parser treats all + argument content as opaque. + + +2. Conventions and Definitions + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL + NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "NOT RECOMMENDED", + "MAY", and "OPTIONAL" in this document are to be interpreted as + described in BCP 14 [RFC2119] [RFC8174] when, and only when, + they appear in all capitals, as shown here. + + The following terms are used throughout this document: + + physical line: A line in the normalized input, delimited by + U+000A (LF). + + logical line: One or more physical lines merged by backslash + continuation (Section 3.2). A physical line that is not + joined with any other line is also a logical line. + + command: A logical line whose first non-whitespace content + matches the command trigger pattern (Section 4.2). + + text block: A maximal contiguous sequence of non-command, non- + fence-body logical lines. + + payload: The argument content of a command, assembled according + to its argument mode (Section 5). + + fence body line: A physical line that falls between an unmatched + fence opener and its corresponding closer or EOF. + + header: The inline, non-fenced argument portion of a command + line, serving as the dispatch or routing segment. + + +3. Input Model + +3.1. Line Ending Normalization + + Before any other processing, the input MUST be normalized as + follows: + + 1. Replace all CRLF (U+000D U+000A) sequences with LF + (U+000A). + + 2. Replace all remaining bare CR (U+000D) characters with LF + (U+000A). + + After normalization, all line terminators are LF. The normalized + input is split on LF to produce a sequence of physical lines. + Each LF terminates a line and is not part of the line content. + A trailing LF at the end of input produces a trailing empty line. + + Literal escape sequences inside content (e.g., the two-character + sequence "\" "n" inside a JSON string) are ordinary characters, + not line terminators. They MUST be preserved verbatim. + +3.2. Line Joining (Backslash Continuation) + + After normalization, logical lines are derived from physical + lines by the following joining rules. This mechanism is + identical in behavior to POSIX shell backslash-newline removal. + + For each physical line, if the line ends with U+005C (REVERSE + SOLIDUS, "\"): + + 1. Remove the trailing reverse solidus. + + 2. Remove the line boundary (the LF that delimits the physical + line). + + 3. Concatenate the remainder directly with the next physical + line. No separator character is inserted. This is true POSIX + backslash-newline removal. + + 4. If the result still ends with "\", repeat from step 1 with + the subsequent physical line. + + Lines that do not end with "\" are left unchanged. + + The join marker is any "\" immediately before the line boundary, + regardless of what precedes it. + + If the final physical line of the input ends with "\" and there + is no subsequent line to join with, the trailing "\" is removed + and the line stands alone. This mirrors POSIX shell behavior. + + Fence body lines (Section 5.2.2) are never subject to line + joining. A trailing "\" inside a fence body is literal content. + +3.3. Logical Lines + + After joining, the parser operates on a sequence of logical + lines. Each logical line maps to one or more physical lines + from the normalized input. + + A conforming parser MUST track this mapping. When output + elements reference line positions, those positions MUST be + zero-based physical line numbers from the normalized input + (before joining). + + +4. Syntax Elements + +4.1. Command Name + + A command name is a string matching the following pattern: + + command-name = LCALPHA *((LCALPHA / DIGIT / "-") LCALPHA / DIGIT) + + Where LCALPHA is %x61-7A (lowercase ASCII letters a-z) and + DIGIT is %x30-39. A single lowercase letter is a valid + command name. Multi-character names MUST NOT end with a + hyphen. + + The formal ABNF is given in Appendix A. + + In prose: + + o Begins with a lowercase ASCII letter. + + o Followed by zero or more lowercase ASCII letters, ASCII + digits, or hyphens. + + o MUST NOT end with a hyphen. Hyphens are word separators + within multi-segment names (e.g., "call-tool"), not + trailing punctuation. + +4.2. Command Line + + A command line is a logical line whose first non-whitespace + character is U+002F (SOLIDUS, "/") and whose characters + immediately following the "/" form a valid command name + (Section 4.1). + + The "/" MUST be immediately followed by the command name with + no intervening whitespace. + +4.3. Arguments + + Everything after the first whitespace character following the + command name is the arguments portion. The whitespace between + the command name and the arguments is consumed as a separator + and is not included in the arguments string. + + The arguments portion may be: + + o Empty (command with no arguments). + + o Inline text (single-line mode, Section 5.1). + + o A fence opener (fence mode, Section 5.2). + + o Inline text followed by a fence opener: the text before the + fence becomes the header, the fenced content becomes the + payload. + +4.4. The Header + + The header is the inline, non-fenced argument portion of the + command line. + + o In single-line mode, the header and payload are identical + (the full arguments text). + + o In fence mode, the header contains the arguments text that + appears before the fence opener on the command line. + +4.5. Invalid Slash Lines + + If a logical line's first non-whitespace character is "/" but the + text after "/" does not match the command name pattern, the line + is NOT a command. Examples include a bare "/", "/123", "/Hello", + "/ space", and "/cmd-" (trailing hyphen). + + Such lines are text lines and are handled per Section 6.3. + + +5. Argument Modes + + Every command has exactly one of two argument modes, determined + at the point of detection. + +5.1. Single-Line Mode + + If the arguments portion does not contain a fence opener: + + o The full arguments text is both the header and the payload. + + o The argument mode is "single-line". + + o There is no fence language identifier. + + o The command is complete on its logical line. + +5.2. Fence Mode + + Fenced blocks attach a raw, multi-line payload to a command + using backtick-delimited block syntax. Fenced payloads are + verbatim: no parsing rules (including line joining) apply to + fence body lines. + + Only backtick (U+0060, "`") fences are recognized. Tilde + (U+007E, "~") fences MUST NOT be treated as fence openers. + +5.2.1. Fence Opener + + In the arguments portion of a command line, the first occurrence + of three or more consecutive backtick characters is a fence + opener. + + o Text before the backtick run, trimmed of trailing whitespace, + becomes the header. + + o The backtick run length is recorded as the fence marker + length. + + o Text after the backtick run, trimmed of leading whitespace, + if non-empty and consisting of a single token (no internal + whitespace), is the fence language identifier. + + The variable-length fence (three or more backticks) allows + content containing triple backticks to be fenced with four or + more backticks, avoiding collision. + +5.2.2. Fence Body + + Fence body lines are the physical lines (from the normalized + input, not joined logical lines) that fall between the fence + opener and the fence closer (or EOF). + + All fence body lines MUST be included in the payload verbatim, + preserving their original content including any trailing + backslashes. Lines are concatenated with LF (U+000A) separators + in the payload. The payload MUST NOT include a trailing LF + after the last body line. + + Fence body lines are never subject to line joining + (Section 3.2). Inside a fence, all content is literal payload: + command triggers, invalid slash lines, blank lines, and any + other content. + +5.2.3. Fence Closer + + A physical line is a fence closer for a given fence opener if, + after trimming leading and trailing whitespace: + + o It consists solely of backtick (U+0060) characters. + + o The number of backticks is greater than or equal to the + opener's backtick count. + + The fence closer line MUST NOT be included in the payload. + + Line joining applies to physical lines outside fences. If the + physical line containing a fence closer ends with a trailing + "\", the fence closes normally, and then the "\" triggers a + line join with the next physical line. The joined result is + outside the fenced command. + +5.2.4. Unclosed Fence + + If the input ends before a fence closer is found: + + o The command is complete with whatever payload has been + accumulated through EOF. + + o A warning of type "unclosed_fence" MUST be produced, + identifying the fence opener's physical line number. + + +6. Document Partitioning + + The input is partitioned into an ordered sequence of commands + and text blocks. Every physical line in the normalized input + belongs to exactly one of the following categories. + +6.1. Fence Body Lines + + A fence body line is any physical line that follows an unmatched + fence opener (Section 5.2.1) and precedes the corresponding + fence closer (Section 5.2.3) or EOF. Fence body lines are + consumed by the command that owns the fence. + +6.2. Command Lines + + A command line (Section 4.2) is any non-fence-body logical line + whose first non-whitespace content matches the command trigger + pattern. In fence mode, the command also consumes the fence + opener line, all fence body lines, and the fence closer line + (if present). + +6.3. Text Lines + + A text line is any non-fence-body logical line that is not a + command line. This includes blank lines, invalid slash lines + (Section 4.5), and lines containing arbitrary prose. + +6.4. Text Blocks + + Consecutive text lines form a single text block. + + o Blank lines within a text region are included in the text + block content. + + o Text block content preserves the original lines concatenated + with LF (U+000A) separators. + + o A new text block begins after a command is finalized, if + text lines follow. + + Text blocks exist so that the roundtrip invariant (Section 8.3) + can be satisfied, and so that tooling can distinguish executable + content from commentary. + +6.5. Ordering and Identification + + Commands MUST be assigned sequential zero-based identifiers in + encounter order: "cmd-0", "cmd-1", "cmd-2", etc. + + Text blocks MUST be assigned identifiers independently in the + same manner: "text-0", "text-1", "text-2", etc. + + Commands and text blocks MUST appear in the output in document + order. + + +7. Output Requirements + + This section defines the minimum information a conforming parser + MUST produce for each element, without prescribing a + serialization format. Companion specifications define concrete + output schemas. + +7.1. Per-Command Output + + For each command, a conforming parser MUST produce: + + identifier: Sequential zero-based ID ("cmd-0", "cmd-1", ...). + + name: The command name without the leading "/". + + raw source: The exact source text from the normalized input + (before line joining). For joined commands, this includes all + physical lines with their backslashes and LF separators. For + fenced commands, this includes the opener line, all body + lines, and the closer line (if present). + + line range: The inclusive physical line range (zero-based) in + the normalized input. + + header: The inline argument text before any fence opener. + + argument mode: Either "single-line" or "fence". + + fence language: The language identifier from the fence opener, + or absent/null if not in fence mode or no language was + specified. + + payload: The assembled argument content. In single-line mode, + identical to header. In fence mode, the verbatim content + between the fence delimiters with lines joined by LF. + +7.2. Per-Text-Block Output + + For each text block, a conforming parser MUST produce: + + identifier: Sequential zero-based ID ("text-0", "text-1", ...). + + line range: The inclusive physical line range (zero-based) in + the normalized input. + + content: The original text with lines joined by LF separators. + +7.3. Warnings + + For each non-fatal issue, a conforming parser MUST produce: + + warning type: A category identifier string. + + start line: The physical line number where the issue originated. + + message: A human-readable description. + +7.4. Warning Types + + This specification defines the following warning type. + Implementations MAY define additional types. + + +------------------+------------------------------------------+ + | Type | Condition | + +------------------+------------------------------------------+ + | unclosed_fence | EOF reached while inside a fenced block | + +------------------+------------------------------------------+ + + The "unclosed_fence" warning MUST include the fence opener's + physical line number as the start line. + + +8. Conformance + +8.1. Conformance Targets + + This specification defines a single conformance target: a + "conforming parser". A conforming parser is any software + component that accepts a UTF-8 string and produces output + satisfying the requirements in Sections 6 and 7. + +8.2. Parser Conformance + + A conforming parser: + + 1. MUST correctly partition the input into commands and text + blocks as defined in Section 6. + + 2. MUST produce the per-element output defined in Section 7. + + 3. MUST treat all argument content as opaque. A conforming + parser MUST NOT interpret, tokenize, or apply quoting, + escaping, or key-value semantics to the header or payload. + + 4. MUST be a total function: it always produces a valid result + for any input. An empty input produces a result with no + commands, no text blocks, and no warnings. Malformed + constructs produce partial results and warnings, never a + failure. + + 5. MUST NOT expose or require an abstract syntax tree as a + public artifact. + + There are no requirements on internal architecture, memory + usage, number of passes, state representation, or pipeline + structure. Any implementation that produces conforming output + for all inputs satisfies this specification. + +8.3. Roundtrip Fidelity Invariant + + Let P be a conforming parser and F be a formatter that generates + a canonical plaintext representation from P's output. The + following invariant MUST hold: + + For any valid input I, parsing, formatting, and parsing again + yields a structurally equivalent result: + + P(F(P(I))) = P(I) + + Structural equivalence means the same sequence of commands and + text blocks with identical values for: name, argument mode, + header, payload, fence language, and text block content. + + The invariant is structural, not byte-for-byte: raw source + values, line numbers, and whitespace details MAY differ due to + normalization and line joining. + +8.4. Determinism + + A conforming parser MUST be deterministic: given identical input + bytes, the output MUST be identical across invocations and across + conforming implementations. + + +9. Security Considerations + + This specification defines a syntax for detecting commands in + text. It does not define command execution semantics. + + Implementations that execute detected commands MUST validate + command names and arguments before execution. The opaque payload + principle (Section 8.2, item 3) means the parser provides no + sanitization of argument content. + + Extremely long lines, deeply nested backslash joins, or very + large fenced payloads could be used to exhaust resources. + Implementations SHOULD impose reasonable limits on line length, + join depth, and payload size. + + +10. IANA Considerations + + This document has no IANA actions. + + +Appendix A. Formal Syntax + + The following ABNF [RFC5234] defines the syntax elements of this + specification. This grammar defines the lexical structure; the + partitioning rules in Section 6 define how the parser assigns + lines to commands and text blocks. + + ; --- Character classes --- + + LCALPHA = %x61-7A + ; lowercase ASCII letters a-z + + HYPHEN = %x2D + ; "-" + + SLASH = %x2F + ; "/" + + BACKSLASH = %x5C + ; "\" + + BACKTICK = %x60 + ; "`" + + LF = %x0A + ; line feed + + CR = %x0D + ; carriage return + + SP = %x20 + ; space + + WSP = SP / %x09 + ; space or horizontal tab + + NONWSP = %x21-7E / %x80-10FFFF + ; any non-whitespace character + + TEXTCHAR = %x20-10FFFF / %x09 + ; any printable or whitespace, excluding + ; control characters other than TAB + + + ; --- Command name --- + + command-name = LCALPHA *((LCALPHA / DIGIT / HYPHEN) + (LCALPHA / DIGIT)) + ; single letter is valid; multi-char names + ; MUST NOT end with a hyphen + + + ; --- Physical and logical lines --- + + physical-line = *TEXTCHAR + + join-marker = BACKSLASH + ; trailing "\" before LF boundary + + ; Line joining produces logical lines from physical lines. + ; The joining algorithm is defined in prose in Section 3.2. + ; It cannot be fully expressed in ABNF because it involves + ; cross-line concatenation. + + + ; --- Command line --- + + command-line = *WSP SLASH command-name [1*WSP arguments] + + arguments = single-line-args / fence-args + + single-line-args = 1*TEXTCHAR + ; inline arguments with no fence opener + + fence-args = [header-text 1*WSP] fence-opener + ; optional header text before the fence + + header-text = 1*TEXTCHAR + ; text before the backtick run; trailing + ; whitespace is trimmed per Section 5.2.1 + + fence-opener = 3*BACKTICK [1*WSP lang-id] + + lang-id = 1*NONWSP + ; single token, no internal whitespace + + fence-closer = *WSP 3*BACKTICK *WSP + ; solely backticks after trimming; + ; count >= opener count (prose rule) + + + ; --- Text --- + + text-line = *TEXTCHAR + ; any logical line that is not a command-line + ; and is not a fence body line + + +Appendix B. Parsing Examples + + The following examples illustrate the expected partitioning for + various inputs. Results are shown in informal notation. + Conforming implementations are not required to produce this + notation; it is used here for clarity. + +B.1. Single-Line Command + + Input: + + /echo hello world + + Result: + + cmd-0: name="echo", mode=single-line, + header="hello world", payload="hello world", + range=[0,0] + +B.2. Joined Multi-Line Command + + Input (three physical lines): + + /deploy production \ + --region us-west-2 \ + --canary + + After joining: + + /deploy production --region us-west-2 --canary + + Result: + + cmd-0: name="deploy", mode=single-line, + header="production --region us-west-2 --canary", + payload=(same as header), + range=[0,2] + +B.3. Fenced Command with Header + + Input: + + /mcp call_tool write_file ```json + { "path": "/src/index.ts" } + ``` + + Result: + + cmd-0: name="mcp", mode=fence, + header="call_tool write_file", + fence_lang="json", + payload='{ "path": "/src/index.ts" }', + range=[0,2] + +B.4. Backslash Join into Fence + + Input (four physical lines): + + /mcp call_tool write_file \ + ```json + { "path": "foo" } + ``` + + After joining, physical lines 0-1 become: + + /mcp call_tool write_file ```json + + Result: + + cmd-0: name="mcp", mode=fence, + header="call_tool write_file", + fence_lang="json", + payload='{ "path": "foo" }', + range=[0,3] + +B.5. Text Blocks and Multiple Commands + + Input: + + Welcome to the deployment system. + (blank line) + /deploy staging + /notify team --channel ops + Deployment complete. + + Result: + + text-0: content="Welcome to the deployment system.\n", + range=[0,1] + cmd-0: name="deploy", payload="staging", + range=[2,2] + cmd-1: name="notify", payload="team --channel ops", + range=[3,3] + text-1: content="Deployment complete.", + range=[4,4] + +B.6. Invalid Slash Lines + + Input: + + /123 not a command + / bare slash + /Hello capitalized + /deploy staging + + Result: + + text-0: content=(first three lines), range=[0,2] + cmd-0: name="deploy", payload="staging", range=[3,3] + +B.7. Unclosed Fence + + Input: + + /mcp call_tool ```json + { "incomplete": true } + + Result: + + cmd-0: name="mcp", mode=fence, + header="call_tool", fence_lang="json", + payload='{ "incomplete": true }', + range=[0,1] + warning: type="unclosed_fence", start_line=0 + +B.8. Fence Closer with Trailing Backslash + + A line containing backticks followed by a trailing backslash is + NOT a valid closer (after trimming, the line is not solely + backticks). + + Input (six physical lines): + + /mcp call_tool write_file -c \ [line 0] + ```json [line 1] + { "path": "foo" } [line 2] + ``` \ [line 3] + \ [line 4] + production [line 5] + + Line 3 after trimming contains "``` \", which is not solely + backticks. The fence never closes. + + Result: + + cmd-0: name="mcp", mode=fence, + header="call_tool write_file -c", + fence_lang="json", + payload=(lines 2-5 verbatim), + range=[0,5] + warning: type="unclosed_fence", start_line=1 + +B.9. Proper Fence Close with Subsequent Content + + Input (six physical lines): + + /mcp call_tool write_file -c\ [line 0] + ```json [line 1] + { "path": "foo" } [line 2] + ``` [line 3] + \ [line 4] + production [line 5] + + Lines 0-1 join. Line 3 is a valid closer. Lines 4-5 join + outside the fence. + + Result: + + cmd-0: name="mcp", mode=fence, + header="call_tool write_file -c", + fence_lang="json", + payload='{ "path": "foo" }', + range=[0,3] + text-0: content="production", range=[4,5] + + + +Appendix C. Reference State Machine + + This appendix is non-normative. It provides a reference state + machine that produces conforming output. Conforming + implementations are not required to use this architecture. + + States: idle, in_fence + + +---------------+----------------------------+------------------+-----------+ + | Current | Condition | Action | Next | + +---------------+----------------------------+------------------+-----------+ + | idle | Logical line is a command | Finalize as | idle | + | | with no fence opener | single-line cmd | | + +---------------+----------------------------+------------------+-----------+ + | idle | Logical line is a command | Begin fence; | in_fence | + | | with a fence opener | record metadata | | + +---------------+----------------------------+------------------+-----------+ + | idle | Logical line is not a | Append to | idle | + | | command | current text blk | | + +---------------+----------------------------+------------------+-----------+ + | in_fence | Physical line is a closing | Finalize fenced | idle | + | | fence | command | | + +---------------+----------------------------+------------------+-----------+ + | in_fence | Physical line is not a | Append to | in_fence | + | | closing fence | payload | | + +---------------+----------------------------+------------------+-----------+ + | in_fence | EOF without closing fence | Finalize cmd | idle | + | | | with warning | | + +---------------+----------------------------+------------------+-----------+ + + + + + +Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, + DOI 10.17487/RFC2119, March 1997, + . + + [RFC5234] Crocker, D., Ed. and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, + DOI 10.17487/RFC5234, January 2008, + . + + [RFC8174] Leiba, B., "Ambiguity of Uppercase vs Lowercase in + RFC 2119 Key Words", BCP 14, RFC 8174, + DOI 10.17487/RFC8174, May 2017, + . + + +Author's Address + + Tom D. Davidson + + Email: tom@tomdavidson.org + URI: tomdavidson.org diff --git a/docs/slash-parser-engine-spec-v0.5.0.md b/docs/slash-parser-engine-spec-v0.5.0.md new file mode 100644 index 0000000..b37cda9 --- /dev/null +++ b/docs/slash-parser-engine-spec-v0.5.0.md @@ -0,0 +1,560 @@ +# Slash Command Parser Engine Specification v0.5.0 + +> **Author:** Tom D. Davidson +> **Email:** tom@tomdavidson.org +> **URI:** https://tomdavidson.org +> **Location:** Utah +> **Date:** March 2026 +> **Copyright:** (c) 2026 Tom D. Davidson. All rights reserved. +> **Distribution:** Unlimited. + +## Abstract + +This document specifies the Slash Command Parser engine: a Rust library crate that implements the syntax rules defined in the Slash Command Syntax v1.1.0 [SYNTAX-RFC]. The engine consumes UTF-8 text and produces Rust domain types. It performs no serialization and has no dependencies on JSON, serde, or any output format. + +Serialization to JSON, WASM bindings, WASI targets, CLI tools, and language-specific SDKs are defined in a separate SDK specification [SDK-SPEC]. + +## 1. Introduction + +The Slash Command Parser engine is the core Rust crate that implements the parsing rules defined in [SYNTAX-RFC]. It is a pure library with the following properties: + +- No I/O. The engine does not read files, make network requests, or interact with the environment. +- No serialization. The engine produces Rust types (structs and enums). It has no dependency on serde, serde_json, or any serialization framework. +- No global mutable state. All inputs are passed as function parameters. All outputs are returned as owned values. +- No unsafe code. The engine uses only safe Rust. +- Minimal dependencies. The only runtime dependency is thiserror for error type derivation. + +The engine crate is designed to be consumed by three targets: + +1. Rust SDK: adds serde serialization, JSON schema conformance, and a higher-level API. +2. WASM target: compiled to WebAssembly with bindings via wasm-bindgen. +3. WASI target: compiled for WASI runtimes with stdin/stdout interfaces. + +All three targets are specified in [SDK-SPEC]. This document covers only the engine itself. + +## 2. Scope and Boundaries + +The engine is responsible for: + +- Implementing all syntax rules from [SYNTAX-RFC] Sections 3-8. +- Producing output conforming to [SYNTAX-RFC] Section 7. +- Exposing domain types sufficient for downstream serialization. + +The engine is NOT responsible for: + +- JSON serialization or JSON schema. These are SDK concerns. +- Context object handling. The engine accepts a version string. Context pass-through is the SDK's responsibility. +- JSONL streaming. The engine parses one document at a time. Pipeline orchestration is the SDK's responsibility. +- CLI argument parsing, file I/O, or any environment interaction. + +## 3. Domain Types + +The engine exposes the following public types from its domain module. All types derive `Debug`, `Clone`, `PartialEq`, and `Eq` (except `ParseResult`, which derives `Debug`, `Clone`, `PartialEq`). All fields are public. + +### 3.1. ParseResult + +The top-level return type from `parse_document`. + +```rust +struct ParseResult { + version: String, + commands: Vec, + text_blocks: Vec, + warnings: Vec, +} +``` + +- `version` is always set to `SPEC_VERSION` (Section 14). +- `commands` contains commands in document order per [SYNTAX-RFC] Section 6.5. +- `text_blocks` contains text blocks in document order per [SYNTAX-RFC] Section 6.5. +- `warnings` contains non-fatal issues. Empty if none. + +`ParseResult` has no `context` field. Context is a serialization concern handled by the SDK. The SDK wraps `ParseResult` into an envelope that includes caller-supplied context. + +### 3.2. Command + +```rust +struct Command { + id: String, + name: String, + raw: String, + range: LineRange, + arguments: CommandArguments, +} +``` + +- `id` follows the pattern `cmd-{n}` where n is a zero-based sequential index per [SYNTAX-RFC] Section 6.5. +- `name` matches `[a-z]([a-z0-9-]*[a-z0-9])?` per [SYNTAX-RFC] Section 4.1. A command name MUST NOT end with a hyphen. +- `raw` contains the exact normalized source text (before line joining) per [SYNTAX-RFC] Section 7.1. +- `range` covers zero-based physical line numbers per [SYNTAX-RFC] Section 3.3. + +### 3.3. CommandArguments + +```rust +struct CommandArguments { + header: String, + mode: ArgumentMode, + fence_lang: Option, + payload: String, +} +``` + +- `header` is the inline argument text before any fence opener per [SYNTAX-RFC] Section 4.4. +- `mode` indicates how the payload was assembled. +- `fence_lang` is `Some(lang)` when the fence opener included a language identifier, `None` otherwise. +- `payload` is the assembled argument content per [SYNTAX-RFC] Section 5. + +### 3.4. ArgumentMode + +```rust +enum ArgumentMode { + SingleLine, + Fence, +} +``` + +Corresponds to the mode string values "single-line" and "fence" in [SYNTAX-RFC] Section 7.1. String serialization is the SDK's responsibility. + +### 3.5. TextBlock + +```rust +struct TextBlock { + id: String, + range: LineRange, + content: String, +} +``` + +- `id` follows the pattern `text-{n}` per [SYNTAX-RFC] Section 6.5. +- `content` contains physical lines joined with LF separators per [SYNTAX-RFC] Section 7.2. + +### 3.6. LineRange + +```rust +struct LineRange { + start_line: usize, + end_line: usize, +} +``` + +Both fields are zero-based physical line indices from the normalized input (after line-ending normalization, before line joining). The range is inclusive on both ends. + +### 3.7. Warning + +```rust +struct Warning { + w_type: String, + start_line: Option, + message: Option, +} +``` + +- `w_type` is a snake_case warning category identifier. +- `start_line` is the physical line where the issue was detected. +- `message` is a human-readable description. + +The field is named `w_type` rather than `type` because `type` is a reserved keyword in Rust. The SDK serializes this field as `type` in JSON output. + +Defined warning types: + +| w_type | When Emitted | Reference | +|---|---|---| +| `"unclosed_fence"` | Fence reaches EOF without a valid closer | [SYNTAX-RFC] Section 5.2.4 | + +## 4. Entry Points + +### 4.1. parse_document + +```rust +pub fn parse_document(input: &str) -> ParseResult +``` + +This is the sole public entry point of the engine. It accepts a UTF-8 string slice and returns a `ParseResult`. + +The function: + +1. Normalizes line endings per [SYNTAX-RFC] Section 3.1. +2. Splits into physical lines per [SYNTAX-RFC] Section 3.1. +3. Processes lines sequentially per [SYNTAX-RFC] Section 6. +4. Returns a complete `ParseResult`. + +There is no second entry point that accepts context. Context is added by the SDK wrapper. + +### 4.2. Total Function Guarantee + +`parse_document` MUST return a valid `ParseResult` for any input. It MUST NOT panic, return an error type, or use any fallible return mechanism. There is no `Result` variant. Malformed input produces commands with partial data and corresponding entries in the warnings vector. + +This requirement derives from [SYNTAX-RFC] Section 8.2. + +## 5. Processing Pipeline + +The engine processes input through three stages. The stages are conceptually sequential but may be interleaved in implementation. + +### 5.1. Stage 1: Normalization + +Implements [SYNTAX-RFC] Section 3.1. + +- Input: `&str` (raw UTF-8 input) +- Output: `String` (all line endings are LF) + +The normalize module performs two replacements: + +1. CRLF -> LF +2. Bare CR -> LF + +This stage is a pure string transformation with no state. + +### 5.2. Stage 2: Physical Line Splitting + +Implements [SYNTAX-RFC] Section 3.1. + +- Input: `&str` (normalized text) +- Output: `Vec<&str>` (physical lines) + +The normalized text is split on LF. Each LF terminates a line and is not part of the line content. A trailing LF produces a trailing empty string. + +Implementation note: the current engine pops a trailing empty element when the input ends with LF. This matches [SYNTAX-RFC] Section 3.1 because the trailing empty line contains no content and cannot affect parsing. Both behaviors (keeping or popping the trailing empty) are conformant as long as the output is identical. + +### 5.3. Stage 3: Sequential Line Processing + +Implements [SYNTAX-RFC] Sections 4, 5, 6, and 7. + +The engine maintains two states: + +- Idle: the engine obtains the next logical line via the line joiner (Section 7), classifies it (Section 8), and dispatches to command or text block accumulation. +- In-fence: the engine obtains the next physical line directly (bypassing the line joiner), checks for a fence closer, and either appends to the payload or finalizes the command. + +The state transitions correspond exactly to [SYNTAX-RFC] Appendix C. + +The key invariant: the engine always knows whether a fence is open before deciding whether to apply line joining to the current physical line. This eliminates any circular dependency between joining and fence detection. + +## 6. Whitespace + +Per [SYNTAX-RFC], whitespace means exactly two characters: U+0020 (SPACE) and U+0009 (HORIZONTAL TAB). + +All whitespace checks in the engine MUST use this definition. Specifically: + +- Command name termination: the first SP or HTAB after the command name ends the name. +- Argument separator: the whitespace between command name and arguments is SP or HTAB. +- Fence closer trimming: leading and trailing SP and HTAB are removed before checking for solely-backtick content. +- Header trimming: trailing SP and HTAB are removed from text before a fence opener. + +The engine MUST NOT use Rust's `char::is_whitespace()` or `char::is_ascii_whitespace()` for these purposes. `char::is_whitespace()` includes 25+ Unicode whitespace characters (U+00A0, U+1680, U+2000-200A, U+2028, U+2029, U+202F, U+205F, U+3000) that are not recognized by this specification. `char::is_ascii_whitespace()` includes U+000A (LF), U+000C (FF), and U+000D (CR) which have already been consumed by normalization and line splitting. + +Implementations SHOULD define a helper function: + +```rust +fn is_wsp(c: char) -> bool { + c == ' ' || c == '\t' +} +``` + +and use it consistently throughout all modules. + +## 7. Line Joining + +### 7.1. POSIX Semantics + +Implements [SYNTAX-RFC] Section 3.2. + +When a physical line ends with U+005C ("\\"), the backslash and the line boundary are removed and the remainder is concatenated directly with the next physical line. No separator character is inserted. + +This is true POSIX backslash-newline removal. The v0.3.0 engine inserted a single space between joined lines. This is no longer the case. + +### 7.2. Fence Immunity + +Per [SYNTAX-RFC] Section 5.2.2, the line joiner MUST NOT be invoked for physical lines consumed while the parser is in in-fence state. + +The engine enforces this by having the document parser call `next_physical()` (which returns the raw physical line) instead of `next_logical()` (which applies joining) when in in-fence state. + +### 7.3. LogicalLine + +The line joiner produces `LogicalLine` values: + +```rust +struct LogicalLine { + text: String, + first_physical: usize, + last_physical: usize, +} +``` + +- `text` is the joined content. +- `first_physical` is the zero-based index of the first physical line that contributed to this logical line. +- `last_physical` is the zero-based index of the last physical line that contributed. + +`LogicalLine` is an internal type, not part of the public API. + +## 8. Line Classification + +Implements [SYNTAX-RFC] Section 4. + +The line classifier receives a logical line and produces one of: + +```rust +enum LineKind { + Command(CommandHeader), + Text, +} +``` + +`CommandHeader` is an internal struct carrying the parsed fields: raw, name, header_text, mode, fence_lang, and fence_backtick_count. + +Classification rules: + +1. Trim leading whitespace (SP and HTAB only). +2. If the first character is not "/", return `Text`. +3. Extract the command name after "/". If it does not match `[a-z]([a-z0-9-]*[a-z0-9])?`, return `Text`. Names ending with a hyphen are invalid per [SYNTAX-RFC] Section 4.1. +4. Extract the arguments portion after the first whitespace. +5. Search the arguments for a fence opener (three or more consecutive backticks). If found, set mode to `Fence` and extract header_text and fence_lang. Otherwise, set mode to `SingleLine`. +6. Return `Command(header)`. + +## 9. Command Accumulation + +### 9.1. PendingCommand + +While a command is being assembled (during fence mode), the engine maintains a `PendingCommand`: + +```rust +struct PendingCommand { + id: usize, + name: String, + header_text: String, + mode: ArgumentMode, + fence_lang: Option, + fence_backtick_count: usize, + start_line: usize, + end_line: usize, + payload_lines: Vec, + raw_lines: Vec, + is_open: bool, +} +``` + +`PendingCommand` is an internal type, not part of the public API. + +For single-line commands, `PendingCommand` is created and immediately finalized (`is_open` is false). For fence commands, `PendingCommand` stays open until a closer is found or EOF. + +### 9.2. AcceptResult + +When a physical line is offered to an open `PendingCommand`: + +```rust +enum AcceptResult { + Consumed, // Line appended to payload. + Completed, // Line was a valid closer; command finalized. + Rejected, // Command was already closed; line not consumed. +} +``` + +### 9.3. Finalization + +The `finalize_command` function consumes a `PendingCommand` and produces a `FinalizedCommand`: + +```rust +struct FinalizedCommand { + command: Command, + warnings: Vec, +} +``` + +Finalization: + +1. Formats `id` as `cmd-{n}` from the sequential counter. +2. Joins `raw_lines` with "\n" to produce the `raw` field. +3. Joins `payload_lines` with "\n" to produce the `payload` field. +4. If the command is a fence and `is_open` is true (EOF reached), emits an `"unclosed_fence"` warning. +5. Returns the `Command` and any warnings. + +`FinalizedCommand` is an internal type. + +## 10. Text Block Accumulation + +Text blocks are accumulated via `PendingText`: + +```rust +struct PendingText { + start_line: usize, + end_line: usize, + lines: Vec, +} +``` + +Physical lines covered by non-command logical lines are appended to `PendingText`. When a command is encountered or EOF is reached, the pending text block is finalized: + +1. Join lines with "\n" to produce `content`. +2. Format `id` as `text-{n}` from the text block counter. +3. Set range from `start_line` and `end_line`. + +Text blocks store physical lines (before joining). A logical line formed by backslash continuation contributes all of its constituent physical lines (with backslashes intact) to the text block content. This preserves the original source for roundtrip fidelity. + +## 11. Warning Types + +All warning type strings use snake_case. + +The engine defines the following warning types: + +| w_type | When Emitted | +|---|---| +| `"unclosed_fence"` | EOF reached while in in-fence state | + +Additional warning types MAY be added in future versions. The `Warning` struct's `w_type` field is a `String` (not an enum) to allow forward-compatible extension without breaking changes. + +## 12. Conformance to Syntax RFC + +The engine implements all normative requirements from [SYNTAX-RFC] Section 8. + +### 12.1. Determinism + +Per [SYNTAX-RFC] Section 8.4, `parse_document` MUST produce identical output for identical input. The engine uses no randomness, no hash maps with non-deterministic iteration order, and no floating point arithmetic. + +### 12.2. Incremental Emission + +The engine finalizes each command and text block as soon as its last physical line is consumed. + +In the current implementation, finalized commands and text blocks are pushed to vectors in the `ParseCtx` and assembled into `ParseResult` at the end. This satisfies incremental emission because each element is finalized (not revisited) on the spot. + +A future streaming implementation could emit elements via a callback or channel instead of collecting into vectors. The engine architecture supports this without structural changes. + +### 12.3. Opaque Payload + +Per [SYNTAX-RFC] Section 8.2, the engine never interprets argument content. The line classifier extracts argument boundaries but does not tokenize, parse, or validate the header or payload strings. + +### 12.4. Roundtrip Fidelity + +Per [SYNTAX-RFC] Section 8.3, the engine preserves sufficient information for a formatter to reconstruct input that re-parses to a structurally equivalent result. The `raw` field and text block `content` field contain the pre-join physical lines needed for reconstruction. + +## 13. Internal Module Architecture + +### 13.1. Module Map + +The engine crate has the following module structure: + +``` +parser/ + src/ + lib.rs Public API re-exports. + domain/ + mod.rs Type re-exports. + types.rs ParseResult, Command, CommandArguments, + ArgumentMode, TextBlock, LineRange, + SPEC_VERSION. + errors.rs Warning. + application/ + mod.rs Module declarations, re-export of + parse_document. + normalize.rs Line-ending normalization (Stage 1). + line_join.rs Backslash continuation (Stage 3, idle). + line_classify.rs Command detection (Stage 3). + command_accumulate.rs PendingCommand, start_command, + accept_line. + command_finalize.rs finalize_command. + text_collect.rs PendingText, start_text, append_text, + finalize_text. + document_parse.rs parse_document orchestration. + tests/ + mod.rs Cross-module integration tests. + proptest.rs Property-based tests. +``` + +### 13.2. Dependency Rules + +Modules within the engine follow a strict dependency hierarchy: + +1. `domain/` depends on nothing. +2. `application/` modules depend on `domain/` types. +3. `application/` modules do not depend on each other except through `document_parse.rs`, which orchestrates all modules. +4. No module depends on any serialization library. +5. No module performs I/O. + +### 13.3. Test Layering + +Tests are organized in two layers: + +Layer 1 (unit tests): Each module file contains `#[cfg(test)]` tests that exercise only that module's logic. These tests use direct function calls, not `parse_document`. + +Layer 2 (integration tests): The `application/tests/` directory contains tests that exercise cross-module composition via `parse_document`. These include the Appendix B examples from [SYNTAX-RFC]. + +Property-based tests use the proptest crate. Slow property tests are gated behind the "tdd" feature flag and excluded from watch-mode iteration: + +```rust +#[cfg_attr(feature = "tdd", ignore)] +``` + +This allows fast red-green-refactor cycles with: + +```sh +cargo nextest run --features tdd +``` + +## 14. Version Constant + +```rust +pub const SPEC_VERSION: &str = "0.5.0"; +``` + +This constant is set in `domain/types.rs` and populated into `ParseResult.version` by `parse_document`. + +The version tracks the engine spec version. The SDK may override the version in the serialized envelope to match the syntax RFC version ("1.1.0") if required by [SYNTAX-RFC]. + +## 15. Migration Notes from v0.4.0 + +The following changes affect existing code when upgrading from engine v0.4.0 to v0.5.0: + +### 15.1. Command Name: Trailing Hyphen Prohibition + +- v0.4.0: Command names matched `[a-z][a-z0-9-]*`, allowing trailing hyphens. +- v0.5.0: Command names match `[a-z]([a-z0-9-]*[a-z0-9])?`. Names ending with a hyphen are invalid and the line is classified as text. + +Impact: Update the regex or matching logic in `line_classify.rs`. Add test cases for inputs like `/cmd-` (should be text, not a command). + +### 15.2. Syntax RFC Reference + +- v0.4.0: Referenced Slash Command Parser Syntax Specification v0.3.1. +- v0.5.0: References Slash Command Syntax v1.1.0 [SYNTAX-RFC]. + +Impact: Update any documentation or comments that reference the old spec version. + +### 15.3. SPEC_VERSION Constant + +- v0.4.0: `SPEC_VERSION = "0.4.0"` +- v0.5.0: `SPEC_VERSION = "0.5.0"` + +Impact: Update `domain/types.rs` and any tests that assert on the version string. + +## 16. Migration Notes from v0.3.0 + +These notes are retained for projects upgrading from v0.3.0 directly. + +### 16.1. Line Joining: No Space Insertion + +- v0.3.0: Joined lines were concatenated with a single space. +- v0.4.0+: Joined lines are concatenated directly (true POSIX). + +Impact: Remove the space insertion in `line_join.rs`. Update all test assertions that included the inserted space. + +### 16.2. Whitespace Definition + +- v0.3.0: Used Rust's `char::is_whitespace()` in some code paths. +- v0.4.0+: All whitespace checks use SP (U+0020) and HTAB (U+0009) only. + +Impact: Replace all uses of `char::is_whitespace()` and `char::is_ascii_whitespace()` with a dedicated `is_wsp()` helper. + +### 16.3. Warning Type String + +- v0.3.0: Used `"unclosed-fence"` (kebab-case). +- v0.4.0+: Uses `"unclosed_fence"` (snake_case). + +Impact: Update the string literal in `command_finalize.rs` and all test assertions that match on the warning type. + +## Normative References + +- **[SYNTAX-RFC]** Davidson, T. D., "Slash Command Syntax, Version 1.1.0", March 2026. +- **[SDK-SPEC]** Davidson, T. D., "Slash Command Parser SDK Specification" (forthcoming). + +## Author + +Tom D. Davidson +Email: tom@tomdavidson.org +URI: https://tomdavidson.org +Utah diff --git a/docs/slash-parser-sdk-spec-v0.2.0.md b/docs/slash-parser-sdk-spec-v0.2.0.md new file mode 100644 index 0000000..d8e2ae8 --- /dev/null +++ b/docs/slash-parser-sdk-spec-v0.2.0.md @@ -0,0 +1,517 @@ +# Slash Command Parser SDK Specification v0.2.0 + +> **Author:** Tom D. Davidson +> **Email:** tom@tomdavidson.org +> **URI:** https://tomdavidson.org +> **Location:** Utah +> **Date:** March 2026 +> **Copyright:** (c) 2026 Tom D. Davidson. All rights reserved. +> **Distribution:** Unlimited. + +## Abstract + +This document specifies the SDK layer for the Slash Command Parser. The SDK wraps the engine crate [ENGINE-SPEC] and provides serialization, bindings, and runtime integration for three targets: a Rust SDK crate, a WebAssembly (WASM) module, and a WASI executable. + +The engine is a pure parsing library that produces Rust types. The SDK is responsible for everything between those Rust types and the outside world: JSON serialization, the context pass-through envelope, JSONL streaming, JavaScript interop via wasm-bindgen, and stdin/stdout I/O for WASI runtimes. + +## 1. Introduction + +The Slash Command Parser engine [ENGINE-SPEC] is a pure Rust library that consumes UTF-8 text and produces Rust domain types. It has no knowledge of JSON, JavaScript, file I/O, or any serialization format. + +The SDK bridges the engine to consumers: + +- Rust applications import the SDK crate and receive serde-enabled types that serialize to a JSON schema conforming to [SYNTAX-RFC]. +- JavaScript/TypeScript applications import a WASM module that accepts a string and returns a JSON object matching the same schema. +- WASI runtimes execute a binary that reads UTF-8 from stdin and writes JSON (or JSONL) to stdout. + +All three targets produce output conforming to the same JSON schema. The SDK is the single point where serialization decisions are made. + +## 2. Scope and Boundaries + +The SDK is responsible for: + +- JSON serialization of engine types. +- The context pass-through envelope (Section 5). +- JSONL streaming for pipeline use cases (Section 9). +- WASM bindings via wasm-bindgen (Section 7). +- WASI binary with stdin/stdout I/O (Section 8). +- TypeScript type declarations for the WASM module (Section 7.5). + +The SDK is NOT responsible for: + +- Parsing logic. All parsing is delegated to the engine. +- CLI argument parsing beyond what is needed for the WASI binary. +- Any behavior not defined in [SYNTAX-RFC] or [ENGINE-SPEC]. + +## 3. Crate Architecture + +The SDK is a separate crate (or set of crates) that depends on the engine crate. The recommended workspace layout: + +``` +workspace/ + parser/ Engine crate [ENGINE-SPEC] + parser-sdk/ Rust SDK crate (this spec) + src/ + lib.rs Public API, re-exports + envelope.rs SlashParserResult assembly + serial.rs Serde configuration + Cargo.toml + parser-wasm/ WASM target crate + src/ + lib.rs wasm-bindgen exports + Cargo.toml + parser-wasi/ WASI target crate + src/ + main.rs stdin/stdout binary + Cargo.toml +``` + +The three output targets (Rust SDK, WASM, WASI) MAY be separate crates or feature-gated within a single crate. This spec defines them as separate crates for clarity. Implementations MAY consolidate them if the dependency footprint is acceptable. + +Dependency direction: + +``` +parser-sdk --> parser (engine) +parser-wasm --> parser-sdk --> parser +parser-wasi --> parser-sdk --> parser +``` + +The engine crate MUST NOT depend on the SDK. + +## 4. JSON Serialization + +### 4.1. Envelope Assembly + +The engine produces a `ParseResult` containing commands, text_blocks, warnings, and a version string. The SDK wraps this into a `SlashParserResult` envelope by adding the context object. + +```rust +struct SlashParserResult { + version: String, + context: serde_json::Value, + commands: Vec, + text_blocks: Vec, + warnings: Vec, +} +``` + +The SDK constructs the envelope by: + +1. Calling `parser::parse_document(input)` to obtain `ParseResult`. +2. Combining `ParseResult` fields with the caller-supplied context (or an empty object) to produce `SlashParserResult`. +3. Serializing `SlashParserResult` to JSON via serde_json. + +The engine's `ParseResult` is never exposed directly in the SDK's public API. Consumers interact only with `SlashParserResult`. + +### 4.2. Field Naming + +All JSON field names use snake_case. Serde's default Rust field naming matches snake_case, so no rename attributes are needed for most fields. + +Exception: the `Warning` struct's Rust field `w_type` MUST be serialized as `type` in JSON output. This requires: + +```rust +#[serde(rename = "type")] +pub w_type: String, +``` + +### 4.3. ArgumentMode Serialization + +The engine's `ArgumentMode` enum has variants `SingleLine` and `Fence`. These MUST serialize to the JSON string values `"single-line"` and `"fence"` respectively. + +```rust +#[serde(rename_all = "kebab-case")] +pub enum ArgumentMode { + SingleLine, + Fence, +} +``` + +This produces `"single-line"` and `"fence"`, matching [SYNTAX-RFC] Section 7.1. + +### 4.4. Warning Type Serialization + +Warning type values use snake_case as plain strings. The `type` field is a `String`, not an enum, to allow forward-compatible extension. + +Currently defined types: + +- `"unclosed_fence"` + +The SDK MUST NOT validate or restrict the warning type string. Future engine versions may introduce new warning types without requiring SDK changes. + +### 4.5. Null Handling + +Optional fields that are `None` in Rust MUST serialize to JSON `null`, not be omitted. Specifically: + +- `fence_lang`: null when not in fence mode or no language specified. +- `start_line` and `message` on Warning: null when not applicable. + +Omitting them would violate the schema. The SDK MUST ensure required fields are always present in the output, even when their value is null. + +Serde configuration for nullable fields: + +```rust +#[serde(serialize_with = "serialize_option_as_null")] +``` + +Or, more simply, `Option` with serde's default behavior combined with the schema's `required` constraint. + +### 4.6. Schema Conformance + +The serialized JSON output of the SDK MUST validate against the JSON schema defined for [SYNTAX-RFC]. The SDK's test suite SHOULD include schema validation tests that deserialize SDK output and validate it against the schema programmatically. + +## 5. Context Object + +### 5.1. Pass-Through Semantics + +The context object is caller-supplied metadata that the SDK passes through to the output envelope unchanged. The engine never sees it. + +The SDK accepts context as a `serde_json::Value` (for the Rust SDK), a `JsValue` (for WASM), or a JSON string (for WASI stdin). + +The SDK MUST NOT: + +- Validate context fields. +- Add, remove, or modify context fields. +- Interpret context content in any way. + +The only constraint is that context MUST be a JSON object (not an array, string, number, or null). If a non-object is provided, the SDK SHOULD substitute an empty object `{}`. + +### 5.2. Default Context + +When no context is provided by the caller, the SDK uses an empty JSON object `{}`. + +The Rust SDK: + +```rust +pub fn parse(input: &str) -> SlashParserResult +pub fn parse_with_context( + input: &str, + context: serde_json::Value +) -> SlashParserResult +``` + +The first form uses `{}` as context. The second passes through the provided value. + +## 6. Rust SDK + +### 6.1. Public API + +The Rust SDK crate exposes: + +Functions: + +- `parse(input: &str) -> SlashParserResult` +- `parse_with_context(input: &str, context: serde_json::Value) -> SlashParserResult` + +Types (all with `Serialize` and `Deserialize`): + +- `SlashParserResult` +- `Command` +- `CommandArguments` +- `ArgumentMode` +- `TextBlock` +- `LineRange` +- `Warning` + +These are SDK wrapper types, not the engine's domain types. They mirror the engine types but add serde derives and any field renames needed for JSON conformance. + +### 6.2. SlashParserResult + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SlashParserResult { + pub version: String, + pub context: serde_json::Value, + pub commands: Vec, + pub text_blocks: Vec, + pub warnings: Vec, +} +``` + +The SDK constructs this from the engine's `ParseResult` plus the caller-supplied context. + +### 6.3. Serde Derives + +All SDK types derive both `Serialize` and `Deserialize`. This enables: + +- Serialization to JSON for output. +- Deserialization from JSON for testing, roundtrip verification, and consumer convenience. + +The SDK depends on: + +- `serde = { version = "1", features = ["derive"] }` +- `serde_json = "1"` + +### 6.4. Feature Flags + +The SDK crate MAY expose feature flags to control optional functionality: + +| Feature | Purpose | +|---|---| +| `"default"` | Rust SDK only (serde + serde_json) | +| `"schema"` | Includes JSON schema validation utilities | + +The WASM and WASI targets are separate crates and do not need feature flags in the SDK crate. + +## 7. WASM Target + +### 7.1. Build Target + +The WASM crate compiles to `wasm32-unknown-unknown` using wasm-bindgen for JavaScript interop. + +Build command: + +```sh +wasm-pack build --target web parser-wasm +``` + +Or for Node.js: + +```sh +wasm-pack build --target nodejs parser-wasm +``` + +### 7.2. Exported Functions + +The WASM module exports two functions: + +```rust +#[wasm_bindgen] +pub fn parse(input: &str) -> JsValue + +#[wasm_bindgen] +pub fn parse_with_context( + input: &str, + context: JsValue +) -> JsValue +``` + +Both return a `JsValue` containing the `SlashParserResult` as a JavaScript object. + +### 7.3. Input and Output + +- Input: a JavaScript string (UTF-16, converted to UTF-8 by wasm-bindgen). +- Output: a JavaScript object matching the JSON schema. The SDK serializes to `serde_json::Value` and converts to `JsValue` via serde-wasm-bindgen or `JsValue::from_serde`. +- Context: a JavaScript object. If `undefined` or `null` is passed, the SDK uses an empty object. + +### 7.4. Error Handling + +The WASM functions MUST NOT throw. The engine is a total function; the SDK wraps it in an envelope. There is no input that produces an error. + +If context conversion fails (e.g., a non-object is passed), the SDK substitutes an empty object and proceeds normally. + +### 7.5. TypeScript Declarations + +The WASM build MUST produce TypeScript declaration files (`.d.ts`) that describe the exported functions and the shape of the return value. + +Minimal declaration: + +```typescript +export interface SlashParserResult { + version: string; + context: Record; + commands: Command[]; + text_blocks: TextBlock[]; + warnings: Warning[]; +} + +export interface Command { + id: string; + name: string; + raw: string; + range: LineRange; + arguments: CommandArguments; +} + +export interface CommandArguments { + header: string; + mode: "single-line" | "fence"; + fence_lang: string | null; + payload: string; +} + +export interface TextBlock { + id: string; + range: LineRange; + content: string; +} + +export interface LineRange { + start_line: number; + end_line: number; +} + +export interface Warning { + type: string; + start_line?: number; + message?: string; +} + +export function parse(input: string): SlashParserResult; +export function parse_with_context( + input: string, + context: Record +): SlashParserResult; +``` + +The declaration file MAY be generated by wasm-bindgen or hand-maintained. Either way, it MUST match the actual output. + +### 7.6. npm Package + +The WASM crate SHOULD be publishable as an npm package. The package name, version, and metadata are configured in `parser-wasm/Cargo.toml` via wasm-pack conventions. + +The package includes: + +- The compiled `.wasm` binary. +- JavaScript glue code (generated by wasm-pack). +- TypeScript declarations (Section 7.5). +- A README with usage examples. + +## 8. WASI Target + +### 8.1. Execution Model + +The WASI crate compiles to `wasm32-wasip1` (or `wasm32-wasip2` when stable) and runs in any WASI-compatible runtime (Wasmtime, Wasmer, WasmEdge). + +The binary is a filter: it reads input from stdin, processes it, and writes output to stdout. + +### 8.2. I/O Protocol + +Single-document mode (default): + +1. Read all of stdin as UTF-8. +2. Parse with `parse_document`. +3. Write one `SlashParserResult` JSON object to stdout. +4. Terminate. + +Context may be supplied via a command-line argument or environment variable (implementation-defined). + +JSONL mode (when `--jsonl` flag is present or when stdin contains multiple documents separated by a delimiter): see Section 9. + +### 8.3. JSONL Streaming + +In JSONL mode, the WASI binary reads multiple documents from stdin. Document boundaries are implementation-defined but MUST be unambiguous. Two approaches are acceptable: + +1. One file path per line on stdin: the binary reads each file and emits one JSONL line per file. +2. A delimiter-separated stream: documents are separated by a specific marker (e.g., a NUL byte or a line containing only `---`). + +Each document produces one JSONL line on stdout per Section 9. + +### 8.4. Exit Codes + +| Code | Meaning | +|---|---| +| 0 | Success (even if warnings were emitted) | +| 1 | I/O error (stdin read failure, stdout write) | +| 2 | Invalid command-line arguments | + +The parser never fails. Exit code 0 is expected for all valid I/O operations, regardless of parse warnings. + +## 9. JSONL Streaming + +### 9.1. Line Format + +Each line of JSONL output is exactly one `SlashParserResult` envelope, serialized as a single JSON object on one line (no embedded newlines within the JSON). + +JSONL mode does NOT emit one command per line. Commands remain elements of the `commands` array inside each envelope. + +### 9.2. Envelope Scoping + +Each JSONL line is a self-contained parse result: + +- Command IDs (`cmd-0`, `cmd-1`) are scoped to their envelope and reset for each input document. +- Text block IDs (`text-0`, `text-1`) are scoped to their envelope and reset for each input document. +- Warnings are scoped to their envelope. + +Consumers can treat each line as an independent unit of work. + +### 9.3. Pipeline Semantics + +In a pipeline processing multiple input documents, each input produces one JSONL line, in the order the inputs were supplied. + +The parser itself is not aware of the pipeline. The JSONL framing is the responsibility of the WASI binary or the Rust SDK consumer. The engine processes one document per call to `parse_document`. + +## 10. Version Mapping + +Three version numbers are in play: + +| Version | Source | Location | +|---|---|---| +| Syntax RFC version | [SYNTAX-RFC] | `version` field in JSON envelope | +| Engine spec version | [ENGINE-SPEC] | `SPEC_VERSION` const | +| SDK spec version | This document | `Cargo.toml` | + +The `version` field in the JSON envelope MUST contain the syntax RFC version (`"1.1.0"`), not the engine or SDK version. This is because consumers validate output against the syntax RFC's JSON schema, and the version field is how they determine which schema to use. + +The SDK is responsible for mapping `SPEC_VERSION` from the engine to the correct syntax RFC version in the serialized envelope. This mapping is maintained in the SDK crate. + +If the engine's `SPEC_VERSION` changes without a corresponding syntax RFC change, the envelope version stays the same. If the syntax RFC version changes, the SDK MUST be updated to emit the new version. + +## 11. Conformance + +### 11.1. JSON Schema Validation + +The serialized output of all three targets (Rust SDK, WASM, WASI) MUST validate against the JSON schema defined for [SYNTAX-RFC]. + +The SDK test suite SHOULD include automated schema validation for a representative set of inputs. + +### 11.2. Determinism + +Per [SYNTAX-RFC] Section 8.4, given identical input and context, the serialized JSON output MUST be byte-for-byte identical across invocations. + +This requires deterministic JSON serialization. serde_json produces deterministic output for a given Rust struct layout. The SDK MUST NOT introduce non-determinism through HashMap iteration order in the context pass-through. If context is accepted as `serde_json::Value`, the `Value::Object` variant uses serde_json's `Map` (which preserves insertion order), which is deterministic. + +### 11.3. Roundtrip Fidelity + +Per [SYNTAX-RFC] Section 8.3, the SDK supports roundtrip testing. The SDK MAY provide a formatter utility that reconstructs plaintext from a `SlashParserResult`. If provided, the roundtrip invariant `P(F(P(I))) = P(I)` MUST hold. + +## 12. Security Considerations + +The SDK inherits the engine's security properties (no I/O, no command execution, pure function). Additional considerations for the SDK layer: + +- WASM: the module runs in a sandboxed environment. It has no access to the filesystem, network, or host memory beyond its linear memory. Output size is bounded by input size. +- WASI: the binary has access to stdin, stdout, and (if file paths are used) the pre-opened filesystem directories granted by the runtime. Implementations SHOULD NOT request more capabilities than needed. +- JSON output: the SDK serializes user-supplied content (command arguments, text blocks) into JSON strings. serde_json handles escaping. The SDK MUST NOT perform any additional escaping or sanitization that would alter the content. + +## 13. Future Work + +The following items are explicitly out of scope for this version but may be addressed in future revisions: + +- Per-command JSONL streaming (one command per line instead of one envelope per line). +- Binary serialization formats (MessagePack, CBOR) as alternatives to JSON. +- A C FFI for integration with C/C++/Python/Ruby via shared library. +- A formatter utility for roundtrip reconstruction. +- Language-specific SDK packages beyond JavaScript/TypeScript (Python, Go, etc.). + +## 14. Migration Notes from v0.1.0 + +### 14.1. Syntax RFC Reference + +- v0.1.0: Referenced Slash Command Parser Syntax Specification v0.3.1. +- v0.2.0: References Slash Command Syntax v1.1.0 [SYNTAX-RFC]. + +Impact: Update the version string emitted in the JSON envelope `version` field from `"0.3.1"` to `"1.1.0"`. + +### 14.2. Engine Spec Reference + +- v0.1.0: Referenced Engine Specification v0.4.0. +- v0.2.0: References Engine Specification v0.5.0 [ENGINE-SPEC]. + +Impact: The engine now rejects command names ending with a hyphen (per [SYNTAX-RFC] Section 4.1). Inputs like `/cmd-` are classified as text, not commands. SDK tests that relied on trailing-hyphen command names must be updated. + +### 14.3. Document Format + +- v0.1.0: RFC-style plaintext (.txt). +- v0.2.0: Markdown (.md), consistent with the engine spec. + +This is an internal project document, not a published RFC. + +## Normative References + +- **[SYNTAX-RFC]** Davidson, T. D., "Slash Command Syntax, Version 1.1.0", March 2026. +- **[ENGINE-SPEC]** Davidson, T. D., "Slash Command Parser Engine Specification, Version 0.5.0", March 2026. + +## Author + +Tom D. Davidson +Email: tom@tomdavidson.org +URI: https://tomdavidson.org +Utah diff --git a/docs/slash-parser-spec-v0.3.0.md b/docs/slash-parser-spec-v0.3.0.md deleted file mode 100644 index 413c3be..0000000 --- a/docs/slash-parser-spec-v0.3.0.md +++ /dev/null @@ -1,1096 +0,0 @@ -# Slash Command Parser Specification - -Version: 0.3.0 Date: 2026-03-18 - -## 1. Overview - -The Slash Command Parser consumes a single UTF-8 text input and produces a structured JSON result -containing all detected slash commands, interleaved text blocks, optional caller-supplied context, -and any parser warnings. - -The parser is deterministic and pure: given the same input and context, it always produces the same -output. It performs no I/O, maintains no global mutable state, and accepts all configuration through -its input parameters. - -### 1.1 Design Principles - -1. **Command-agnostic.** The parser never interprets argument content. It determines argument - boundaries and transport mode but treats all argument strings as opaque byte sequences. - Tokenization, quoting, key-value parsing, and any other semantic interpretation of arguments are - the sole responsibility of the command implementation. - -2. **Flat output, no AST.** The parser emits a flat list of commands and text blocks in document - order. There is no intermediate abstract syntax tree. - -3. **Incremental emission.** A conforming implementation must be capable of finalizing and emitting - each command or text block as soon as its last line has been consumed, without buffering the - entire document. Memory usage should be bounded by the size of the largest single command payload - or text block, not the total input size. - -4. **Single forward pass.** The parser processes the input in a single left-to-right pass (after - normalization and line joining). It never backtracks. - -## 2. Input Model - -### 2.1 Line Ending Normalization - -Before any other processing: - -1. Replace all `\r\n` (CRLF) sequences with `\n` (LF). -2. Replace all remaining `\r` (bare CR) characters with `\n`. - -After normalization, all line terminators are LF. The parser splits the normalized input on `\n` to -produce a sequence of physical lines. Each `\n` terminates a line and is not part of the line -content. A trailing `\n` at the end of input produces a trailing empty line. - -Literal escape sequences inside content (e.g., `\n` in a JSON string `"blah\nblah"`) are ordinary -characters, not line terminators. They are preserved verbatim in the output. - -### 2.2 Line Joining (Backslash Continuation) - -After line-ending normalization and before command parsing, the parser performs a line-joining -pre-pass. This mechanism is identical in behavior to POSIX shell backslash-newline removal. - -For each physical line, if the line ends with a backslash (`\`) character: - -1. Remove the trailing backslash. -2. Remove the line terminator (the split boundary). -3. Concatenate the remainder with the next physical line, separated by a single space. -4. Repeat: if the joined result still ends with `\`, continue joining with subsequent lines. - -Lines that do not end with `\` are left unchanged. - -The join marker is any backslash character immediately before the physical line terminator, -regardless of what precedes it. There is no requirement for a space or any other character before -the backslash. This includes lines that serve other syntactic roles, such as a closing fence line -followed by a trailing backslash (e.g., `` ``` \ ``). In all cases, a trailing `\` triggers line -joining. This matches POSIX shell behavior. - -After this pass, the parser operates on a sequence of logical lines. Each logical line maps back to -one or more physical lines from the original input. - -#### 2.2.1 Physical Line Tracking - -The parser must track the mapping from each logical line back to its original physical line range. -This mapping is used to populate `range.start_line` and `range.end_line` in the output, which always -refer to zero-based physical line numbers from the normalized input (before joining). - -#### 2.2.2 Trailing Backslash at EOF - -If the final physical line ends with a backslash and there is no subsequent line to join with, the -trailing backslash is removed and the line stands alone. This mirrors POSIX shell behavior where a -backslash-newline pair removes the newline; at EOF, the backslash simply disappears. - -Example: - -```text -/echo hello \ -``` - -Logical line: `/echo hello` - -The command parses as a single-line command with `arguments.payload` of `"hello "`. - -#### 2.2.3 Joining Examples - -Physical input (three physical lines): - -```text -/mcp call_tool read_file \ - --path src/index.ts \ - --format json -``` - -Logical line after joining: - -```text -/mcp call_tool read_file --path src/index.ts --format json -``` - -This logical line maps to physical lines 0 through 2. - -A line ending with a literal backslash that is not intended as a join marker should use a fenced -block instead (see Section 5.2). - -### 2.3 Fence Immunity - -Line joining does not apply inside fenced blocks. Once the parser enters fence mode (see Section -5.2), all physical lines are consumed verbatim until the closing fence or EOF. A trailing `\` inside -a fence is literal content, not a join marker. - -Because line joining is conceptually a pre-pass and fences are detected during command parsing, the -implementation must either (a) perform joining lazily during parsing, skipping lines inside fences, -or (b) use a two-pass approach where fence boundaries are identified first. Either strategy is -acceptable as long as fence content is never subject to line joining. - -## 3. Command Detection - -A command line is any logical line (after normalization and joining) whose first non-whitespace -character is `/` (U+002F) and whose subsequent characters form a valid command name. - -### 3.1 Command Name - -The command name: - -- Starts immediately after the leading `/` with no intervening space. -- Ends at the first whitespace character or end-of-line. -- Must match the pattern `[a-z][a-z0-9-]*`: - - Begins with a lowercase ASCII letter (a-z). - - Followed by zero or more lowercase ASCII letters, ASCII digits, or hyphens. - -### 3.2 Invalid Slash Lines - -If a logical line's first non-whitespace character is `/` but the text after `/` does not match the -command name pattern (for example, a bare `/`, `/123`, `/Hello`, or `/ space`), the line is not a -command. In `idle` state, such lines are treated as ordinary text and may become part of a text -block (Section 6). In `in_fence` state, all lines (including invalid slash lines) are literal -payload content. - -### 3.3 Arguments - -Everything after the first whitespace following the command name is the arguments portion. The -whitespace between the command name and the arguments is consumed as a separator and is not included -in the arguments string. - -The arguments portion may be: - -- Empty (command with no arguments). -- Inline text (single-line mode, Section 5.1). -- A fence opener (fence mode, Section 5.2). -- Inline text followed by a fence opener (the text before the fence becomes the `header`, the fenced - content becomes the `payload`). - -### 3.4 The `header` Field - -The `header` field contains the inline, non-fenced argument portion of the command line. It serves -as the dispatch or routing segment of the command and is present in both argument modes: - -- In single-line mode, `header` and `payload` contain the same string (the full arguments text). -- In fence mode, `header` contains the arguments text that appears before the fence opener on the - command line. This is typically used for subcommand names and flags, while the fenced `payload` - carries bulk data such as JSON or code. - -## 4. Parser States - -The parser uses two states: - -- `idle`: not inside a fenced block. The parser scans logical lines for commands and collects - non-command lines into text blocks. -- `in_fence`: collecting raw physical lines inside a fenced block for the current command. - -There is no "accumulating" or "continuation" state. Multi-physical-line commands that are not fenced -are handled entirely by line joining (Section 2.2) before the state machine runs. - -### 4.1 State Transitions - -| Current State | Condition | Action | Next State | -| ------------- | ---------------------------------------------------- | --------------------------------------------- | ---------- | -| `idle` | Logical line is a valid command with no fence opener | Finalize as single-line command | `idle` | -| `idle` | Logical line is a valid command with a fence opener | Begin fence; record header and fence metadata | `in_fence` | -| `idle` | Logical line is not a command | Append to current text block | `idle` | -| `in_fence` | Physical line is a closing fence | Finalize fenced command | `idle` | -| `in_fence` | Physical line is not a closing fence | Append to payload | `in_fence` | -| `in_fence` | EOF reached without closing fence | Finalize command with warning | `idle` | - -## 5. Argument Modes - -### 5.1 Single-Line Mode - -If a command's arguments portion (after joining) does not contain a fence opener: - -- The full arguments text is stored in both `header` and `payload`. -- `mode` is `"single-line"`. -- `fence_lang` is `null`. -- The command is finalized on that logical line. - -Example: - -```text -/deploy production --region us-west-2 -``` - -Result: - -- `name`: `"deploy"` -- `arguments.header`: `"production --region us-west-2"` -- `arguments.mode`: `"single-line"` -- `arguments.fence_lang`: `null` -- `arguments.payload`: `"production --region us-west-2"` - -### 5.2 Fence Mode - -Fenced blocks allow attaching a raw, multi-line payload to a command using markdown-style code fence -syntax. Fenced payloads are completely verbatim: no parsing rules (including line joining) apply to -content lines, eliminating escaping concerns. - -Only backtick (`` ` ``) fences are recognized. Tilde (`~`) fences are not supported. - -#### 5.2.1 Fence Opener - -In the arguments portion of a command line, the first occurrence of three or more consecutive -backtick characters is treated as a fence opener. - -- Text before the backtick run (trimmed of trailing whitespace) becomes `arguments.header`. -- The backtick run length is recorded as the fence marker length. -- Text after the backtick run (trimmed of leading whitespace), if non-empty and consisting of a - single token (no internal whitespace), is the optional language identifier stored in - `arguments.fence_lang`. -- The parser transitions to `in_fence`. - -The variable-length backtick fence (three or more) means content containing triple backticks can be -fenced with four or more backticks, avoiding collision. - -#### 5.2.2 Fence Body - -While in `in_fence`, the parser reads physical lines from the normalized input (not joined logical -lines). All lines are appended to the payload verbatim, preserving their original content including -any trailing backslashes. - -Lines are joined in the payload with `\n` separators. The payload does not include a trailing `\n` -after the last content line. - -#### 5.2.3 Fence Lifetime - -A fenced argument block extends from the fence opener line through either: - -- The first closing fence line, or -- End of input (EOF), in which case the fence is considered unclosed and a warning is emitted (see - Section 5.2.5). - -There is no other mechanism that terminates a fence. Inside a fence, command triggers, invalid slash -lines, blank lines, and all other content are literal payload. - -#### 5.2.4 Fence Closer - -A physical line is a closing fence if, after trimming leading and trailing whitespace: - -- It consists solely of backtick characters. -- The number of backticks is greater than or equal to the opener's backtick count. - -The closing fence line is not included in the payload. Once found, the parser finalizes the command -and returns to `idle`. - -Note that line joining still applies to lines outside fences. If a closing fence line ends with a -trailing backslash (e.g., `` ``` \ ``), the fence closes normally, and then the backslash triggers a -line join between the closing fence and the next physical line. The joined result is outside the -fenced command and is parsed normally (see Appendix A.8 for an example). - -#### 5.2.5 Unclosed Fence - -If the input ends before a closing fence is found: - -- The parser finalizes the command with whatever payload has been accumulated through EOF. -- A warning object with `"type": "unclosed_fence"` and `"start_line"` set to the fence opener's - physical line number is added to the `warnings` array in the result envelope. - -The parser emits the command with its partial payload. Consumers that require strictly well-formed -fenced payloads SHOULD treat `unclosed_fence` warnings as hard errors and reject the affected -command. - -#### 5.2.6 Backslash Joining Around Fences - -Backslash line joining applies to physical lines outside of fences. This means joining can merge -lines before the fence opener and lines after the fence closer, but never lines inside the fence -body (see Section 2.3). - -Joining into a fence opener. When backslash joining merges a command line with a line containing a -fence opener, the fence is detected in the resulting logical line. This is the natural way to place -a fence opener on a separate physical line from the command name. - -Example (four physical lines): - -/mcp call_tool write_file \ - -```json -{ "path": "foo" } -``` - -After joining, physical lines 0 and 1 become one logical line: - -/mcp call_tool write_file ```json - -The backtick run is detected as a fence opener. arguments.header is "call_tool write_file", -arguments.fence_lang is "json", and the JSON body is collected as the fenced payload. - -Joining after a fence closer. Once the closing fence is found and the parser returns to idle, any -subsequent physical lines are again subject to normal line joining. If lines following the closing -fence end with trailing backslashes, they join with each other as usual. - -Example (six physical lines): - -/mcp call_tool write_file -c\ - -```json -{ "path": "foo" } -``` - -\ -production - - Lines 0 and 1 join into: /mcp call_tool write_file -c ```json - - Lines 2 and 3 are inside and closing the fence (no joining applies). - - Lines 4 and 5 join into: production (the backslash on line 4 is removed and line 5 is - appended). - -The fenced command has header of "call_tool write_file -c", fence_lang of "json", and payload of "{ -\"path\": \"foo\" }". The joined production line is outside the command and becomes a text block. -See Appendix A.9 for the full output. - -## 6. Text Blocks - -Non-command logical lines encountered in `idle` state are collected into text blocks. - -- Consecutive non-command lines form a single text block. -- Blank lines that are part of a text region are included in the text block content. -- Text block content preserves the original lines joined with `\n` separators. -- Text blocks use physical line numbers for their `range`. -- A new text block begins after a command is finalized, if non-command lines follow. - -Text blocks exist so that: - -- The roundtrip invariant (Section 9.4) can be satisfied: a formatter needs to know where - non-command regions are to reconstruct the input. -- Tooling can distinguish "what will run" from "what is commentary" when rendering or analyzing - documents that mix commands with prose. - -Consumers that do not need text blocks may ignore the `text_blocks` array. - -## 7. Multiple Commands and Ordering - -The parser walks the input line by line: - -1. In `idle`, when it encounters a command line, it starts a new command according to the argument - mode rules (Section 5). -2. After a command is finalized, the parser returns to `idle` and continues scanning. -3. Non-command lines in `idle` are collected into text blocks (Section 6). - -Commands are assigned sequential zero-based IDs in encounter order: `cmd-0`, `cmd-1`, `cmd-2`, and -so on. Text blocks are assigned IDs independently in the same manner: `text-0`, `text-1`, `text-2`, -and so on. - -Commands and text blocks appear in the output arrays in document order. - -## 8. Output Format - -### 8.1 Envelope - -A parser run produces a single JSON object (the "envelope") conforming to the `SlashParserResult` -schema defined in Section 8.5. The envelope contains: - -- `version`: the spec version (`"0.3.0"`). -- `context`: caller-supplied metadata passed through to the output unchanged. -- `commands`: an ordered array of all detected commands. -- `text_blocks`: an ordered array of all non-command text regions. -- `warnings`: an array of parser warnings (e.g., unclosed fences). Empty if no warnings. - -The parser is a total function: it always produces a valid envelope for any input. There is no input -that causes the parser to fail or return an error instead of an envelope. An empty input produces an -envelope with empty `commands`, `text_blocks`, and `warnings` arrays. Malformed constructs (such as -unclosed fences) produce commands with partial data and corresponding entries in the `warnings` -array, never a parse failure. - -### 8.2 The `raw` Field - -The `raw` field on each command contains the exact source text for that command as it appeared in -the normalized input (after line-ending normalization but before line joining). For single-line -commands that span multiple physical lines via backslash joining, `raw` contains all the physical -lines including the backslash characters and the `\n` separators between them. For fenced commands, -`raw` includes the command line, all fence body lines, and the closing fence line (if present). - -### 8.3 JSONL Streaming Mode - -When a parser implementation supports JSON Lines (JSONL) output, each line of the JSONL stream is -exactly one complete `SlashParserResult` envelope, encoded as a single JSON object on one line. - -JSONL mode does not emit one command per line. Commands remain elements of the `commands` array -inside each envelope. In a pipeline processing multiple input files, each input produces one JSONL -line. - -This design was chosen because: - -- Each JSONL line is a self-contained parse result with commands, text blocks, warnings, and - context, making each line independently meaningful. -- Command IDs (`cmd-0`, `cmd-1`) are scoped per envelope, so per-envelope lines maintain stable ID - semantics. -- Consumers can treat each line as a complete unit of work corresponding to one input document. - -Per-command JSONL (one command per line) is out of scope for this specification. Implementations may -offer it as a non-standard extension. - -### 8.4 Envelope Example - -```json -{ - "version": "0.3.0", - "context": { "source": "deploy.md", "user": "tom" }, - "commands": [ - { - "id": "cmd-0", - "name": "deploy", - "raw": "/deploy production \\\n --region us-west-2 \\\n --canary", - "range": { "start_line": 0, "end_line": 2 }, - "arguments": { - "header": "production --region us-west-2 --canary", - "mode": "single-line", - "fence_lang": null, - "payload": "production --region us-west-2 --canary" - } - } - ], - "text_blocks": [ - { "id": "text-0", "range": { "start_line": 3, "end_line": 3 }, "content": "Deploy initiated." } - ], - "warnings": [] -} -``` - -### 8.5 JSON Schema - -```json -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://slash-parser.dev/schema/v0.3.0/parse-result.json", - "title": "SlashParserResult", - "description": "Output envelope for the Slash Command Parser v0.3.0.", - "type": "object", - "required": ["version", "context", "commands", "text_blocks", "warnings"], - "additionalProperties": false, - "properties": { - "version": { - "type": "string", - "const": "0.3.0", - "description": "Specification version that produced this result." - }, - "context": { - "type": "object", - "description": "Caller-supplied metadata passed through to the output unchanged.", - "additionalProperties": true, - "properties": { - "source": { - "type": ["string", "null"], - "description": "Identifier for the input source (e.g., filename, URI)." - }, - "timestamp": { - "type": ["string", "null"], - "description": "ISO 8601 timestamp, if provided by the caller." - }, - "user": { - "type": ["string", "null"], - "description": "User identifier, if provided by the caller." - }, - "session_id": { - "type": ["string", "null"], - "description": "Session identifier, if provided by the caller." - }, - "extra": { - "type": ["object", "null"], - "description": "Arbitrary additional metadata.", - "additionalProperties": true - } - } - }, - "commands": { - "type": "array", - "description": "Ordered array of all detected commands, in document order.", - "items": { "$ref": "#/$defs/Command" } - }, - "text_blocks": { - "type": "array", - "description": "Ordered array of non-command text regions, in document order.", - "items": { "$ref": "#/$defs/TextBlock" } - }, - "warnings": { - "type": "array", - "description": "Parser warnings (e.g., unclosed fences). Empty array if no warnings.", - "items": { "$ref": "#/$defs/Warning" } - } - }, - "$defs": { - "LineRange": { - "type": "object", - "description": "Zero-based physical line range (inclusive on both ends).", - "required": ["start_line", "end_line"], - "additionalProperties": false, - "properties": { - "start_line": { - "type": "integer", - "minimum": 0, - "description": "First physical line (zero-based) covered by this element." - }, - "end_line": { - "type": "integer", - "minimum": 0, - "description": "Last physical line (zero-based) covered by this element." - } - } - }, - "CommandArguments": { - "type": "object", - "description": "Parsed argument structure for a command. Content is opaque to the parser.", - "required": ["header", "mode", "fence_lang", "payload"], - "additionalProperties": false, - "properties": { - "header": { - "type": "string", - "description": "Inline argument text from the command line, before any fence opener. Serves as the dispatch/routing portion in both single-line and fence modes." - }, - "mode": { - "type": "string", - "enum": ["single-line", "fence"], - "description": "How the arguments were assembled." - }, - "fence_lang": { - "type": ["string", "null"], - "description": "Language identifier from the fence opener, or null if not in fence mode or no language was specified." - }, - "payload": { - "type": "string", - "description": "Assembled argument content. In single-line mode, identical to header. In fence mode, the verbatim content between the fence delimiters." - } - } - }, - "Command": { - "type": "object", - "description": "A single parsed slash command.", - "required": ["id", "name", "raw", "range", "arguments"], - "additionalProperties": false, - "properties": { - "id": { - "type": "string", - "pattern": "^cmd-[0-9]+$", - "description": "Sequential zero-based identifier (cmd-0, cmd-1, ...)." - }, - "name": { - "type": "string", - "pattern": "^[a-z][a-z0-9-]*$", - "description": "Command name without the leading slash." - }, - "raw": { - "type": "string", - "description": "Exact source text from the normalized input (before line joining), including backslashes and physical newlines for joined commands, and fence delimiters for fenced commands." - }, - "range": { - "$ref": "#/$defs/LineRange", - "description": "Physical line range in the normalized input covered by this command." - }, - "arguments": { "$ref": "#/$defs/CommandArguments" } - } - }, - "TextBlock": { - "type": "object", - "description": "A contiguous region of non-command text.", - "required": ["id", "range", "content"], - "additionalProperties": false, - "properties": { - "id": { - "type": "string", - "pattern": "^text-[0-9]+$", - "description": "Sequential zero-based identifier (text-0, text-1, ...)." - }, - "range": { - "$ref": "#/$defs/LineRange", - "description": "Physical line range covered by this text block." - }, - "content": { - "type": "string", - "description": "Original text content with lines joined by newline separators." - } - } - }, - "Warning": { - "type": "object", - "description": "A parser warning indicating a non-fatal issue.", - "required": ["type"], - "additionalProperties": true, - "properties": { - "type": { - "type": "string", - "description": "Warning category identifier (e.g., 'unclosed_fence')." - }, - "start_line": { - "type": "integer", - "minimum": 0, - "description": "Physical line number where the issue was detected." - }, - "message": { - "type": "string", - "description": "Human-readable description of the warning." - } - } - } - } -} -``` - -### 8.6 Parser Engine Interface - -The parser engine exposes two entry points. Implementations may adapt the signatures to their host -language, but the semantics must be preserved. - -1. **Parse with default context.** Accepts a single UTF-8 string input. Returns a - `SlashParserResult` envelope with an empty `context` object. - -Logical signature: - - parse(input: string) → SlashParserResult - -2. **Parse with caller-supplied context.** Accepts a UTF-8 string input and a context object. The - context is passed through to the envelope unchanged. The parser does not interpret context - fields. - -Logical signature: - - parse(input: string, context: object) → SlashParserResult - -Both entry points are pure functions: no I/O, no global mutable state, no side effects. All -configuration is supplied through the input parameters. - -The return type is always `SlashParserResult` as defined in Section 8.5. Implementations must not -use exceptions, panics, or error return channels for parse outcomes. Warnings about malformed input -(such as unclosed fences) are reported inside the envelope's `warnings` array. - -## 9. Conformance - -### 9.1 No AST Requirement - -A conforming implementation must not expose or require an abstract syntax tree as a public artifact. -The only standardized observable output is the JSON envelope defined in Section 8. Implementations -may use any internal data structures, but the contract with consumers is the envelope schema alone. - -### 9.2 Incremental Emission - -A conforming implementation must support finalizing each command or text block as soon as its last -physical line has been consumed. Memory usage must be bounded by the size of the largest single -command payload or text block, not the total size of the input. - -### 9.3 Opaque Payload - -The parser is command-agnostic. It never interprets argument content, does not tokenize, and does -not apply quoting, escaping, or key-value semantics to the `header` or `payload` fields. All -argument strings are treated as opaque byte sequences. Any higher-level parsing (e.g., splitting on -spaces, parsing JSON or YAML) is the sole responsibility of the command implementation. - -### 9.4 Roundtrip Fidelity Invariant - -Let `P` be a conforming parser and `F` be a formatter that generates a canonical plaintext -representation from `P`'s JSON envelope output. The following invariant must hold: - -For any valid input `I`, parsing, formatting, and parsing again yields a structurally equivalent -result: - -``` -P(F(P(I))) ≡ P(I) -``` - -Structural equivalence means the same sequence of commands and text blocks with identical values for -`name`, `arguments.mode`, `arguments.header`, `arguments.payload`, `arguments.fence_lang`, and text -block `content`. - -The invariant is structural, not byte-for-byte: `raw` values, line numbers, and whitespace details -may differ between the two parse results due to normalization and line joining. These are allowed -lossy transformations. - -### 9.5 Determinism - -A conforming implementation must be deterministic: given identical input bytes and identical -context, the output envelope must be byte-for-byte identical across invocations and across -implementations. - -### 9.6 JSONL Envelope Semantics - -In JSONL streaming mode, each line of the output stream is exactly one `SlashParserResult` envelope -encoded as a single JSON object. One input unit (e.g., one file or one string passed to the parse -function) produces exactly one JSONL line. - -The `commands` and `text_blocks` arrays inside each envelope are ordered by document position. -Command IDs (`cmd-0`, `cmd-1`, ...) and text block IDs (`text-0`, `text-1`, ...) are scoped to their -envelope and reset for each input unit. - -Implementations that process multiple inputs in a pipeline emit one JSONL line per input, in the -order the inputs were supplied. The parser itself is not aware of the pipeline; the JSONL framing is -the responsibility of the host runtime or CLI layer. - -## Appendix A: Parsing Examples - -### A.1 Single-Line Command - -Input: - -```text -/echo hello world -``` - -Output (commands array element): - -```json -{ - "id": "cmd-0", - "name": "echo", - "raw": "/echo hello world", - "range": { "start_line": 0, "end_line": 0 }, - "arguments": { - "header": "hello world", - "mode": "single-line", - "fence_lang": null, - "payload": "hello world" - } -} -``` - -### A.2 Joined Multi-Line Command - -Input (three physical lines): - -```text -/deploy production \ - --region us-west-2 \ - --canary -``` - -After joining: `/deploy production --region us-west-2 --canary` - -Output: - -```json -{ - "id": "cmd-0", - "name": "deploy", - "raw": "/deploy production \\\n --region us-west-2 \\\n --canary", - "range": { "start_line": 0, "end_line": 2 }, - "arguments": { - "header": "production --region us-west-2 --canary", - "mode": "single-line", - "fence_lang": null, - "payload": "production --region us-west-2 --canary" - } -} -``` - -### A.3 Fenced Command with Header - -Input: - -````text -/mcp call_tool write_file ```json -{ "path": "/src/index.ts" } -``` -```` - -Output: - -````json -{ - "id": "cmd-0", - "name": "mcp", - "raw": "/mcp call_tool write_file ```json\n{ \"path\": \"/src/index.ts\" }\n```", - "range": { "start_line": 0, "end_line": 2 }, - "arguments": { - "header": "call_tool write_file", - "mode": "fence", - "fence_lang": "json", - "payload": "{ \"path\": \"/src/index.ts\" }" - } -} -```` - -### A.4 Backslash Join into Fence - -Input (four physical lines): - -````text -/mcp call_tool write_file \ -```json -{ "path": "foo" } -``` -```` - -After joining, physical lines 0 and 1 become: ``/mcp call_tool write_file ```json`` - -Output: - -````json -{ - "id": "cmd-0", - "name": "mcp", - "raw": "/mcp call_tool write_file \\\n```json\n{ \"path\": \"foo\" }\n```", - "range": { "start_line": 0, "end_line": 3 }, - "arguments": { - "header": "call_tool write_file", - "mode": "fence", - "fence_lang": "json", - "payload": "{ \"path\": \"foo\" }" - } -} -```` - -### A.5 Text Blocks and Multiple Commands - -Input: - -```text -Welcome to the deployment system. - -/deploy staging -/notify team --channel ops -Deployment complete. -``` - -Output: - -```json -{ - "version": "0.3.0", - "context": {}, - "commands": [ - { - "id": "cmd-0", - "name": "deploy", - "raw": "/deploy staging", - "range": { "start_line": 2, "end_line": 2 }, - "arguments": { - "header": "staging", - "mode": "single-line", - "fence_lang": null, - "payload": "staging" - } - }, - { - "id": "cmd-1", - "name": "notify", - "raw": "/notify team --channel ops", - "range": { "start_line": 3, "end_line": 3 }, - "arguments": { - "header": "team --channel ops", - "mode": "single-line", - "fence_lang": null, - "payload": "team --channel ops" - } - } - ], - "text_blocks": [ - { - "id": "text-0", - "range": { "start_line": 0, "end_line": 1 }, - "content": "Welcome to the deployment system.\n" - }, - { - "id": "text-1", - "range": { "start_line": 4, "end_line": 4 }, - "content": "Deployment complete." - } - ], - "warnings": [] -} -``` - -### A.6 Invalid Slash Lines - -Input: - -```text -/123 not a command -/ bare slash -/Hello capitalized -/deploy staging -``` - -Output: the first three lines form `text-0`; `/deploy staging` is `cmd-0`. - -### A.7 Unclosed Fence - -Input: - -````text -/mcp call_tool ```json -{ "incomplete": true } -```` - -Output: the command is finalized with the accumulated payload, and a warning is emitted: - -````json -{ - "version": "0.3.0", - "context": {}, - "commands": [ - { - "id": "cmd-0", - "name": "mcp", - "raw": "/mcp call_tool ```json\n{ \"incomplete\": true }", - "range": { "start_line": 0, "end_line": 1 }, - "arguments": { - "header": "call_tool", - "mode": "fence", - "fence_lang": "json", - "payload": "{ \"incomplete\": true }" - } - } - ], - "text_blocks": [], - "warnings": [ - { - "type": "unclosed_fence", - "start_line": 0, - "message": "Fenced block opened at line 0 was never closed." - } - ] -} -```` - -### A.8 Closing Fence with Trailing Backslash - -A closing fence line that ends with a trailing backslash closes the fence normally, then the -backslash triggers a line join with the next physical line. The joined content is outside the fenced -command. - -Input (six physical lines): - -````text -/mcp call_tool write_file -c \ -```json -{ "path": "foo" } -``` \ -\ -production -```` - -Physical lines: - -- 0: `/mcp call_tool write_file -c \` -- 1: `` ```json `` -- 2: `{ "path": "foo" }` -- 3: `` ``` \ `` -- 4: `\` -- 5: `production` - -Line joining (outside fences): - -- Lines 0 and 1 join: ``/mcp call_tool write_file -c ```json`` -- Lines 2 is inside the fence (verbatim, no joining). -- Line 3 is the closing fence (backticks only after trim? No: it contains `` ``` \ ``, which after - trimming whitespace is `` ```\ `` or `` ``` \ ``. The backslash means this is not solely - backticks, so it is not a valid closing fence.) - -Because line 3 is not a valid closing fence, the fence never closes. All remaining lines (2 through -5) become part of the fenced payload, and the parser emits an `unclosed_fence` warning. - -Output: - -````json -{ - "version": "0.3.0", - "context": {}, - "commands": [ - { - "id": "cmd-0", - "name": "mcp", - "raw": "/mcp call_tool write_file -c \\\n```json\n{ \"path\": \"foo\" }\n``` \\\n\\\nproduction", - "range": { "start_line": 0, "end_line": 5 }, - "arguments": { - "header": "call_tool write_file -c", - "mode": "fence", - "fence_lang": "json", - "payload": "{ \"path\": \"foo\" }\n``` \\\n\\\nproduction" - } - } - ], - "text_blocks": [], - "warnings": [ - { - "type": "unclosed_fence", - "start_line": 1, - "message": "Fenced block opened at line 1 was never closed." - } - ] -} -```` - -To avoid this, write the closing fence on its own line without a trailing backslash: - -````text -/mcp call_tool write_file -c \ -```json -{ "path": "foo" } -``` -```` - -### A.9 Proper Fence Close Followed by Additional Content - -Input (six physical lines): - -````text -/mcp call_tool write_file -c\ -```json -{ "path": "foo" } -``` -\ -production -```` - -Physical lines: - -- 0: `/mcp call_tool write_file -c\` -- 1: `` ```json `` -- 2: `{ "path": "foo" }` -- 3: `` ``` `` -- 4: `\` -- 5: `production` - -Line joining (outside fences): - -- Lines 0 and 1 join: ``/mcp call_tool write_file -c ```json`` -- Line 2 is inside the fence (verbatim). -- Line 3 is a valid closing fence (solely backticks after trim). Fence closes. Parser returns to - `idle`. -- Lines 4 and 5 join: `production` - -Output: - -````json -{ - "version": "0.3.0", - "context": {}, - "commands": [ - { - "id": "cmd-0", - "name": "mcp", - "raw": "/mcp call_tool write_file -c\\\n```json\n{ \"path\": \"foo\" }\n```", - "range": { "start_line": 0, "end_line": 3 }, - "arguments": { - "header": "call_tool write_file -c", - "mode": "fence", - "fence_lang": "json", - "payload": "{ \"path\": \"foo\" }" - } - } - ], - "text_blocks": [ - { "id": "text-0", "range": { "start_line": 4, "end_line": 5 }, "content": " production" } - ], - "warnings": [] -} -```` - -## Appendix B: Change Log - -Changes from version 0.2.0: - -- Replaced continuation markers (`" /"`) with POSIX-style backslash line joining, reducing the state - machine from three states to two. -- Added explicit treatment of invalid slash lines in idle state (Section 3.2). -- Defined fence lifetime as "until closing fence or EOF" (Section 5.2.3). -- Formalized trailing backslash at EOF behavior (Section 2.2.2). -- Added backslash-join-then-fence example (Section 5.2.6). -- Added closing-fence-with-backslash examples (Appendix A.8, A.9). -- Stated opaque payload principle as a conformance requirement (Section 9.3). -- Stated roundtrip fidelity invariant as a testable property (Section 9.4). -- Stated incremental emission as a conformance constraint (Section 9.2). -- Committed to no-AST as a conformance requirement (Section 9.1). -- Added determinism requirement (Section 9.5). -- Clarified JSONL streaming semantics with rationale (Section 8.3). -- Documented `header` as the dispatch portion available in both modes (Section 3.4). -- Introduced `warnings` array for non-fatal issues (Section 5.2.5). -- Removed unused `children` field from Command schema. -- Only backtick fences are supported; tilde fences are explicitly excluded (Section 5.2). -- Version bumped to 0.3.0. diff --git a/parser/Cargo.toml b/parser/Cargo.toml index 5c06d40..98f42fa 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "parser" +name = "slasher_engine" version.workspace = true edition.workspace = true license.workspace = true diff --git a/parser/direction.md b/parser/direction.md deleted file mode 100644 index 6f738bc..0000000 --- a/parser/direction.md +++ /dev/null @@ -1,122 +0,0 @@ -Your direction matches your own Rust + clean-arch guidelines and is idiomatic Rust, with a couple of -naming tweaks I’d suggest. - -## Function structure and colocated tests - -Breaking `to_plaintext` into helpers like this is exactly what you want: - -```rust -fn write_command(output: &mut String, cmd: &Command) { - match cmd.arguments.mode { - ArgumentMode::SingleLine => write_single_line(output, cmd), - ArgumentMode::Continuation => write_continuation(output, cmd), - ArgumentMode::Fence => write_fence(output, cmd), - } -} -``` - -Having `write_command`, `write_single_line`, `write_continuation`, and `write_fence` in the same -file, plus a `#[cfg(test)] mod tests` at the bottom with fast, happy-path unit tests, is perfectly -aligned with your Rust testing doc: fast tests live in `cfg(test) mod tests` in the same file; -heavier property tests and integration tests move to `tests/` and fuzz harnesses live under `fuzz/`. - -That gives you: - -- Small, focused functions per concern. -- Colocated “fast” tests with direct positive/negative assertions. -- Property-based and fuzz tests elsewhere, as you prefer. - -## Folder and layer structure - -For the `parser` crate’s internal layout, mapping your JS boundaries to Rust: - -- **domain/**: pure types and invariants: command/text-block types, argument modes, maybe - lexer/token types later. No serde, no IO. This matches your “domain.rs pure, no serde” rule. -- **application/**: parsing state machine and formatting / printing (JSON and plaintext), since they - orchestrate domain types and build DTOs. This matches your “use cases / orchestration / ports” - bucket. -- **infrastructure/**: WASM bindings, FFI shims, logging/tracing glue. That’s the adapter layer. -- **tests/**: slower integration-style tests that exercise whole flows or multiple modules together. -- **proptest-regressions/** and **fuzz/**: stay as-is, under the parser crate instead of repo root. - -In Rust terms, you might land on something like: - -```text -parser/ - src/ - domain/ - mod.rs // re-exports - types.rs // Command, TextBlock, ArgumentMode, etc. - errors.rs // ParseError, domain-level errors - application/ - mod.rs - parse.rs // parse_to_domain, state machine - format_json.rs // JSON serialization boundary - format_text.rs // to_plaintext + helpers - infrastructure/ - mod.rs - wasm_js.rs // wasm-bindgen adapter - wasm_wasi.rs // WIT adapter later - lib.rs // pub use domain::*, application::parse_to_domain, etc. - tests/ - roundtrip.rs // slower multi-file/integration-ish tests - proptest-regressions/ - fuzz/ -``` - -This lines up with the Rust patterns doc: organize by feature, then by layer (`domain.rs`, `app.rs`, -`infra.rs`), promote to multiple files once a module grows, and keep domain free of serde. - -## Naming recommendations - -You’re right that `serialize.rs` vs `to_plaintext.rs` is asymmetric. I’d suggest: - -- `format_json.rs` (or `output_json.rs`): owns DTOs + `to_json`. -- `format_text.rs` (or `output_text.rs`): owns `to_plaintext` and helpers. - -Both live under `application/` since they are output-format concerns, not infrastructure (they don’t -talk to IO, just shape data). That also distances them from your infra error-mapping and transport -adapters in other projects. - -Within `application::format_text`, functions like: - -```rust -pub fn to_plaintext(result: &SlashParseResult) -> String { … } - -fn write_command(output: &mut String, cmd: &Command) { … } - -fn write_single_line(output: &mut String, cmd: &Command) { … } - -fn write_continuation(output: &mut String, cmd: &Command) { … } - -fn write_fence(output: &mut String, cmd: &Command) { … } - -#[cfg(test)] -mod tests { … } -``` - -fit your “free functions for domain logic, not methods” pattern. - -## Clean-arch fit and “is this bad practice?” - -Given your own rules: - -- Domain imports nothing external; DTOs and serde live outside domain. -- Application orchestrates domain, defines ports, but doesn’t know about transport details. -- Infrastructure implements ports and touches IO. - -Your proposed layout is consistent with that and is not “bad Rust practice.” Rust doesn’t require -flat `src/lib.rs` with everything; feature-and-layer submodules are common in more complex crates, -and your approach is in line with your Rust patterns doc’s “organize by feature, then by layer, -promote to workspace crates when it gets big.” - -One minor tweak: I’d keep `parser/` as the crate name rather than `parser-core` so it matches -ADR-0006 and avoids “application/parser.rs” repetition. That’s purely naming and you already plan to -rename. - -## Clarifying question - -For the textual formatting layer: do you want `to_plaintext` to be strictly “round-trip faithful” to -spec semantics only, or byte-for-byte identical to original input (including all whitespace), where -that’s possible? Your testing doc leans hard into roundtrip correctness, but exact byte preservation -may constrain how we model headers vs payloads. diff --git a/parser/moon.yml b/parser/moon.yml index 93a9f0c..b98006f 100644 --- a/parser/moon.yml +++ b/parser/moon.yml @@ -2,7 +2,3 @@ $schema: "https://moonrepo.dev/schemas/project.json" id: "parser" language: "rust" tags: ["core"] - -workspace: - inheritedTasks: - include: [] diff --git a/parser/proptest-regressions/application/command_accumulate.txt b/parser/proptest-regressions/application/command_accumulate.txt deleted file mode 100644 index ccfb93a..0000000 --- a/parser/proptest-regressions/application/command_accumulate.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc 1657315ea6a9fa6475f78c9454ab64e78fab99d9eef8792ed95cd52b7ecd3bdc # shrinks to name = "a", body_lines = [" "] diff --git a/parser/proptest-regressions/application/tests/mod.txt b/parser/proptest-regressions/application/tests/mod.txt deleted file mode 100644 index 647bf83..0000000 --- a/parser/proptest-regressions/application/tests/mod.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc d1207456a51fa8d19d9c2072c9b707b0c1aac013c87cfd4ae9a65975cdf9227c # shrinks to body = "\r" -cc ca107577def8080df35b3c6887f98d087144422311d1bc59ca5f0c535b65b1c2 # shrinks to lines = ["a\r/a"] diff --git a/parser/src/application/command_accumulate.rs b/parser/src/application/command_accumulate.rs deleted file mode 100644 index bede54e..0000000 --- a/parser/src/application/command_accumulate.rs +++ /dev/null @@ -1,413 +0,0 @@ -use super::line_classify::CommandHeader; -use crate::domain::ArgumentMode; - -/// Result of offering a physical line to an in-progress command. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AcceptResult { - Consumed, - Completed, - Rejected, -} - -/// In-progress command being assembled from physical lines. -/// -/// v0.3.0: Continuation mode is no longer handled here. Multi-physical-line commands -/// that are not fenced are resolved entirely by line joining (§2.2) before this module runs. -#[derive(Debug, Clone)] -pub struct PendingCommand { - pub id: usize, - pub name: String, - pub header_text: String, - pub mode: ArgumentMode, - pub fence_lang: Option, - pub fence_backtick_count: usize, - pub start_line: usize, - pub end_line: usize, - pub payload_lines: Vec, - pub raw_lines: Vec, - pub is_open: bool, -} - -/// Begin accumulating a new command from its parsed header. -/// -/// §7: id is the caller-supplied sequential zero-based counter used to produce cmd-0, cmd-1, ... -/// §8.2: raw_lines is seeded with the header's raw text; physical lines are appended by accept_line. -pub fn start_command(header: CommandHeader, line_index: usize, id: usize) -> PendingCommand { - // §5.1: single-line commands are fully resolved on their logical line; is_open = false. - // §5.2.1: fence commands open immediately and stay open until a closer is found. - let (payload_lines, is_open) = match &header.mode { - ArgumentMode::SingleLine => { - let lines = if header.header_text.is_empty() { - vec![] - } else { - vec![header.header_text.clone()] - }; - (lines, false) - } - ArgumentMode::Fence => (vec![], true), - }; - - PendingCommand { - id, - name: header.name, - header_text: header.header_text, - mode: header.mode, - fence_lang: header.fence_lang, - fence_backtick_count: header.fence_backtick_count, - start_line: line_index, - end_line: line_index, - payload_lines, - raw_lines: vec![header.raw], - is_open, - } -} - -/// Offer one physical line to an open command. -/// -/// §5.2.2: fence body lines are appended to payload_lines verbatim. -/// §5.2.4: a physical line that satisfies is_fence_closer closes the fence. -/// §8.2: every offered line, including the closer, is appended to raw_lines. -pub fn accept_line( - cmd: PendingCommand, - line_index: usize, - line: &str, -) -> (PendingCommand, AcceptResult) { - if !cmd.is_open { - return (cmd, AcceptResult::Rejected); - } - - match cmd.mode { - ArgumentMode::Fence => accept_fence(cmd, line_index, line), - // §4: only idle and in-fence states exist in v0.3.0; all other modes are closed. - _ => (cmd, AcceptResult::Rejected), - } -} - -fn accept_fence( - mut cmd: PendingCommand, - line_index: usize, - line: &str, -) -> (PendingCommand, AcceptResult) { - // §8.2: raw accumulation happens unconditionally before the closer check. - cmd.raw_lines.push(line.to_string()); - cmd.end_line = line_index; - - if is_fence_closer(line, cmd.fence_backtick_count) { - // §5.2.4: the closing fence line is not included in the payload. - cmd.is_open = false; - (cmd, AcceptResult::Completed) - } else { - cmd.payload_lines.push(line.to_string()); - (cmd, AcceptResult::Consumed) - } -} - -/// §5.2.4: A physical line is a closing fence if, after trimming leading and trailing -/// whitespace, it consists solely of backtick characters and the count is >= the opener's count. -fn is_fence_closer(line: &str, opener_count: usize) -> bool { - let trimmed = line.trim(); - !trimmed.is_empty() && trimmed.chars().all(|c| c == '`') && trimmed.len() >= opener_count -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::application::line_classify::{LineKind, classify_line}; - - fn header_from(line: &str) -> CommandHeader { - match classify_line(line) { - LineKind::Command(h) => h, - _ => panic!("expected command header from: {line:?}"), - } - } - - // --- start_command --- - - #[test] - fn single_line_is_immediately_closed() { - // §5.1: a single-line command is finalized on its logical line; no further lines consumed. - let cmd = start_command(header_from("/help"), 0, 0); - assert!(!cmd.is_open); - } - - #[test] - fn fence_opens_with_is_open_true() { - // §5.2.3: fence lifetime begins at the opener and extends until a valid closer or EOF. - let cmd = start_command(header_from("/code ```rust"), 0, 0); - assert!(cmd.is_open); - } - - #[test] - fn id_is_stored_on_pending_command() { - // §7: commands are assigned sequential zero-based IDs in encounter order (cmd-0, cmd-1, ...). - let cmd = start_command(header_from("/deploy production"), 3, 7); - assert_eq!(cmd.id, 7); - } - - #[test] - fn raw_lines_initialized_with_opener() { - // §8.2: raw includes the command line itself; accumulation starts at start_command. - let cmd = start_command(header_from("/mcp call_tool ```json"), 0, 0); - assert_eq!(cmd.raw_lines, vec!["/mcp call_tool ```json"]); - } - - #[test] - fn single_line_payload_initialized_from_header_text() { - // §5.1: in single-line mode header and payload contain the same string. - let cmd = start_command(header_from("/deploy production --region us-west-2"), 0, 0); - assert_eq!(cmd.payload_lines, vec!["production --region us-west-2"]); - } - - #[test] - fn single_line_no_args_has_empty_payload() { - // §5.1 (implied): a command with no arguments produces an empty payload_lines. - let cmd = start_command(header_from("/ping"), 0, 0); - assert!(cmd.payload_lines.is_empty()); - } - - #[test] - fn fence_payload_lines_initially_empty() { - // §5.2.2: the fence body begins empty; content is appended as physical lines arrive. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - assert!(cmd.payload_lines.is_empty()); - } - - #[test] - fn fence_backtick_count_carried_from_header() { - // §5.2.1: the opener's backtick run length is recorded for use by the closer check (§5.2.4). - let cmd = start_command(header_from("/cmd ```rust"), 0, 0); - assert_eq!(cmd.fence_backtick_count, 3); - } - - // --- accept_line: fence body --- - - #[test] - fn fence_body_line_is_consumed() { - // §5.2.2: physical lines that are not a closer are appended to the payload verbatim. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, res) = accept_line(cmd, 1, "fn main() {}"); - assert_eq!(res, AcceptResult::Consumed); - assert!(cmd.is_open); - assert_eq!(cmd.payload_lines, vec!["fn main() {}"]); - } - - #[test] - fn fence_body_line_added_to_raw_lines() { - // §8.2: for fenced commands raw includes the command line, all body lines, and the closer. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "first"); - let (cmd, _) = accept_line(cmd, 2, "second"); - assert_eq!(cmd.raw_lines, vec!["/cmd ```", "first", "second"]); - } - - #[test] - fn fence_body_preserves_lines_verbatim() { - // §5.2.2: content is appended verbatim; no parsing rules including line joining apply. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let line = r" leading spaces and trailing backslash\"; - let (cmd, _) = accept_line(cmd, 1, line); - assert_eq!(cmd.payload_lines, vec![line]); - } - - #[test] - fn blank_line_inside_fence_is_payload() { - // §5.2.3: inside a fence, blank lines are literal payload content. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, res) = accept_line(cmd, 1, ""); - assert_eq!(res, AcceptResult::Consumed); - assert_eq!(cmd.payload_lines, vec![""]); - } - - #[test] - fn command_line_inside_fence_is_payload() { - // §5.2.3: inside a fence, command triggers are literal payload; no state change. - let cmd = start_command(header_from("/outer ```"), 0, 0); - let (cmd, res) = accept_line(cmd, 1, "/inner arg"); - assert_eq!(res, AcceptResult::Consumed); - assert!(cmd.is_open); - } - - // --- accept_line: fence closer --- - - #[test] - fn fence_closer_completes_command() { - // §5.2.4: a valid closer transitions the accumulator to closed and returns Completed. - let cmd = start_command(header_from("/cmd ```rust"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "fn main() {}"); - let (cmd, res) = accept_line(cmd, 2, "```"); - assert_eq!(res, AcceptResult::Completed); - assert!(!cmd.is_open); - } - - #[test] - fn fence_closer_added_to_raw_lines() { - // §8.2: raw includes the closing fence line if present. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "content"); - let (cmd, _) = accept_line(cmd, 2, "```"); - assert_eq!(cmd.raw_lines, vec!["/cmd ```", "content", "```"]); - } - - #[test] - fn fence_closer_not_in_payload() { - // §5.2.4: the closing fence line is not included in the payload. - let cmd = start_command(header_from("/cmd ```rust"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "fn main() {}"); - let (cmd, _) = accept_line(cmd, 2, "```"); - assert_eq!(cmd.payload_lines, vec!["fn main() {}"]); - } - - #[test] - fn fence_closer_with_more_backticks_than_opener() { - // §5.2.4: closer count >= opener count; four backticks close a triple-backtick fence. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, res) = accept_line(cmd, 1, "````"); - assert_eq!(res, AcceptResult::Completed); - assert!(!cmd.is_open); - } - - #[test] - fn fewer_backticks_than_opener_is_not_a_closer() { - // §5.2.4: closer count must be >= opener count; two backticks cannot close a triple fence. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, res) = accept_line(cmd, 1, "``"); - assert_eq!(res, AcceptResult::Consumed); - assert!(cmd.is_open); - } - - #[test] - fn line_with_backticks_and_trailing_text_is_not_a_closer() { - // §5.2.4: after trimming whitespace the line must consist solely of backtick characters. - // A line like "```rust" or "``` trailing" is body content, not a closer. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, res) = accept_line(cmd, 1, "```rust"); - assert_eq!(res, AcceptResult::Consumed); - assert!(cmd.is_open); - } - - #[test] - fn fence_closer_with_leading_whitespace_is_valid() { - // §5.2.4: closer is checked after trimming leading and trailing whitespace. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, res) = accept_line(cmd, 1, " ``` "); - assert_eq!(res, AcceptResult::Completed); - assert!(!cmd.is_open); - } - - #[test] - fn closed_fence_rejects_subsequent_lines() { - // §5.2.3: once the closing fence is found the command is finalized; no further lines accepted. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "content"); - let (cmd, _) = accept_line(cmd, 2, "```"); - let (cmd, res) = accept_line(cmd, 3, "after"); - assert_eq!(res, AcceptResult::Rejected); - assert_eq!(cmd.payload_lines, vec!["content"]); - } - - // --- accept_line: single-line --- - - #[test] - fn single_line_rejects_any_subsequent_line() { - // §5.1: single-line commands are finalized on their logical line; accept_line is a no-op. - let cmd = start_command(header_from("/help"), 0, 0); - let (cmd, res) = accept_line(cmd, 1, "anything"); - assert_eq!(res, AcceptResult::Rejected); - assert!(cmd.payload_lines == vec![""] || cmd.payload_lines.is_empty()); - } - - // --- range tracking --- - - #[test] - fn end_line_advances_with_each_accepted_line() { - // §2.2.1: range.endLine is the zero-based index of the last physical line consumed. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "line one"); - let (cmd, _) = accept_line(cmd, 2, "line two"); - let (cmd, _) = accept_line(cmd, 3, "```"); - assert_eq!(cmd.start_line, 0); - assert_eq!(cmd.end_line, 3); - } - - // --- Property tests --- - - use proptest::prelude::*; - - fn valid_command_name() -> impl Strategy { - "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) - } - - proptest! { - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn raw_lines_count_equals_lines_consumed( - // §8.2: raw accumulates opener + every body line + closer (if present). - name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 0..8) - ) { - let input = format!("/{name} ```"); - let cmd = start_command(header_from(&input), 0, 0); - let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { - let (next, _) = accept_line(cmd, i + 1, line); - next - }); - let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); - prop_assert_eq!(cmd.raw_lines.len(), body_lines.len() + 2); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn payload_never_contains_closer( - // §5.2.4: the closing fence line is never part of the payload. - // The check mirrors is_fence_closer: non-empty after trim, all backticks. - name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 0..8) - ) { - let input = format!("/{name} ```"); - let cmd = start_command(header_from(&input), 0, 0); - let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { - let (next, _) = accept_line(cmd, i + 1, line); - next - }); - let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); - let no_closer = !cmd.payload_lines.iter().any(|l| { - let t = l.trim(); - !t.is_empty() && t.chars().all(|c| c == '`') - }); - prop_assert!(no_closer); - } - - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn fence_closer_property_count_gte_opener( - // §5.2.4: any line of N or more backticks (N >= opener) closes the fence. - name in valid_command_name(), - extra in 0usize..5 - ) { - let input = format!("/{name} ```"); - let cmd = start_command(header_from(&input), 0, 0); - let closer = "`".repeat(3 + extra); - let (cmd, res) = accept_line(cmd, 1, &closer); - prop_assert_eq!(res, AcceptResult::Completed); - prop_assert!(!cmd.is_open); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn id_is_preserved_through_accumulation( - // §7: the id assigned at start_command is stable throughout accumulation. - name in valid_command_name(), - id in 0usize..1000, - body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 0..5) - ) { - let input = format!("/{name} ```"); - let cmd = start_command(header_from(&input), 0, id); - let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { - let (next, _) = accept_line(cmd, i + 1, line); - next - }); - prop_assert_eq!(cmd.id, id); - } - } -} diff --git a/parser/src/application/command_finalize.rs b/parser/src/application/command_finalize.rs deleted file mode 100644 index a7e9288..0000000 --- a/parser/src/application/command_finalize.rs +++ /dev/null @@ -1,392 +0,0 @@ -use super::command_accumulate::PendingCommand; -use crate::domain::{ArgumentMode, Command, CommandArguments, LineRange, Warning}; - -#[derive(Debug)] -pub struct FinalizedCommand { - pub command: Command, - pub warnings: Vec, -} - -/// Consume a PendingCommand and produce the finalized Command value plus any warnings. -/// -/// §7: id is formatted as cmd-{n} using the zero-based encounter index from start_command. -/// §8.2: raw is the physical lines joined with newline separators (opener + body + closer). -/// §5.2.5: an unclosed fence produces a Warning rather than a hard parse failure. -pub fn finalize_command(pending: PendingCommand) -> FinalizedCommand { - let mut warnings = Vec::new(); - - // §5.2.5: if the parser reached EOF without finding a closing fence, emit one warning - // and finalize with whatever payload was accumulated. - if pending.mode == ArgumentMode::Fence && pending.is_open { - warnings.push(Warning { - wtype: "unclosed-fence".to_string(), - start_line: Some(pending.start_line), - message: Some(format!( - "Fenced block opened at line {} was never closed.", - pending.start_line - )), - }); - } - - // §8.2: raw is built from all accumulated physical lines joined with newline separators. - // For single-line commands this is just the one header line. - // For fenced commands this includes the opener, all body lines, and the closer if present. - let raw = pending.raw_lines.join("\n"); - - // §5.1: single-line payload equals header_text (initialised at start_command). - // §5.2.2: fence payload is the verbatim body lines joined with newline separators. - let payload = pending.payload_lines.join("\n"); - - let command = Command { - // §7: sequential zero-based id formatted as cmd-0, cmd-1, ... - id: format!("cmd-{}", pending.id), - name: pending.name, - raw, - range: LineRange { - start_line: pending.start_line, - end_line: pending.end_line, - }, - arguments: CommandArguments { - header: pending.header_text, - mode: pending.mode, - fence_lang: pending.fence_lang, - payload, - }, - }; - - FinalizedCommand { command, warnings } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - application::{ - command_accumulate::{accept_line, start_command}, - line_classify::{CommandHeader, LineKind, classify_line}, - }, - domain::ArgumentMode, - }; - - fn header_from(line: &str) -> CommandHeader { - match classify_line(line) { - LineKind::Command(h) => h, - _ => panic!("expected command header from: {line:?}"), - } - } - - // --- id --- - - #[test] - fn id_is_formatted_as_cmd_zero() { - // §7: commands are assigned sequential zero-based IDs formatted as cmd-0, cmd-1, ... - let cmd = start_command(header_from("/deploy staging"), 0, 0); - let result = finalize_command(cmd); - assert_eq!(result.command.id, "cmd-0"); - } - - #[test] - fn id_reflects_encounter_index() { - // §7: the id counter is caller-supplied; cmd-5 identifies the sixth command encountered. - let cmd = start_command(header_from("/deploy staging"), 0, 5); - let result = finalize_command(cmd); - assert_eq!(result.command.id, "cmd-5"); - } - - // --- raw --- - - #[test] - fn single_line_raw_is_header_line_only() { - // §8.2: for a single-line command raw contains only the original command line. - let cmd = start_command(header_from("/deploy production --region us-west-2"), 0, 0); - let result = finalize_command(cmd); - assert_eq!(result.command.raw, "/deploy production --region us-west-2"); - } - - #[test] - fn fenced_raw_includes_opener_body_and_closer() { - // §8.2: for fenced commands raw includes the command line, all body lines, - // and the closing fence line, joined with newline separators. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "line one"); - let (cmd, _) = accept_line(cmd, 2, "line two"); - let (cmd, _) = accept_line(cmd, 3, "```"); - let result = finalize_command(cmd); - assert_eq!(result.command.raw, "/cmd ```\nline one\nline two\n```"); - } - - #[test] - fn unclosed_fence_raw_includes_opener_and_partial_body() { - // §8.2: if the fence is unclosed raw still contains all accumulated physical lines; - // there is simply no closer line to include. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "partial"); - let result = finalize_command(cmd); - assert_eq!(result.command.raw, "/cmd ```\npartial"); - } - - // --- warnings --- - - #[test] - fn unclosed_fence_emits_exactly_one_warning() { - // §5.2.5: an unclosed fence produces exactly one warning; the parser does not fail. - let cmd = start_command(header_from("/cmd ```rust"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "line1"); - let (cmd, _) = accept_line(cmd, 2, "line2"); - let result = finalize_command(cmd); - assert_eq!(result.warnings.len(), 1); - } - - #[test] - fn unclosed_fence_warning_wtype_is_unclosed_fence() { - // §5.2.5: the warning type string is "unclosed-fence". - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let result = finalize_command(cmd); - assert_eq!(result.warnings[0].wtype, "unclosed-fence"); - } - - #[test] - fn unclosed_fence_warning_start_line_is_opener_physical_line() { - // §5.2.5: start_line is set to the fence opener's physical line number. - let cmd = start_command(header_from("/cmd ```"), 4, 0); - let result = finalize_command(cmd); - assert_eq!(result.warnings[0].start_line, Some(4)); - } - - #[test] - fn unclosed_fence_warning_message_is_present() { - // §5.2.5 (implied): a human-readable message is present and non-empty. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let result = finalize_command(cmd); - assert!( - result.warnings[0] - .message - .as_deref() - .map(|m| !m.is_empty()) - .unwrap_or(false) - ); - } - - #[test] - fn unclosed_fence_warning_message_references_start_line() { - // §5.2.5 (implied): the message should identify the offending line number for diagnostics. - let cmd = start_command(header_from("/cmd ```"), 7, 0); - let result = finalize_command(cmd); - let msg = result.warnings[0].message.as_deref().unwrap_or(""); - assert!(msg.contains('7')); - } - - #[test] - fn closed_fence_has_no_warning() { - // §5.2.3: once the closing fence is found the command is well-formed; no warning emitted. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "content"); - let (cmd, _) = accept_line(cmd, 2, "```"); - let result = finalize_command(cmd); - assert!(result.warnings.is_empty()); - } - - #[test] - fn single_line_has_no_warning() { - // §5.1 (implied): a single-line command is always complete; no warning emitted. - let cmd = start_command(header_from("/help"), 0, 0); - let result = finalize_command(cmd); - assert!(result.warnings.is_empty()); - } - - // --- payload --- - - #[test] - fn single_line_payload_equals_header_text() { - // §5.1: in single-line mode header and payload contain the same string. - let cmd = start_command(header_from("/hello world"), 0, 0); - let result = finalize_command(cmd); - assert_eq!(result.command.arguments.payload, "world"); - assert_eq!(result.command.arguments.mode, ArgumentMode::SingleLine); - } - - #[test] - fn single_line_no_args_has_empty_payload() { - // §5.1 (implied): a command with no arguments has an empty header and empty payload. - let cmd = start_command(header_from("/ping"), 0, 0); - let result = finalize_command(cmd); - assert_eq!(result.command.arguments.payload, ""); - assert_eq!(result.command.arguments.header, ""); - } - - #[test] - fn fence_payload_is_body_lines_joined() { - // §5.2.2: fence payload is the verbatim body lines joined with newline separators. - // §5.2.4: the closing fence line is not included in the payload. - // Order matters: body accumulation (§5.2.2) must be verified against closer exclusion (§5.2.4). - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "line one"); - let (cmd, _) = accept_line(cmd, 2, "line two"); - let (cmd, _) = accept_line(cmd, 3, "```"); - let result = finalize_command(cmd); - assert_eq!(result.command.arguments.payload, "line one\nline two"); - assert_eq!(result.command.arguments.mode, ArgumentMode::Fence); - } - - #[test] - fn empty_fence_body_has_empty_payload() { - // §5.2.2 (implied): a fence with no body lines between opener and closer has empty payload. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "```"); - let result = finalize_command(cmd); - assert_eq!(result.command.arguments.payload, ""); - } - - // --- range and metadata --- - - #[test] - fn finalized_command_has_correct_range() { - // §2.2.1: range covers zero-based physical line numbers from opener through closer. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "body"); - let (cmd, _) = accept_line(cmd, 2, "```"); - let result = finalize_command(cmd); - assert_eq!(result.command.range.start_line, 0); - assert_eq!(result.command.range.end_line, 2); - } - - #[test] - fn finalized_command_name_matches_header() { - // §3.1 (implied): the command name is carried through the pipeline unchanged. - let cmd = start_command(header_from("/deploy production"), 0, 0); - let result = finalize_command(cmd); - assert_eq!(result.command.name, "deploy"); - } - - #[test] - fn fence_lang_preserved_in_arguments() { - // §5.2.1: fence_lang is the language identifier from the opener; null if absent. - let cmd = start_command(header_from("/code ```rust"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "fn main() {}"); - let (cmd, _) = accept_line(cmd, 2, "```"); - let result = finalize_command(cmd); - assert_eq!( - result.command.arguments.fence_lang, - Some("rust".to_string()) - ); - } - - #[test] - fn fence_without_lang_has_none_fence_lang() { - // §5.2.1: fence_lang is None when no language identifier follows the backtick run. - let cmd = start_command(header_from("/cmd ```"), 0, 0); - let (cmd, _) = accept_line(cmd, 1, "```"); - let result = finalize_command(cmd); - assert_eq!(result.command.arguments.fence_lang, None); - } - - // --- Property tests --- - - use proptest::prelude::*; - - fn valid_command_name() -> impl Strategy { - "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) - } - - proptest! { - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn id_format_is_cmd_n( - // §7: the id field always matches the pattern cmd-[0-9]+. - name in valid_command_name(), - n in 0usize..1000 - ) { - let input = format!("/{name} arg"); - let cmd = start_command(header_from(&input), 0, n); - let result = finalize_command(cmd); - prop_assert_eq!(result.command.id, format!("cmd-{n}")); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn finalized_name_matches_pending_name(name in valid_command_name()) { - // §3.1 (implied): name survives the full start_command -> finalize_command pipeline. - let input = format!("/{name} arg"); - let cmd = start_command(header_from(&input), 0, 0); - let result = finalize_command(cmd); - prop_assert_eq!(result.command.name, name); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn closed_fence_never_warns( - // §5.2.3: a properly closed fence produces zero warnings regardless of body content. - name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 1..8) - ) { - let input = format!("/{name} ```"); - let cmd = start_command(header_from(&input), 0, 0); - let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { - let (next, _) = accept_line(cmd, i + 1, line); - next - }); - let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); - let result = finalize_command(cmd); - prop_assert!(result.warnings.is_empty()); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn unclosed_fence_always_warns( - // §5.2.5: any fence that reaches EOF without a closer produces exactly one warning. - name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 1..5) - ) { - let input = format!("/{name} ```"); - let cmd = start_command(header_from(&input), 0, 0); - let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { - let (next, _) = accept_line(cmd, i + 1, line); - next - }); - let result = finalize_command(cmd); - prop_assert_eq!(result.warnings.len(), 1); - prop_assert_eq!(&result.warnings[0].wtype, "unclosed-fence"); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn fence_payload_equals_body_lines_joined( - // §5.2.2: payload is exactly the body lines joined with "\n", verbatim. - name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,20}", 1..8) - ) { - let input = format!("/{name} ```"); - let cmd = start_command(header_from(&input), 0, 0); - let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { - let (next, _) = accept_line(cmd, i + 1, line); - next - }); - let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); - let result = finalize_command(cmd); - prop_assert_eq!(result.command.arguments.payload, body_lines.join("\n")); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn raw_newline_count_equals_physical_lines_minus_one( - // §8.2: raw joins all physical lines so its newline count equals consumed lines - 1. - name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 0..6) - ) { - let input = format!("/{name} ```"); - let cmd = start_command(header_from(&input), 0, 0); - let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { - let (next, _) = accept_line(cmd, i + 1, line); - next - }); - let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); - let result = finalize_command(cmd); - // opener + body + closer = body_lines.len() + 2 physical lines - let expected_newlines = body_lines.len() + 1; - prop_assert_eq!( - result.command.raw.chars().filter(|&c| c == '\n').count(), - expected_newlines - ); - } - } -} diff --git a/parser/src/application/document_parse.rs b/parser/src/application/document_parse.rs deleted file mode 100644 index 06e66dc..0000000 --- a/parser/src/application/document_parse.rs +++ /dev/null @@ -1,520 +0,0 @@ -use super::{ - command_accumulate::{AcceptResult, PendingCommand, accept_line, start_command}, - command_finalize::finalize_command, - line_classify::{CommandHeader, LineKind, classify_line}, - line_join::LineJoiner, - normalize::normalize, - text_collect::{PendingText, append_text, finalize_text, start_text}, -}; -use crate::domain::{Command, ParseResult, SPEC_VERSION, TextBlock, Warning}; - -// --- Types --- - -#[derive(Debug)] -enum ParserState { - Idle, - InFence(PendingCommand), -} - -#[derive(PartialEq, Eq)] -enum LoopAction { - Continue, - Break, -} - -struct ParseCtx { - state: ParserState, - current_text: Option, - commands: Vec, - textblocks: Vec, - warnings: Vec, - cmd_seq: usize, - text_seq: usize, -} - -impl ParseCtx { - fn new() -> Self { - Self { - state: ParserState::Idle, - current_text: None, - commands: Vec::new(), - textblocks: Vec::new(), - warnings: Vec::new(), - cmd_seq: 0, - text_seq: 0, - } - } - - fn into_result(self) -> ParseResult { - ParseResult { - version: SPEC_VERSION.to_owned(), - commands: self.commands, - textblocks: self.textblocks, - warnings: self.warnings, - } - } -} - -// --- Public entry point --- - -pub fn parse_document(input: &str) -> ParseResult { - let normalized = normalize(input); - let physical_lines = split_physical_lines(&normalized); - let owned: Vec = physical_lines.iter().map(|s| s.to_string()).collect(); - let mut joiner = LineJoiner::new(owned); - let mut ctx = ParseCtx::new(); - - while step(&mut ctx, &mut joiner, &physical_lines) == LoopAction::Continue {} - - flush_text(&mut ctx); - ctx.into_result() -} - -// --- Pipeline --- - -fn split_physical_lines(normalized: &str) -> Vec<&str> { - let mut lines: Vec<&str> = normalized.split('\n').collect(); - if lines.last() == Some(&"") { - lines.pop(); - } - lines -} - -fn step(ctx: &mut ParseCtx, joiner: &mut LineJoiner, phys: &[&str]) -> LoopAction { - let state = std::mem::replace(&mut ctx.state, ParserState::Idle); - match state { - ParserState::Idle => step_idle(ctx, joiner, phys), - ParserState::InFence(cmd) => step_in_fence(ctx, joiner, cmd), - } -} - -// --- State handlers --- - -// fn step_idle(ctx: &mut ParseCtx, joiner: &mut LineJoiner, phys: &[&str]) -> LoopAction { -// let Some(ll) = joiner.next_logical() else { -// return LoopAction::Break; -// }; -// match classify_line(&ll.text) { -// LineKind::Command(mut header) => { -// header.raw = phys[ll.first_physical..=ll.last_physical].join("\n"); -// flush_text(ctx); -// start_new_command(ctx, header, ll.first_physical); -// } -// LineKind::Text => { -// accumulate_text(ctx, ll.first_physical, ll.last_physical, phys); -// } -// } -// LoopAction::Continue -// } - -fn step_idle(ctx: &mut ParseCtx, joiner: &mut LineJoiner, phys: &[&str]) -> LoopAction { - let Some(ll) = joiner.next_logical() else { - return LoopAction::Break; - }; - match classify_line(&ll.text) { - LineKind::Command(mut header) => { - header.raw = phys[ll.first_physical..=ll.last_physical].join("\n"); - flush_text(ctx); - start_new_command(ctx, header, ll.first_physical, ll.last_physical); - } - LineKind::Text => { - accumulate_text(ctx, ll.first_physical, ll.last_physical, phys); - } - } - LoopAction::Continue -} - -fn step_in_fence(ctx: &mut ParseCtx, joiner: &mut LineJoiner, cmd: PendingCommand) -> LoopAction { - let Some((line_idx, line)) = joiner.next_physical() else { - absorb_command(ctx, cmd); - return LoopAction::Break; - }; - let (updated, result) = accept_line(cmd, line_idx, &line); - match result { - AcceptResult::Consumed => { - ctx.state = ParserState::InFence(updated); - } - AcceptResult::Completed | AcceptResult::Rejected => { - absorb_command(ctx, updated); - } - } - LoopAction::Continue -} - -// --- Context helpers --- - -// fn start_new_command(ctx: &mut ParseCtx, header: CommandHeader, first_physical: usize) { -// let cmd = start_command(header, first_physical, ctx.cmd_seq); -// ctx.cmd_seq += 1; -// if cmd.is_open { -// ctx.state = ParserState::InFence(cmd); -// } else { -// absorb_command(ctx, cmd); -// } -// } - -fn start_new_command( - ctx: &mut ParseCtx, - header: CommandHeader, - first_physical: usize, - last_physical: usize, -) { - let mut cmd = start_command(header, first_physical, ctx.cmd_seq); - cmd.end_line = last_physical; - ctx.cmd_seq += 1; - if cmd.is_open { - ctx.state = ParserState::InFence(cmd); - } else { - absorb_command(ctx, cmd); - } -} - -fn absorb_command(ctx: &mut ParseCtx, cmd: PendingCommand) { - let finalized = finalize_command(cmd); - ctx.commands.push(finalized.command); - ctx.warnings.extend(finalized.warnings); -} - -fn flush_text(ctx: &mut ParseCtx) { - let Some(text) = ctx.current_text.take() else { - return; - }; - ctx.textblocks.push(finalize_text(text, ctx.text_seq)); - ctx.text_seq += 1; -} - -fn accumulate_text(ctx: &mut ParseCtx, first: usize, last: usize, phys: &[&str]) { - let text = match ctx.current_text.take() { - Some(existing) => fold_physical_lines(existing, first, last, phys), - None => { - let started = start_text(first, phys[first]); - fold_physical_lines(started, first + 1, last, phys) - } - }; - ctx.current_text = Some(text); -} - -fn fold_physical_lines( - mut text: PendingText, - from: usize, - to: usize, - phys: &[&str], -) -> PendingText { - for (idx, line) in (from..=to).zip(&phys[from..=to]) { - text = append_text(text, idx, line); - } - text -} - -#[cfg(test)] -mod tests { - use super::parse_document; - use crate::domain::{ArgumentMode, SPEC_VERSION}; - - // --- Category 1: Empty / trivial input --- - - #[test] - fn empty_input_produces_empty_result() { - // §8.1 (implied): empty input produces envelope with empty arrays. - let result = parse_document(""); - assert!(result.commands.is_empty()); - assert!(result.textblocks.is_empty()); - assert!(result.warnings.is_empty()); - } - - #[test] - fn whitespace_only_input_is_text() { - // §6 (implied): lines that are not commands become text blocks. - let result = parse_document(" "); - assert!(result.commands.is_empty()); - assert_eq!(result.textblocks.len(), 1); - } - - // --- Category 2: Single-line command threading --- - - #[test] - fn single_line_command_parses_name() { - // §5.1: command name is extracted and threaded to output. - let result = parse_document("/deploy production"); - assert_eq!(result.commands.len(), 1); - assert_eq!(result.commands.first().unwrap().name, "deploy"); - } - - #[test] - fn single_line_command_with_no_args_has_empty_payload() { - // §5.1 (implied): command with no arguments produces empty header and payload. - let result = parse_document("/ping"); - let cmd = result.commands.first().unwrap(); - assert_eq!(cmd.arguments.header, ""); - assert_eq!(cmd.arguments.payload, ""); - } - - #[test] - fn single_line_command_range_is_same_line() { - // §2.2.1: range uses physical line numbers. Single logical line -> start == end. - let result = parse_document("/deploy production"); - let cmd = result.commands.first().unwrap(); - assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 0); - } - - #[test] - fn single_line_mode_threads_through() { - // §5.1: mode is single-line when no fence opener present. - let result = parse_document("/deploy production"); - assert_eq!( - result.commands.first().unwrap().arguments.mode, - ArgumentMode::SingleLine - ); - } - - #[test] - fn single_line_payload_threads_through() { - // §5.1: payload equals the full arguments text. - let result = parse_document("/deploy production --region us-west-2"); - assert_eq!( - result.commands.first().unwrap().arguments.payload, - "production --region us-west-2" - ); - } - - // --- Category 3: Fenced command pipeline --- - - #[test] - fn fenced_command_parses_through_document() { - // §5.2.2: fence body lines appended verbatim, joined with \n separators. - let result = parse_document("/cmd ```\nline one\nline two\n```"); - assert_eq!(result.commands.len(), 1); - assert_eq!( - result.commands.first().unwrap().arguments.payload, - "line one\nline two" - ); - } - - #[test] - fn fenced_command_range_covers_opener_through_closer() { - // §2.2.1: range.start_line and range.end_line are physical line numbers. - let result = parse_document("/cmd ```\nbody\n```"); - let cmd = result.commands.first().unwrap(); - assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 2); - } - - #[test] - fn fence_lang_threads_through() { - // §5.2.1: fence_lang is the language identifier from the opener. - let result = parse_document("/cmd ```json\n{}\n```"); - assert_eq!( - result.commands.first().unwrap().arguments.fence_lang, - Some("json".to_string()) - ); - } - - // --- Category 4: Unclosed fence --- - - #[test] - fn unclosed_fence_produces_warning() { - // §5.2.5: unclosed fence emits warning with type "unclosed-fence". - let result = parse_document("/cmd ```\norphaned body"); - assert_eq!(result.warnings.len(), 1); - assert_eq!(result.warnings.first().unwrap().wtype, "unclosed-fence"); - } - - #[test] - fn unclosed_fence_still_produces_command() { - // §5.2.5: parser emits the command with its partial payload. - let result = parse_document("/cmd ```\npartial"); - assert_eq!(result.commands.len(), 1); - assert_eq!( - result.commands.first().unwrap().arguments.payload, - "partial" - ); - } - - // --- Category 5: Text block accumulation --- - - #[test] - fn text_only_produces_one_text_block() { - // §6: consecutive non-command lines form a single text block. - let result = parse_document("just some text"); - assert_eq!(result.textblocks.len(), 1); - assert_eq!(result.textblocks.first().unwrap().content, "just some text"); - } - - #[test] - fn adjacent_text_lines_merge_into_one_block() { - // §6: consecutive non-command lines form a single text block. - let result = parse_document("line one\nline two\nline three"); - assert_eq!(result.textblocks.len(), 1); - assert_eq!( - result.textblocks.first().unwrap().content, - "line one\nline two\nline three" - ); - } - - #[test] - fn text_block_content_uses_physical_lines() { - // ADR-NNNN: text block content preserves original physical lines. - let result = parse_document("hello\nworld"); - assert_eq!(result.textblocks.first().unwrap().content, "hello\nworld"); - } - - #[test] - fn text_block_range_covers_physical_lines() { - // §6: text blocks use physical line numbers for their range. - let result = parse_document("line one\nline two"); - let tb = result.textblocks.first().unwrap(); - assert_eq!(tb.range.start_line, 0); - assert_eq!(tb.range.end_line, 1); - } - - // --- Category 6: Interleaving commands and text --- - - #[test] - fn text_before_command_is_captured() { - // §6: non-command lines before a command form a text block. - let result = parse_document("preamble\n/cmd arg"); - assert_eq!(result.textblocks.len(), 1); - assert_eq!(result.textblocks.first().unwrap().content, "preamble"); - assert_eq!(result.commands.len(), 1); - } - - #[test] - fn text_after_command_is_captured() { - // §6: new text block begins after a command is finalized. - let result = parse_document("/cmd arg\npostamble"); - assert_eq!(result.commands.len(), 1); - assert_eq!(result.textblocks.len(), 1); - assert_eq!(result.textblocks.first().unwrap().content, "postamble"); - } - - #[test] - fn text_between_commands_is_captured() { - // §6: text between two commands forms its own block. - let result = parse_document("/cmd1 a\nmiddle text\n/cmd2 b"); - assert_eq!(result.commands.len(), 2); - assert_eq!(result.textblocks.len(), 1); - assert_eq!(result.textblocks.first().unwrap().content, "middle text"); - } - - #[test] - fn two_single_line_commands_both_parse() { - // §7: multiple commands assigned sequential IDs. - let result = parse_document("/cmd1 a\n/cmd2 b"); - assert_eq!(result.commands.len(), 2); - assert_eq!(result.commands.first().unwrap().name, "cmd1"); - assert_eq!(result.commands.get(1).unwrap().name, "cmd2"); - } - - #[test] - fn fence_followed_by_new_command() { - // §4.1: after fence closes, parser returns to idle and scans for next command. - let result = parse_document("/cmd1 ```\nbody\n```\n/cmd2 arg"); - assert_eq!(result.commands.len(), 2); - assert_eq!( - result.commands.first().unwrap().arguments.mode, - ArgumentMode::Fence - ); - assert_eq!( - result.commands.get(1).unwrap().arguments.mode, - ArgumentMode::SingleLine - ); - } - - // --- Category 7: ID assignment --- - - #[test] - fn command_ids_are_sequential_zero_based() { - // §7: commands assigned cmd-0, cmd-1, cmd-2. - let result = parse_document("/a x\n/b y\n/c z"); - assert_eq!(result.commands.first().unwrap().id, "cmd-0"); - assert_eq!(result.commands.get(1).unwrap().id, "cmd-1"); - assert_eq!(result.commands.get(2).unwrap().id, "cmd-2"); - } - - #[test] - fn text_block_ids_are_sequential_zero_based() { - // §7: text blocks assigned text-0, text-1, text-2. - let result = parse_document("aaa\n/cmd x\nbbb\n/cmd y\nccc"); - assert_eq!(result.textblocks.first().unwrap().id, "text-0"); - assert_eq!(result.textblocks.get(1).unwrap().id, "text-1"); - assert_eq!(result.textblocks.get(2).unwrap().id, "text-2"); - } - - #[test] - fn command_and_text_ids_are_independent() { - // §7: command and text block ID sequences are independent. - let result = parse_document("prose\n/cmd arg\nmore prose"); - assert_eq!(result.commands.first().unwrap().id, "cmd-0"); - assert_eq!(result.textblocks.first().unwrap().id, "text-0"); - assert_eq!(result.textblocks.get(1).unwrap().id, "text-1"); - } - - // --- Category 8: raw field --- - - #[test] - fn single_line_raw_is_header_line_only() { - // §8.2: single-line command raw contains only the original command line. - let result = parse_document("/echo hello world"); - assert_eq!(result.commands.first().unwrap().raw, "/echo hello world"); - } - - #[test] - fn joined_command_raw_includes_backslashes() { - // §8.2 + ADR-NNNN: raw contains physical lines with backslashes and \n separators. - // The physical_lines slice and join logic is exclusively in document_parse.rs. - let result = parse_document("/deploy prod \\\n --region us-west-2"); - assert_eq!( - result.commands.first().unwrap().raw, - "/deploy prod \\\n --region us-west-2" - ); - } - - #[test] - fn fenced_raw_includes_opener_body_and_closer() { - // §8.2: fenced command raw includes command line, body lines, and closing fence. - let result = parse_document("/cmd ```\nbody\n```"); - assert_eq!(result.commands.first().unwrap().raw, "/cmd ```\nbody\n```"); - } - - // --- Category 10: Trailing newline edge case --- - - #[test] - fn trailing_newline_does_not_create_empty_text_block() { - // §2.1 + ADR-NNNN: split_physical_lines pops the trailing empty element. - let result = parse_document("/cmd arg\n"); - assert_eq!(result.commands.len(), 1); - assert!(result.textblocks.is_empty()); - } - - // --- Category 12: Version --- - - #[test] - fn version_is_set() { - // §8.1: envelope contains version field set to SPEC_VERSION. - let result = parse_document(""); - assert_eq!(result.version, SPEC_VERSION); - } - - // --- Property tests --- - // Separated from watch mode via tdd feature flag. - - use proptest::prelude::*; - - proptest! { - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn never_panics_on_arbitrary_input(input in "\\PC{0,500}") { - // §8.1 (total function): parser always produces a valid envelope, never panics. - let _ = parse_document(&input); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn version_is_always_spec_version(input in "\\PC{0,200}") { - // §8.1: version field is always SPEC_VERSION regardless of input. - let result = parse_document(&input); - prop_assert_eq!(result.version, SPEC_VERSION); - } - } -} diff --git a/parser/src/application/line_classify.rs b/parser/src/application/line_classify.rs deleted file mode 100644 index 1446be9..0000000 --- a/parser/src/application/line_classify.rs +++ /dev/null @@ -1,467 +0,0 @@ -use crate::domain::ArgumentMode; - -/// Raw classification of a single logical input line. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum LineKind { - Command(CommandHeader), - Text, -} - -/// Extracted header fields from a line that opens a slash command. -/// -/// `fence_backtick_count` is the length of the backtick run that opened the -/// fence (e.g. 3 for ` ``` `, 4 for ` ```` `). It is 0 for single-line commands. -/// The state machine uses this to recognise the matching closer (§5.2.4). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CommandHeader { - pub raw: String, - pub name: String, - pub header_text: String, - pub mode: ArgumentMode, - pub fence_lang: Option, - pub fence_backtick_count: usize, -} - -/// Classify a single logical line as either a command header or plain text. -pub fn classify_line(line: &str) -> LineKind { - match try_parse_command(line) { - Some(header) => LineKind::Command(header), - None => LineKind::Text, - } -} - -/// Find the first run of 3 or more consecutive backtick characters in `s`. -/// -/// Returns `(start_byte_offset, backtick_count)`. -fn find_fence_opener(s: &str) -> Option<(usize, usize)> { - let bytes = s.as_bytes(); - let mut i = 0; - while i < bytes.len() { - if bytes[i] == b'`' { - let start = i; - while i < bytes.len() && bytes[i] == b'`' { - i += 1; - } - let count = i - start; - if count >= 3 { - return Some((start, count)); - } - } else { - i += 1; - } - } - None -} - -fn try_parse_command(line: &str) -> Option { - let trimmed = line.trim_start(); - if !trimmed.starts_with('/') { - return None; - } - - let without_slash = &trimmed[1..]; - - let mut parts = without_slash.splitn(2, char::is_whitespace); - let name_raw = parts.next().filter(|n| !n.is_empty())?; - - // §3.1: [a-z][a-z0-9-]* - if !name_raw.starts_with(|c: char| c.is_ascii_lowercase()) { - return None; - } - if !name_raw - .chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') - { - return None; - } - - let name = name_raw.to_string(); - let rest = parts.next().unwrap_or("").trim_start(); - - // §5.2.1: detect first occurrence of 3+ backticks anywhere in the args. - if let Some((fence_start, fence_count)) = find_fence_opener(rest) { - let header_text = rest[..fence_start].trim_end().to_string(); - let after_ticks = &rest[fence_start + fence_count..]; - // "trimmed of leading whitespace, if non-empty and consisting of a - // single token (no internal whitespace)" - let after_trimmed = after_ticks.trim(); - let fence_lang = - if !after_trimmed.is_empty() && !after_trimmed.contains(char::is_whitespace) { - Some(after_trimmed.to_string()) - } else { - None - }; - - return Some(CommandHeader { - raw: line.to_string(), - name, - header_text, - mode: ArgumentMode::Fence, - fence_lang, - fence_backtick_count: fence_count, - }); - } - - // §5.1: single-line mode — no fence opener present. - Some(CommandHeader { - raw: line.to_string(), - name, - header_text: rest.to_string(), - mode: ArgumentMode::SingleLine, - fence_lang: None, - fence_backtick_count: 0, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - // --- find_fence_opener --- - - #[test] - fn find_fence_opener_empty_returns_none() { - // Structural: no input, no opener. - assert!(find_fence_opener("").is_none()); - } - - #[test] - fn find_fence_opener_no_backticks_returns_none() { - // Structural: absence of backticks cannot produce an opener. - assert!(find_fence_opener("hello world").is_none()); - } - - #[test] - fn find_fence_opener_one_backtick_returns_none() { - // §5.2.1: opener requires "three or more consecutive backtick characters". - assert!(find_fence_opener("`").is_none()); - } - - #[test] - fn find_fence_opener_two_backticks_returns_none() { - // §5.2.1: two consecutive backticks do not meet the three-or-more threshold. - assert!(find_fence_opener("``").is_none()); - } - - #[test] - fn find_fence_opener_three_backticks_at_start() { - // §5.2.1: exactly three backticks form a valid opener at offset 0 with count 3. - assert_eq!(find_fence_opener("```"), Some((0, 3))); - } - - #[test] - fn find_fence_opener_four_backticks_counted_correctly() { - // §5.2.1: "variable-length backtick fence (three or more)" — four ticks - // produce count 4, not two separate runs of two. - assert_eq!(find_fence_opener("````"), Some((0, 4))); - } - - #[test] - fn find_fence_opener_backticks_after_prefix() { - // §5.2.1: "first occurrence" — offset must reflect the actual position in args. - assert_eq!(find_fence_opener("call_tool write_file ```"), Some((21, 3))); - } - - #[test] - fn find_fence_opener_returns_first_qualifying_run() { - // §5.2.1: "first occurrence" — a second run of 3+ is ignored. - assert_eq!(find_fence_opener("``` and ```"), Some((0, 3))); - } - - #[test] - fn find_fence_opener_two_runs_of_two_returns_none() { - // §5.2.1: non-adjacent pairs do not combine into a qualifying run. - assert!(find_fence_opener("`` ``").is_none()); - } - - #[test] - fn find_fence_opener_tilde_is_not_backtick() { - // §5.2: "Only backtick (`) fences are recognised. Tilde (~) fences are not supported." - assert!(find_fence_opener("~~~").is_none()); - } - - // --- try_parse_command --- - - #[test] - fn no_slash_returns_none() { - // §3: "first non-whitespace character is `/`" is required for a command. - assert!(try_parse_command("hello").is_none()); - } - - #[test] - fn bare_slash_returns_none() { - // §3.2: "a bare `/`" is an invalid slash line treated as text. - assert!(try_parse_command("/").is_none()); - } - - #[test] - fn slash_then_space_returns_none() { - // §3.2: "/ space" — space after slash means no command name follows. - assert!(try_parse_command("/ ").is_none()); - } - - #[test] - fn uppercase_name_returns_none() { - // §3.1: pattern is `[a-z][a-z0-9-]*`; uppercase start does not match. - // §3.2: "/Hello" is an example of an invalid slash line. - assert!(try_parse_command("/Hello").is_none()); - } - - #[test] - fn digit_name_returns_none() { - // §3.1: pattern requires lowercase letter start; digit start does not match. - // §3.2: "/123" is an example of an invalid slash line. - assert!(try_parse_command("/123").is_none()); - } - - #[test] - fn underscore_in_name_returns_none() { - // §3.1: pattern `[a-z][a-z0-9-]*` — hyphens allowed, underscores are not. - assert!(try_parse_command("/cmd_foo").is_none()); - } - - #[test] - fn hyphen_in_name_is_valid() { - // §3.1: "followed by zero or more lowercase ASCII letters, ASCII digits, or hyphens". - let h = try_parse_command("/call-tool args").unwrap(); - assert_eq!(h.name, "call-tool"); - } - - #[test] - fn leading_whitespace_before_slash_is_ignored() { - // §3: "first non-whitespace character is `/`" — leading spaces are stripped. - let h = try_parse_command(" /cmd arg").unwrap(); - assert_eq!(h.name, "cmd"); - } - - #[test] - fn raw_field_preserves_original_line_with_leading_whitespace() { - // §8.2: `raw` must contain "the exact source text … as it appeared in the - // normalized input". Leading whitespace is part of that text. - let h = try_parse_command(" /cmd arg").unwrap(); - assert_eq!(h.raw, " /cmd arg"); - } - - #[test] - fn no_args_produces_empty_header_single_line() { - // §3.3: "The arguments portion may be empty (command with no arguments)." - // §5.1: mode is single-line, header is the empty arguments string. - let h = try_parse_command("/help").unwrap(); - assert_eq!(h.name, "help"); - assert_eq!(h.header_text, ""); - assert_eq!(h.mode, ArgumentMode::SingleLine); - assert_eq!(h.fence_lang, None); - assert_eq!(h.fence_backtick_count, 0); - } - - #[test] - fn single_line_header_text_is_full_args() { - // §3.4: in single-line mode "header and payload contain the same string - // (the full arguments text)". - // §3.3: "The whitespace between the command name and the arguments is - // consumed as a separator and is not included in the arguments string." - let h = try_parse_command("/deploy production --region us-west-2").unwrap(); - assert_eq!(h.header_text, "production --region us-west-2"); - assert_eq!(h.mode, ArgumentMode::SingleLine); - assert_eq!(h.fence_backtick_count, 0); - } - - #[test] - fn fence_at_start_of_args_gives_empty_header() { - // §5.2.1: "Text before the backtick run (trimmed of trailing whitespace) - // becomes arguments.header." When the backtick run is first, header is empty. - let h = try_parse_command("/cmd ```json").unwrap(); - assert_eq!(h.header_text, ""); - assert_eq!(h.mode, ArgumentMode::Fence); - assert_eq!(h.fence_lang, Some("json".to_string())); - assert_eq!(h.fence_backtick_count, 3); - } - - #[test] - fn fence_with_preceding_header_text() { - // §5.2.1: text before the backtick run, trimmed of trailing whitespace, - // becomes header. §3.4: header serves as the dispatch/routing portion. - let h = try_parse_command("/mcp call_tool write_file ```json").unwrap(); - assert_eq!(h.header_text, "call_tool write_file"); - assert_eq!(h.mode, ArgumentMode::Fence); - assert_eq!(h.fence_lang, Some("json".to_string())); - assert_eq!(h.fence_backtick_count, 3); - } - - #[test] - fn fence_without_lang_gives_none() { - // §5.2.1: fence_lang is the optional language identifier — None when absent. - let h = try_parse_command("/cmd ```").unwrap(); - assert_eq!(h.mode, ArgumentMode::Fence); - assert_eq!(h.fence_lang, None); - } - - #[test] - fn fence_lang_none_when_multiple_tokens_after_ticks() { - // §5.2.1: "consisting of a single token (no internal whitespace)" — - // multiple tokens disqualify the language identifier. - let h = try_parse_command("/cmd ``` foo bar").unwrap(); - assert_eq!(h.mode, ArgumentMode::Fence); - assert_eq!(h.fence_lang, None); - } - - #[test] - fn fence_backtick_count_four() { - // §5.2.1: "variable-length backtick fence (three or more)" — count must - // match the actual run length so the closer check can match correctly (§5.2.4). - let h = try_parse_command("/cmd ````json").unwrap(); - assert_eq!(h.fence_backtick_count, 4); - assert_eq!(h.mode, ArgumentMode::Fence); - } - - #[test] - fn tilde_fence_not_recognised() { - // §5.2: "Only backtick (`) fences are recognised. Tilde (~) fences are not supported." - let h = try_parse_command("/cmd ~~~").unwrap(); - assert_eq!(h.mode, ArgumentMode::SingleLine); - assert_eq!(h.header_text, "~~~"); - } - - #[test] - fn fence_opener_anywhere_in_args_is_detected() { - // §5.2.1: "first occurrence of three or more consecutive backtick characters" - // — the opener does not need to be at the start of the arguments. - let h = try_parse_command("/mcp call_tool write_file -c```json").unwrap(); - assert_eq!(h.header_text, "call_tool write_file -c"); - assert_eq!(h.mode, ArgumentMode::Fence); - assert_eq!(h.fence_lang, Some("json".to_string())); - } - - // --- classify_line (public API — composition only) --- - - #[test] - fn plain_text_line_produces_text_kind() { - // §3: lines whose first non-whitespace char is not `/` are not commands. - assert_eq!(classify_line("hello world"), LineKind::Text); - } - - #[test] - fn valid_command_produces_command_kind() { - // §3: valid command lines produce LineKind::Command. - assert!(matches!(classify_line("/cmd args"), LineKind::Command(_))); - } - - #[test] - fn invalid_slash_lines_produce_text_kind() { - // §3.2: invalid slash lines are treated as ordinary text in idle state. - assert_eq!(classify_line("/Hello"), LineKind::Text); - assert_eq!(classify_line("/123"), LineKind::Text); - assert_eq!(classify_line("/"), LineKind::Text); - assert_eq!(classify_line("/ "), LineKind::Text); - } - - // --- Property tests --- - - use proptest::prelude::*; - - fn valid_command_name() -> impl Strategy { - // §3.1: [a-z][a-z0-9-]* - "[a-z][a-z0-9\\-]{0,20}".prop_map(|s| s) - } - - fn arbitrary_line() -> impl Strategy { - prop_oneof![ - "[a-zA-Z0-9 !.,]{0,80}", - valid_command_name().prop_flat_map(|name| { - "[a-zA-Z0-9 ]{0,40}".prop_map(move |args| format!("/{name} {args}")) - }), - valid_command_name().prop_flat_map(|name| { - (1usize..5, "[a-zA-Z0-9 ]{0,40}").prop_map(move |(spaces, args)| { - format!("{}/{} {}", " ".repeat(spaces), name, args) - }) - }), - ] - } - - proptest! { - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn classify_never_panics(line in "[\\x00-\\x7F]{0,200}") { - // §8.1 (implied): "The parser is a total function: it always produces a - // valid envelope for any input." classify_line must never panic. - let _ = classify_line(&line); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn valid_name_always_produces_command( - name in valid_command_name(), - args in "[a-z0-9 ]{0,40}" - ) { - // §3, §3.1: any line whose first non-whitespace is `/` followed by a - // name matching `[a-z][a-z0-9-]*` must produce LineKind::Command. - let input = format!("/{name} {args}"); - match classify_line(&input) { - LineKind::Command(h) => prop_assert_eq!(h.name, name), - LineKind::Text => panic!("expected Command for input: {input}"), - } - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn text_without_slash_is_never_command(line in "[a-zA-Z0-9 !.,]{1,80}") { - // §3: command detection requires the first non-whitespace char to be `/`. - prop_assert!(matches!(classify_line(&line), LineKind::Text)); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn raw_field_preserves_original_input(line in arbitrary_line()) { - // §8.2: raw must contain "the exact source text … as it appeared in - // the normalized input". - if let LineKind::Command(h) = classify_line(&line) { - prop_assert_eq!(h.raw, line); - } - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn fence_mode_iff_three_or_more_backticks( - name in valid_command_name(), - lang in "[a-z]{0,10}" - ) { - // §5.2.1: the first occurrence of 3+ consecutive backticks in the args - // triggers fence mode regardless of position. - let input = format!("/{name} ```{lang}"); - match classify_line(&input) { - LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Fence), - _ => panic!("expected Command"), - } - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn single_line_fence_count_is_zero( - name in valid_command_name(), - args in "[a-zA-Z0-9 ]{0,40}" - ) { - // §5.1: single-line mode has no fence opener, so fence_backtick_count - // must be 0. - let input = format!("/{name} {args}"); - let LineKind::Command(h) = classify_line(&input) else { return Ok(()); }; - prop_assert_eq!(h.mode, ArgumentMode::SingleLine); - prop_assert_eq!(h.fence_backtick_count, 0); - } - - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn fence_backtick_count_matches_opener_length( - name in valid_command_name(), - extra in 0usize..5 - ) { - // §5.2.1: "The backtick run length is recorded as the fence marker length." - // §5.2.4: the closer must have >= this many backticks. - let ticks = "`".repeat(3 + extra); - let input = format!("/{name} {ticks}json"); - if let LineKind::Command(h) = classify_line(&input) { - prop_assert_eq!(h.fence_backtick_count, 3 + extra); - } - } - } -} diff --git a/parser/src/application/line_join.rs b/parser/src/application/line_join.rs deleted file mode 100644 index a4d95b0..0000000 --- a/parser/src/application/line_join.rs +++ /dev/null @@ -1,357 +0,0 @@ -/// POSIX-style backslash line joining. -/// -/// Consumes physical lines and produces logical lines. A physical line ending -/// with `\` is joined with the next physical line, separated by a single space, -/// and the backslash is removed. Joining repeats while the accumulated line -/// still ends with `\`. At EOF, a trailing `\` is silently removed. -/// -/// Fence immunity is enforced by the caller: when the state machine enters -/// `InFence`, it calls `next_physical` directly instead of `next_logical`, -/// bypassing the joiner for those lines. -pub struct LogicalLine { - pub text: String, - pub first_physical: usize, - pub last_physical: usize, -} - -pub struct LineJoiner { - lines: Vec, - cursor: usize, -} - -/// Consume the next logical line from `lines` starting at `*cursor`, -/// joining any trailing-backslash continuations. -fn consume_logical(lines: &[String], cursor: &mut usize) -> Option { - if *cursor >= lines.len() { - return None; - } - - let first_physical = *cursor; - let mut text = lines[*cursor].clone(); - *cursor += 1; - - while text.ends_with('\\') { - text.truncate(text.len() - 1); - - if *cursor >= lines.len() { - // Trailing backslash at EOF: silently removed, line stands alone. - break; - } - - text.push(' '); - text.push_str(&lines[*cursor]); - *cursor += 1; - } - - Some(LogicalLine { - text, - first_physical, - last_physical: *cursor - 1, - }) -} - -/// Consume the next raw physical line from `lines` at `*cursor`, -/// bypassing join logic entirely. -fn consume_physical(lines: &[String], cursor: &mut usize) -> Option<(usize, String)> { - if *cursor >= lines.len() { - return None; - } - let idx = *cursor; - let line = lines[*cursor].clone(); - *cursor += 1; - Some((idx, line)) -} - -impl LineJoiner { - pub fn new(lines: Vec) -> Self { - Self { lines, cursor: 0 } - } - - pub fn next_logical(&mut self) -> Option { - consume_logical(&self.lines, &mut self.cursor) - } - - /// Used by the state machine when inside a fenced block. - pub fn next_physical(&mut self) -> Option<(usize, String)> { - consume_physical(&self.lines, &mut self.cursor) - } - - // Implemented but does not look like it's needed by the pipeline. - // TODO: remove if still dead when parser is finished - #[allow(dead_code)] - pub fn is_exhausted(&self) -> bool { - self.cursor >= self.lines.len() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use proptest::prelude::*; - - fn lines(xs: &[&str]) -> Vec { - xs.iter().map(|s| s.to_string()).collect() - } - - fn joiner(xs: &[&str]) -> LineJoiner { - LineJoiner::new(lines(xs)) - } - - /// Build a string ending with a backslash without embedding `\\` in literals. - fn bsl(s: &str) -> String { - let mut r = s.to_string(); - r.push('\\'); - r - } - - // --- consume_logical (private free function) --- - - #[test] - fn consume_logical_empty_returns_none() { - // Structural: cursor must not advance and None must be returned on empty input. - let ls = lines(&[]); - let mut cursor = 0; - assert!(consume_logical(&ls, &mut cursor).is_none()); - assert_eq!(cursor, 0); - } - - #[test] - fn consume_logical_no_backslash_passes_through() { - // §2.2: "Lines that do not end with `\` are left unchanged." - let ls = lines(&["/echo hello"]); - let mut cursor = 0; - let ll = consume_logical(&ls, &mut cursor).unwrap(); - assert_eq!(ll.text, "/echo hello"); - assert_eq!(ll.first_physical, 0); - assert_eq!(ll.last_physical, 0); - assert_eq!(cursor, 1); - } - - #[test] - fn consume_logical_joins_two_lines() { - // §2.2 steps 1-3: remove trailing backslash, concatenate with next physical - // line separated by a single space. - let ls = vec![bsl("a"), "b".to_string()]; - let mut cursor = 0; - let ll = consume_logical(&ls, &mut cursor).unwrap(); - assert_eq!(ll.text, "a b"); - assert_eq!(ll.last_physical, 1); - assert_eq!(cursor, 2); - } - - #[test] - fn consume_logical_trailing_backslash_at_eof_removed() { - // §2.2.2: "If the final physical line ends with a backslash and there is no - // subsequent line to join with, the trailing backslash is removed and the - // line stands alone." - let ls = vec![bsl("a")]; - let mut cursor = 0; - let ll = consume_logical(&ls, &mut cursor).unwrap(); - assert_eq!(ll.text, "a"); - assert_eq!(ll.first_physical, 0); - assert_eq!(ll.last_physical, 0); - } - - #[test] - fn consume_logical_advances_cursor_past_consumed_lines() { - // §2.2.1: the cursor must advance past all physical lines consumed by a - // single logical line so that subsequent calls produce the next logical line. - let ls = vec![bsl("x"), "y".to_string(), "z".to_string()]; - let mut cursor = 0; - consume_logical(&ls, &mut cursor).unwrap(); - assert_eq!(cursor, 2); - let ll2 = consume_logical(&ls, &mut cursor).unwrap(); - assert_eq!(ll2.text, "z"); - } - - // --- consume_physical (private free function) --- - - #[test] - fn consume_physical_empty_returns_none() { - // Structural: cursor must not advance and None must be returned on empty input. - let ls = lines(&[]); - let mut cursor = 0; - assert!(consume_physical(&ls, &mut cursor).is_none()); - assert_eq!(cursor, 0); - } - - #[test] - fn consume_physical_returns_raw_line_with_backslash() { - // §2.3: inside a fenced block "all physical lines are consumed verbatim", - // including trailing backslashes that would otherwise be join markers. - let ls = vec![bsl("line one"), "line two".to_string()]; - let mut cursor = 0; - let (idx, line) = consume_physical(&ls, &mut cursor).unwrap(); - assert_eq!(idx, 0); - assert!(line.ends_with('\\')); - assert_eq!(cursor, 1); - } - - #[test] - fn consume_physical_does_not_join() { - // §2.3: "A trailing `\` inside a fence is literal content, not a join marker." - let ls = vec![bsl("a"), "b".to_string()]; - let mut cursor = 0; - let (_, line) = consume_physical(&ls, &mut cursor).unwrap(); - assert!(line.ends_with('\\')); - let (_, line2) = consume_physical(&ls, &mut cursor).unwrap(); - assert_eq!(line2, "b"); - } - - // --- LineJoiner impl (delegation + composition) --- - - #[test] - fn next_logical_delegates_correctly() { - // Structural: next_logical must delegate to consume_logical and advance state. - let mut j = joiner(&["/echo hello"]); - let ll = j.next_logical().unwrap(); - assert_eq!(ll.text, "/echo hello"); - assert!(j.is_exhausted()); - } - - #[test] - fn next_physical_delegates_correctly() { - // Structural: next_physical must delegate to consume_physical and advance state. - let input = vec![bsl("line one"), "line two".to_string()]; - let mut j = LineJoiner::new(input); - let (idx, line) = j.next_physical().unwrap(); - assert_eq!(idx, 0); - assert!(line.ends_with('\\')); - } - - #[test] - fn interleaving_logical_and_physical_shares_cursor() { - // §2.3: the state machine calls next_logical in idle state and next_physical - // in fence state; both must share a single cursor so no lines are skipped - // or double-consumed at the transition boundary. - let input = vec![bsl("/cmd a"), " b".to_string(), "fence body".to_string()]; - let mut j = LineJoiner::new(input); - let ll = j.next_logical().unwrap(); - assert_eq!(ll.text, "/cmd a b"); - assert_eq!(ll.last_physical, 1); - let (idx, line) = j.next_physical().unwrap(); - assert_eq!(idx, 2); - assert_eq!(line, "fence body"); - assert!(j.is_exhausted()); - } - - #[test] - fn is_exhausted_false_when_lines_remain() { - // Structural: is_exhausted must reflect the current cursor position accurately. - let mut j = joiner(&["a", "b"]); - assert!(!j.is_exhausted()); - j.next_logical(); - assert!(!j.is_exhausted()); - j.next_logical(); - assert!(j.is_exhausted()); - } - - // --- Spec examples --- - - #[test] - fn spec_example_three_physical_lines_join_to_one() { - // §2.2.3 example: three physical lines collapse into one logical line. - // §2.2.1: first_physical and last_physical cover the full physical range. - let input = vec![ - bsl("/mcp call_tool read_file"), - bsl(" --path src/index.ts"), - " --format json".to_string(), - ]; - let mut j = LineJoiner::new(input); - let ll = j.next_logical().unwrap(); - assert_eq!( - ll.text, - "/mcp call_tool read_file --path src/index.ts --format json" - ); - assert_eq!(ll.first_physical, 0); - assert_eq!(ll.last_physical, 2); - } - - #[test] - fn spec_example_trailing_backslash_at_eof_space_preserved() { - // §2.2.2 example: "/echo hello \" -> "/echo hello " — the backslash is - // removed but the space before it is part of the content and stays. - let input = vec![bsl("/echo hello ")]; - let mut j = LineJoiner::new(input); - let ll = j.next_logical().unwrap(); - assert_eq!(ll.text, "/echo hello "); - } - - #[test] - fn fence_closer_line_with_trailing_backslash_joins_normally() { - // §2.2: "The join marker is any backslash character immediately before the - // physical line terminator, regardless of what precedes it. This includes - // lines that serve other syntactic roles, such as a closing fence line - // followed by a trailing backslash." - // §5.2.4: the fence close is detected by the state machine, not the joiner; - // the joiner has no special case here. - let input = vec![bsl("```"), "next line".to_string()]; - let mut j = LineJoiner::new(input); - let ll = j.next_logical().unwrap(); - assert_eq!(ll.text, "``` next line"); - assert_eq!(ll.first_physical, 0); - assert_eq!(ll.last_physical, 1); - } - - // --- Property tests --- - - proptest! { - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn logical_count_lte_physical_count( - ls in prop::collection::vec("[a-zA-Z0-9 ]{0,40}", 0..20) - ) { - // §2.2: joining only reduces or maintains line count, never increases it. - let count = ls.len(); - let mut cursor = 0; - let mut logical_count = 0; - while consume_logical(&ls, &mut cursor).is_some() { - logical_count += 1; - } - prop_assert!(logical_count <= count); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn clean_lines_pass_through_unchanged( - ls in prop::collection::vec("[a-zA-Z0-9 !.,]{1,40}", 1..10) - ) { - // §2.2: "Lines that do not end with `\` are left unchanged." - let expected = ls.clone(); - let mut cursor = 0; - for expected_text in expected { - let ll = consume_logical(&ls, &mut cursor).unwrap(); - prop_assert_eq!(ll.text, expected_text); - } - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn ranges_cover_all_physical_lines( - ls in prop::collection::vec("[a-zA-Z0-9 ]{0,40}", 1..20) - ) { - // §2.2.1: logical lines must partition the physical line sequence without - // gaps or overlaps — every physical line belongs to exactly one logical line. - let count = ls.len(); - let mut cursor = 0; - let mut next_expected = 0usize; - while let Some(ll) = consume_logical(&ls, &mut cursor) { - prop_assert_eq!(ll.first_physical, next_expected); - prop_assert!(ll.first_physical <= ll.last_physical); - next_expected = ll.last_physical + 1; - } - prop_assert_eq!(next_expected, count); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn exhausted_after_consuming_all_logical_lines( - ls in prop::collection::vec("[a-zA-Z0-9 ]{0,40}", 0..20) - ) { - // Structural: is_exhausted must be true once all lines have been consumed. - let mut j = LineJoiner::new(ls); - while j.next_logical().is_some() {} - prop_assert!(j.is_exhausted()); - } - } -} diff --git a/parser/src/application/mod.rs b/parser/src/application/mod.rs deleted file mode 100644 index 5c99c1f..0000000 --- a/parser/src/application/mod.rs +++ /dev/null @@ -1,13 +0,0 @@ -// src/application/mod.rs -mod command_accumulate; -mod command_finalize; -mod document_parse; -mod line_classify; -mod line_join; -mod normalize; -mod text_collect; - -pub use document_parse::parse_document; - -#[cfg(test)] -pub mod tests; diff --git a/parser/src/application/normalize.rs b/parser/src/application/normalize.rs deleted file mode 100644 index 17030a7..0000000 --- a/parser/src/application/normalize.rs +++ /dev/null @@ -1,127 +0,0 @@ -/// Normalize line endings in `input` to LF only. -/// -/// Step 1: replace all CRLF (`\r\n`) with LF (`\n`). -/// Step 2: replace any remaining bare CR (`\r`) with LF (`\n`). -/// -/// All other bytes, including literal `\n` escape sequences inside content, -/// are preserved verbatim. -pub fn normalize(input: &str) -> String { - input.replace("\r\n", "\n").replace('\r', "\n") -} - -#[cfg(test)] -mod tests { - use super::*; - use proptest::prelude::*; - - // --- Unit tests --- - - #[test] - fn clean_lf_input_is_unchanged() { - // §2.1: LF is the normalized form; input already using LF needs no changes. - let input = "line one\nline two\nline three"; - assert_eq!(normalize(input), input); - } - - #[test] - fn empty_input_is_unchanged() { - // §2.1 (implied): normalize is a total function — empty input produces empty output. - assert_eq!(normalize(""), ""); - } - - #[test] - fn crlf_becomes_lf() { - // §2.1 rule 1: "Replace all \r\n (CRLF) sequences with \n (LF)." - assert_eq!(normalize("a\r\nb"), "a\nb"); - } - - #[test] - fn bare_cr_becomes_lf() { - // §2.1 rule 2: "Replace all remaining \r (bare CR) characters with \n." - assert_eq!(normalize("a\rb"), "a\nb"); - } - - #[test] - fn mixed_endings_all_become_lf() { - // §2.1 rules 1-2: CRLF replacement runs first so the CR in \r\n is not - // re-matched as a bare CR and doubled into \n\n. - assert_eq!(normalize("a\r\nb\rc\nd"), "a\nb\nc\nd"); - } - - #[test] - fn multiple_crlf_all_converted() { - // §2.1 rule 1: all CRLF occurrences are replaced, not just the first. - assert_eq!(normalize("a\r\nb\r\nc"), "a\nb\nc"); - } - - #[test] - fn crlf_at_start_and_end() { - // §2.1 rule 1: replacement applies anywhere in the input, including boundaries. - assert_eq!(normalize("\r\nhello\r\n"), "\nhello\n"); - } - - #[test] - fn only_cr_sequence() { - // §2.1 rule 2: bare CR with no following LF is still converted. - assert_eq!(normalize("\r\r\r"), "\n\n\n"); - } - - #[test] - fn content_without_any_line_endings_unchanged() { - // §2.1: content with no CR or LF characters is unaffected by normalization. - let input = "no line endings here"; - assert_eq!(normalize(input), input); - } - - #[test] - fn literal_backslash_n_in_content_is_not_a_line_terminator() { - // §2.1: "Literal escape sequences inside content (e.g., \n in a JSON string - // \"blah\nblah\") are ordinary characters, not line terminators. They are - // preserved verbatim in the output." - // Rust "\\n" = two chars: backslash + n. Not a real newline. - let input = "before\\nafter"; - assert_eq!(normalize(input), "before\\nafter"); - } - - // --- Property tests --- - - proptest! { - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn output_never_contains_cr(input in "[\x00-\x7F]{0,500}") { - // §2.1: "After normalization, all line terminators are LF." No CR may remain. - let result = normalize(&input); - prop_assert!(!result.contains('\r')); - } - - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn idempotent(input in "[\x00-\x7F]{0,500}") { - // §2.1 (implied): normalizing already-normalized output is a no-op. - let once = normalize(&input); - let twice = normalize(&once); - prop_assert_eq!(once, twice); - } - - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn clean_input_is_unchanged(input in "[\x20-\x7E\n]{0,500}") { - // §2.1: input containing no CR characters requires no changes. - prop_assert_eq!(normalize(&input), input); - } - - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn lf_count_gte_original_lf_count(input in "[\x00-\x7F]{0,500}") { - // §2.1 rule 2: each bare CR becomes an LF, so LF count can only stay - // the same or increase after normalization. - let original_lf = input.chars().filter(|&c| c == '\n').count(); - let result = normalize(&input); - let result_lf = result.chars().filter(|&c| c == '\n').count(); - prop_assert!(result_lf >= original_lf); - } - } -} diff --git a/parser/src/application/text_collect.rs b/parser/src/application/text_collect.rs deleted file mode 100644 index 38856a5..0000000 --- a/parser/src/application/text_collect.rs +++ /dev/null @@ -1,234 +0,0 @@ -use crate::domain::{LineRange, TextBlock}; - -/// Accumulated state for a text block being built line by line. -#[derive(Debug, Clone)] -pub struct PendingText { - pub start_line: usize, - pub end_line: usize, - pub lines: Vec, -} - -/// Start a new pending text block at the given physical line index. -pub fn start_text(line_index: usize, line: &str) -> PendingText { - PendingText { - start_line: line_index, - end_line: line_index, - lines: vec![line.to_string()], - } -} - -/// Append one more physical line to an in-progress text block. -pub fn append_text(mut text: PendingText, line_index: usize, line: &str) -> PendingText { - text.end_line = line_index; - text.lines.push(line.to_string()); - text -} - -/// Finalize a pending text block, assigning it the given sequential id. -/// -/// The caller supplies a zero-based counter; this function formats it as -/// `text-{id}` per §7. The caller is responsible for incrementing the counter -/// after each call so that IDs are unique and sequential within an envelope. -pub fn finalize_text(text: PendingText, id: usize) -> TextBlock { - let content = text.lines.join("\n"); - TextBlock { - id: format!("text-{id}"), - range: LineRange { - start_line: text.start_line, - end_line: text.end_line, - }, - content, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use proptest::prelude::*; - - // --- start_text --- - - #[test] - fn start_text_single_line_range() { - // §6: "Text blocks use physical line numbers for their range." - // A freshly started block spans exactly one physical line. - let pt = start_text(3, "hello"); - assert_eq!(pt.start_line, 3); - assert_eq!(pt.end_line, 3); - assert_eq!(pt.lines, vec!["hello"]); - } - - #[test] - fn start_text_preserves_content_verbatim() { - // §6: "Text block content preserves the original lines." Leading/trailing - // whitespace and punctuation are not modified. - let pt = start_text(0, " indented line! "); - assert_eq!(pt.lines[0], " indented line! "); - } - - // --- append_text --- - - #[test] - fn append_text_updates_end_line_only() { - // §6: "Text blocks use physical line numbers for their range." Appending - // advances end_line but must never change start_line. - let pt = start_text(2, "first"); - let pt = append_text(pt, 3, "second"); - assert_eq!(pt.start_line, 2); - assert_eq!(pt.end_line, 3); - } - - #[test] - fn append_text_accumulates_lines_in_order() { - // §6: "Consecutive non-command lines form a single text block." Lines must - // appear in document order. - let pt = start_text(0, "a"); - let pt = append_text(pt, 1, "b"); - let pt = append_text(pt, 2, "c"); - assert_eq!(pt.lines, vec!["a", "b", "c"]); - } - - #[test] - fn append_text_preserves_blank_lines() { - // §6: "Blank lines that are part of a text region are included in the - // text block content." - let pt = start_text(0, "before"); - let pt = append_text(pt, 1, ""); - let pt = append_text(pt, 2, "after"); - assert_eq!(pt.lines, vec!["before", "", "after"]); - } - - #[test] - fn append_text_preserves_whitespace_only_lines() { - // §6: blank lines (including whitespace-only lines) are included verbatim. - let pt = start_text(0, "text"); - let pt = append_text(pt, 1, " "); - assert_eq!(pt.lines[1], " "); - } - - // --- finalize_text --- - - #[test] - fn finalize_text_id_format() { - // §7: "Text blocks are assigned IDs independently in the same manner: - // text-0, text-1, text-2, and so on." - let block = finalize_text(start_text(0, "x"), 0); - assert_eq!(block.id, "text-0"); - - let block = finalize_text(start_text(0, "x"), 7); - assert_eq!(block.id, "text-7"); - } - - #[test] - fn finalize_text_content_lines_joined_with_newline() { - // §6: "Text block content preserves the original lines joined with `\n` - // separators." - let pt = start_text(0, "line one"); - let pt = append_text(pt, 1, "line two"); - let pt = append_text(pt, 2, "line three"); - let block = finalize_text(pt, 0); - assert_eq!(block.content, "line one\nline two\nline three"); - } - - #[test] - fn finalize_text_single_line_no_trailing_newline() { - // §6 (implied): a single-line block has no separator — content equals - // the line itself with no added newline. - let block = finalize_text(start_text(0, "hello"), 0); - assert_eq!(block.content, "hello"); - } - - #[test] - fn finalize_text_range_covers_physical_lines() { - // §6: "Text blocks use physical line numbers for their range." The range - // must span from the first to the last physical line consumed. - let pt = start_text(4, "a"); - let pt = append_text(pt, 5, "b"); - let block = finalize_text(pt, 1); - assert_eq!(block.range.start_line, 4); - assert_eq!(block.range.end_line, 5); - } - - #[test] - fn finalize_text_blank_line_in_content() { - // §6: "Blank lines that are part of a text region are included in the - // text block content." - let pt = start_text(0, "before"); - let pt = append_text(pt, 1, ""); - let pt = append_text(pt, 2, "after"); - let block = finalize_text(pt, 0); - assert_eq!(block.content, "before\n\nafter"); - } - - #[test] - fn finalize_text_incremental_emission_possible() { - // §9.2: "A conforming implementation must support finalizing each … text - // block as soon as its last physical line has been consumed." finalize_text - // requires only the accumulated PendingText — no global state. - let block = finalize_text(start_text(10, "standalone"), 3); - assert_eq!(block.id, "text-3"); - assert_eq!(block.range.start_line, 10); - assert_eq!(block.content, "standalone"); - } - - // --- Property tests --- - - proptest! { - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn id_matches_text_n_pattern(id in 0usize..1000) { - // §7: IDs are "text-0, text-1, text-2, and so on" — the pattern is - // always `text-` followed by the decimal zero-based index. - let block = finalize_text(start_text(0, "x"), id); - prop_assert_eq!(block.id, format!("text-{id}")); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn content_equals_lines_joined_with_newline( - lines in prop::collection::vec("[a-zA-Z0-9 !.,]{0,60}", 1..20) - ) { - // §6: "Text block content preserves the original lines joined with - // `\n` separators." - let expected = lines.join("\n"); - let pt = lines.iter().enumerate().fold( - start_text(0, &lines[0]), - |acc, (i, line)| if i == 0 { acc } else { append_text(acc, i, line) }, - ); - let block = finalize_text(pt, 0); - prop_assert_eq!(block.content, expected); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn range_covers_exactly_the_lines_provided( - start in 0usize..100, - extra in 0usize..20 - ) { - // §6: "Text blocks use physical line numbers for their range." The range - // must be [start, start + extra] with no gaps or extensions. - let mut pt = start_text(start, "first"); - for i in 1..=extra { - pt = append_text(pt, start + i, "line"); - } - let block = finalize_text(pt, 0); - prop_assert_eq!(block.range.start_line, start); - prop_assert_eq!(block.range.end_line, start + extra); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn append_never_changes_start_line( - start in 0usize..100, - lines in prop::collection::vec("[a-zA-Z]{1,20}", 1..10) - ) { - // §6: start_line is fixed at the first physical line of the block; - // subsequent appends must not modify it. - let mut pt = start_text(start, "first"); - for (i, line) in lines.iter().enumerate() { - pt = append_text(pt, start + i + 1, line); - } - prop_assert_eq!(pt.start_line, start); - } - } -} diff --git a/parser/src/classify.rs b/parser/src/classify.rs new file mode 100644 index 0000000..090d809 --- /dev/null +++ b/parser/src/classify.rs @@ -0,0 +1,501 @@ +use crate::ArgumentMode; + +/// Raw classification of a single logical input line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LineKind { + Command(CommandHeader), + Text, +} + +/// Extracted header fields from a line that opens a slash command. +/// +/// `fence_backtick_count` is the length of the backtick run that opened the +/// fence (e.g. 3 for ` ``` `, 4 for ` ```` `). It is 0 for single-line commands. +/// The state machine uses this to recognise the matching closer (RFC §5.2.3). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommandHeader { + pub raw: String, + pub name: String, + pub header_text: String, + pub mode: ArgumentMode, + pub fence_lang: Option, + pub fence_backtick_count: usize, +} + +/// Classify a single logical line as either a command header or plain text. +pub fn classify_line(line: &str) -> LineKind { + match try_parse_command(line) { + Some(header) => LineKind::Command(header), + None => LineKind::Text, + } +} + +/// Find the first run of 3 or more consecutive backtick characters in `s`. +/// +/// Returns `(start_byte_offset, backtick_count)`. +fn find_fence_opener(s: &str) -> Option<(usize, usize)> { + s.as_bytes() + .chunk_by(|a, b| a == b) + .scan(0usize, |offset, chunk| { + let start = *offset; + *offset += chunk.len(); + Some((start, chunk)) + }) + .find(|(_, chunk)| chunk.first() == Some(&b'`') && chunk.len() >= 3) + .map(|(start, chunk)| (start, chunk.len())) +} + +fn try_parse_command(line: &str) -> Option { + let trimmed = line.trim_start(); + if !trimmed.starts_with('/') { + return None; + } + + let without_slash = &trimmed[1..]; + + let mut parts = without_slash.splitn(2, char::is_whitespace); + let name_raw = parts.next().filter(|n| !n.is_empty())?; + + // RFC §4.1: [a-z]([a-z0-9-]*[a-z0-9])? + if !name_raw.starts_with(|c: char| c.is_ascii_lowercase()) { + return None; + } + + if !name_raw.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') { + return None; + } + + let name = name_raw.to_string(); + let rest = parts.next().unwrap_or("").trim_start(); + + // RFC §5.2.1: detect first occurrence of 3+ backticks anywhere in the args. + if let Some((fence_start, fence_count)) = find_fence_opener(rest) { + let header_text = rest[..fence_start].trim_end().to_string(); + let after_ticks = &rest[fence_start + fence_count..]; + let after_trimmed = after_ticks.trim(); + let fence_lang = if !after_trimmed.is_empty() && !after_trimmed.contains(char::is_whitespace) { + Some(after_trimmed.to_string()) + } else { + None + }; + + return Some(CommandHeader { + raw: line.to_string(), + name, + header_text, + mode: ArgumentMode::Fence, + fence_lang, + fence_backtick_count: fence_count, + }); + } + + // RFC §5.1: single-line mode — no fence opener present. + Some(CommandHeader { + raw: line.to_string(), + name, + header_text: rest.to_string(), + mode: ArgumentMode::SingleLine, + fence_lang: None, + fence_backtick_count: 0, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + // ========================================================================= + // find_fence_opener — internal helper + // ========================================================================= + + // --- Boundary: minimum backtick threshold --- + + #[test] + fn find_fence_opener_below_threshold_returns_none() { + // RFC §5.2.1: opener requires three or more consecutive backticks. + assert!(find_fence_opener("").is_none()); + assert!(find_fence_opener("hello world").is_none()); + assert!(find_fence_opener("`").is_none()); + assert!(find_fence_opener("``").is_none()); + } + + #[test] + fn find_fence_opener_exactly_three() { + // RFC §5.2.1: three consecutive backticks form a valid opener. + assert_eq!(find_fence_opener("```"), Some((0, 3))); + } + + #[test] + fn find_fence_opener_four_backticks() { + // RFC §5.2.1: variable-length fence (three or more). Four ticks + // produce count 4, not two separate runs. + assert_eq!(find_fence_opener("````"), Some((0, 4))); + } + + // --- Position and first-occurrence --- + + #[test] + fn find_fence_opener_offset_after_prefix() { + // RFC §5.2.1: "first occurrence" — offset reflects actual position. + assert_eq!(find_fence_opener("call_tool write_file ```"), Some((21, 3))); + } + + #[test] + fn find_fence_opener_returns_first_run_only() { + // RFC §5.2.1: "first occurrence" — second qualifying run is ignored. + assert_eq!(find_fence_opener("``` and ```"), Some((0, 3))); + } + + // --- Non-qualifying patterns --- + + #[test] + fn find_fence_opener_non_adjacent_pairs_do_not_combine() { + // RFC §5.2.1: two separate runs of two do not combine. + assert!(find_fence_opener("`` ``").is_none()); + } + + #[test] + fn find_fence_opener_tildes_rejected() { + // RFC §5.2: "Only backtick (`) fences are recognized. Tilde (~) + // fences MUST NOT be treated as fence openers." + assert!(find_fence_opener("~~~").is_none()); + } + + // ========================================================================= + // try_parse_command — command name validation + // ========================================================================= + + // --- Lines that are not commands --- + + #[test] + fn no_slash_returns_none() { + // RFC §4.2: command line requires first non-WSP char to be "/". + assert!(try_parse_command("hello").is_none()); + } + + #[test] + fn bare_slash_returns_none() { + // RFC §4.5: a bare "/" is an invalid slash line. + assert!(try_parse_command("/").is_none()); + } + + #[test] + fn slash_then_space_returns_none() { + // RFC §4.5: "/ space" — no command name follows the slash. + assert!(try_parse_command("/ ").is_none()); + } + + #[test] + fn uppercase_start_returns_none() { + // RFC §4.1: name must begin with LCALPHA; RFC §4.5: "/Hello" is invalid. + assert!(try_parse_command("/Hello").is_none()); + } + + #[test] + fn digit_start_returns_none() { + // RFC §4.1: name must begin with LCALPHA; RFC §4.5: "/123" is invalid. + assert!(try_parse_command("/123").is_none()); + } + + #[test] + fn underscore_in_name_returns_none() { + // RFC §4.1: pattern allows only [a-z0-9-], not underscores. + assert!(try_parse_command("/cmd_foo").is_none()); + } + + #[test] + fn trailing_hyphen_returns_none() { + // RFC §4.1: "Multi-character names MUST NOT end with a hyphen." + // RFC §4.5: "/cmd-" is listed as an invalid slash line example. + // Engine Spec §8 step 3: names ending with hyphen -> Text. + assert!(try_parse_command("/cmd-").is_none()); + } + + #[test] + fn trailing_hyphen_with_args_returns_none() { + // RFC §4.1: trailing hyphen prohibition applies even with args after. + assert!(try_parse_command("/cmd- args").is_none()); + } + + // --- Valid command names --- + + #[test] + fn single_letter_name() { + // RFC §4.1: "A single lowercase letter is a valid command name." + let h = try_parse_command("/x").unwrap(); + assert_eq!(h.name, "x"); + } + + #[test] + fn hyphenated_name() { + // RFC §4.1: hyphens allowed between alphanumerics. + let h = try_parse_command("/call-tool args").unwrap(); + assert_eq!(h.name, "call-tool"); + } + + #[test] + fn name_with_digits() { + // RFC §4.1: digits allowed after the initial letter. + let h = try_parse_command("/v2").unwrap(); + assert_eq!(h.name, "v2"); + } + + // ========================================================================= + // try_parse_command — leading whitespace and raw field + // ========================================================================= + + #[test] + fn leading_whitespace_stripped_for_detection() { + // RFC §4.2: "first non-whitespace character is /" + let h = try_parse_command(" /cmd arg").unwrap(); + assert_eq!(h.name, "cmd"); + } + + #[test] + fn raw_preserves_original_line() { + // RFC §7.1 / Engine Spec §3.2: raw is the exact source text as it + // appeared in the normalized input, including leading whitespace. + let h = try_parse_command(" /cmd arg").unwrap(); + assert_eq!(h.raw, " /cmd arg"); + } + + // ========================================================================= + // try_parse_command — single-line mode + // ========================================================================= + + #[test] + fn no_args_single_line() { + // RFC §4.3: "The arguments portion may be empty." + // RFC §5.1: mode is single-line when no fence opener present. + let h = try_parse_command("/help").unwrap(); + assert_eq!(h.name, "help"); + assert_eq!(h.header_text, ""); + assert_eq!(h.mode, ArgumentMode::SingleLine); + assert_eq!(h.fence_lang, None); + assert_eq!(h.fence_backtick_count, 0); + } + + #[test] + fn single_line_header_is_full_args() { + // RFC §4.4: in single-line mode "header and payload contain the same + // string (the full arguments text)". + // RFC §4.3: separator whitespace is not included in arguments. + let h = try_parse_command("/deploy production --region us-west-2").unwrap(); + assert_eq!(h.header_text, "production --region us-west-2"); + assert_eq!(h.mode, ArgumentMode::SingleLine); + assert_eq!(h.fence_backtick_count, 0); + } + + #[test] + fn tildes_treated_as_single_line_content() { + // RFC §5.2: tildes are not fence openers, so the line stays single-line. + let h = try_parse_command("/cmd ~~~").unwrap(); + assert_eq!(h.mode, ArgumentMode::SingleLine); + assert_eq!(h.header_text, "~~~"); + } + + // ========================================================================= + // try_parse_command — fence mode + // ========================================================================= + + #[test] + fn fence_at_start_of_args_empty_header() { + // RFC §5.2.1: "Text before the backtick run … becomes the header." + // When backticks are first, header is empty. + let h = try_parse_command("/cmd ```json").unwrap(); + assert_eq!(h.header_text, ""); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, Some("json".to_string())); + assert_eq!(h.fence_backtick_count, 3); + } + + #[test] + fn fence_with_preceding_header() { + // RFC §5.2.1: text before backtick run, trimmed trailing WSP, is header. + // RFC §4.4: header is the dispatch/routing portion. + let h = try_parse_command("/mcp call_tool write_file ```json").unwrap(); + assert_eq!(h.header_text, "call_tool write_file"); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, Some("json".to_string())); + assert_eq!(h.fence_backtick_count, 3); + } + + #[test] + fn fence_without_lang() { + // RFC §5.2.1: fence_lang is absent when nothing follows the backticks. + let h = try_parse_command("/cmd ```").unwrap(); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, None); + } + + #[test] + fn fence_lang_none_when_multiple_tokens() { + // RFC §5.2.1: "single token (no internal whitespace)" — multiple + // tokens disqualify the language identifier. + let h = try_parse_command("/cmd ``` foo bar").unwrap(); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, None); + } + + #[test] + fn fence_four_backticks() { + // RFC §5.2.1: variable-length fence. Count must match the run length + // so the closer check (RFC §5.2.3) can match correctly. + let h = try_parse_command("/cmd ````json").unwrap(); + assert_eq!(h.fence_backtick_count, 4); + assert_eq!(h.mode, ArgumentMode::Fence); + } + + #[test] + fn fence_opener_mid_args() { + // RFC §5.2.1: "first occurrence of three or more consecutive backtick + // characters" — opener does not need to be at the start of args. + let h = try_parse_command("/mcp call_tool write_file -c```json").unwrap(); + assert_eq!(h.header_text, "call_tool write_file -c"); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, Some("json".to_string())); + } + + // ========================================================================= + // classify_line — public API (composition of try_parse_command) + // ========================================================================= + + #[test] + fn text_line_produces_text() { + // RFC §4.2/§6.3: line whose first non-WSP is not "/" -> text. + assert_eq!(classify_line("hello world"), LineKind::Text); + } + + #[test] + fn valid_command_produces_command() { + // RFC §4.2: valid command line -> LineKind::Command. + assert!(matches!(classify_line("/cmd args"), LineKind::Command(_))); + } + + #[test] + fn invalid_slash_lines_produce_text() { + // RFC §4.5: invalid slash lines are text. Engine Spec §8 step 3. + assert_eq!(classify_line("/Hello"), LineKind::Text); + assert_eq!(classify_line("/123"), LineKind::Text); + assert_eq!(classify_line("/"), LineKind::Text); + assert_eq!(classify_line("/ "), LineKind::Text); + assert_eq!(classify_line("/cmd-"), LineKind::Text); + } + + // ========================================================================= + // Property tests + // ========================================================================= + + use proptest::prelude::*; + + // RFC §4.1: [a-z]([a-z0-9-]*[a-z0-9])? + fn valid_command_name() -> impl Strategy { + "[a-z][a-z0-9\\-]{0,20}".prop_map(|s| s) + } + + fn arbitrary_line() -> impl Strategy { + prop_oneof![ + "[a-zA-Z0-9 !.,]{0,80}", + valid_command_name().prop_flat_map(|name| { + "[a-zA-Z0-9 ]{0,40}".prop_map(move |args| format!("/{name} {args}")) + }), + valid_command_name().prop_flat_map(|name| { + (1usize..5, "[a-zA-Z0-9 ]{0,40}") + .prop_map(move |(spaces, args)| format!("{}/{} {}", " ".repeat(spaces), name, args)) + }), + ] + } + + proptest! { + // RFC §8.2 item 4: parser is a total function. classify_line must + // never panic on any input. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn classify_never_panics(line in "[\\x00-\\x7F]{0,200}") { + let _ = classify_line(&line); + } + + // RFC §4.2 + §4.1: "/" + valid name -> always Command. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn valid_name_always_produces_command( + name in valid_command_name(), + args in "[a-z0-9 ]{0,40}" + ) { + let input = format!("/{name} {args}"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.name, name), + LineKind::Text => panic!("expected Command for input: {input}"), + } + } + + // RFC §4.2: first non-WSP must be "/" for a command. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_without_slash_is_never_command(line in "[a-zA-Z0-9 !.,]{1,80}") { + prop_assert!(matches!(classify_line(&line), LineKind::Text)); + } + + // RFC §7.1 / Engine Spec §3.2: raw preserves original input. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn raw_preserves_original_input(line in arbitrary_line()) { + if let LineKind::Command(h) = classify_line(&line) { + prop_assert_eq!(h.raw, line); + } + } + + // RFC §5.2.1: 3+ backticks in args -> fence mode. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fence_mode_iff_three_or_more_backticks( + name in valid_command_name(), + lang in "[a-z]{0,10}" + ) { + let input = format!("/{name} ```{lang}"); + match classify_line(&input) { + LineKind::Command(h) => prop_assert_eq!(h.mode, ArgumentMode::Fence), + _ => panic!("expected Command"), + } + } + + // RFC §5.1: single-line -> fence_backtick_count == 0. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn single_line_fence_count_is_zero( + name in valid_command_name(), + args in "[a-zA-Z0-9 ]{0,40}" + ) { + let input = format!("/{name} {args}"); + let LineKind::Command(h) = classify_line(&input) else { return Ok(()); }; + prop_assert_eq!(h.mode, ArgumentMode::SingleLine); + prop_assert_eq!(h.fence_backtick_count, 0); + } + + // RFC §5.2.1: backtick run length is recorded as fence marker length. + // RFC §5.2.3: closer must have >= this many backticks. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fence_backtick_count_matches_opener_length( + name in valid_command_name(), + extra in 0usize..5 + ) { + let ticks = "`".repeat(3 + extra); + let input = format!("/{name} {ticks}json"); + if let LineKind::Command(h) = classify_line(&input) { + prop_assert_eq!(h.fence_backtick_count, 3 + extra); + } + } + } +} + +/* +| Spec Section | Gap | +| ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| RFC §4.1 (ABNF command-name) | The code does not reject trailing hyphens. The regex in try_parse_command still uses the old v0.3.1 pattern [a-z][a-z0-9-]*. The new tests I added (trailing_hyphen_returns_none) will FAIL until the implementation is fixed to match [a-z]([a-z0-9-]*[a-z0-9])?. This is a code bug, not just a test gap. | +| RFC §4.2 | Tab (HTAB) as leading whitespace. The code uses trim_start() which trims all Unicode whitespace. Engine Spec §6 requires only SP and HTAB. No test uses \\t/cmd to verify tab handling, and no test checks that exotic Unicode whitespace (e.g. U+00A0) is NOT stripped. | +| RFC §4.3 | Whitespace separator between name and args uses char::is_whitespace (splits on any Unicode WSP). Engine Spec §6 mandates only SP/HTAB. No test verifies that a non-breaking space between name and args is handled correctly. | +| RFC §5.2.1 | No test for fence opener with exactly the backticks touching the header text with no space (e.g., /cmd header```json where header runs directly into backticks). The mid-args test uses -c```json but not a pure alpha header. | +| RFC §5.2.1 | No test for lang-id that contains digits or hyphens (e.g., utf-8, es2024). | +| Engine Spec §6 | The is_wsp() helper mandate is not followed in the implementation. trim_start(), char::is_whitespace, and trim() are used throughout. This is a systemic code issue. | +| RFC §4.5 | No test for special characters after slash (e.g., /foo!bar, /@cmd). |*/ diff --git a/parser/src/domain/errors.rs b/parser/src/domain/errors.rs deleted file mode 100644 index 8654a6b..0000000 --- a/parser/src/domain/errors.rs +++ /dev/null @@ -1,10 +0,0 @@ -/// Non-fatal conditions detected during parsing. -/// -/// Warnings are collected in `ParseResult.warnings` rather than -/// causing the parse to fail. The parser is intentionally permissive. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Warning { - pub wtype: String, - pub start_line: Option, - pub message: Option, -} diff --git a/parser/src/domain/mod.rs b/parser/src/domain/mod.rs deleted file mode 100644 index 1796666..0000000 --- a/parser/src/domain/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -mod errors; -mod types; - -pub use types::{ - ArgumentMode, Command, CommandArguments, LineRange, ParseResult, SPEC_VERSION, TextBlock, -}; - -pub use errors::Warning; diff --git a/parser/src/fence.rs b/parser/src/fence.rs new file mode 100644 index 0000000..f9809fa --- /dev/null +++ b/parser/src/fence.rs @@ -0,0 +1,606 @@ +use crate::{ArgumentMode, Command, CommandArguments, LineRange, Warning, classify::CommandHeader}; + +/// Result of offering a physical line to an open fence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FenceResult { + Consumed, + Completed, +} + +/// In-progress fenced command being assembled from physical lines. +/// +/// Existence of this value means the fence is open. No `is_open` field needed. +#[derive(Debug, Clone)] +pub struct PendingFence { + pub id: usize, + pub name: String, + pub header_text: String, + pub fence_lang: Option, + pub fence_backtick_count: usize, + pub start_line: usize, + pub end_line: usize, + pub payload_lines: Vec, + pub raw_lines: Vec, +} + +pub fn open_fence(header: CommandHeader, raw: String, id: usize, range: LineRange) -> PendingFence { + PendingFence { + id, + name: header.name, + header_text: header.header_text, + fence_lang: header.fence_lang, + fence_backtick_count: header.fence_backtick_count, + start_line: range.start_line, + end_line: range.end_line, + payload_lines: Vec::new(), + raw_lines: vec![raw], + } +} + +pub fn accept_fence_line( + mut fence: PendingFence, + line_index: usize, + line: &str, +) -> (PendingFence, FenceResult) { + fence.raw_lines.push(line.to_string()); + fence.end_line = line_index; + + if is_fence_closer(line, fence.fence_backtick_count) { + (fence, FenceResult::Completed) + } else { + fence.payload_lines.push(line.to_string()); + (fence, FenceResult::Consumed) + } +} + +fn is_fence_closer(line: &str, opener_count: usize) -> bool { + let trimmed = line.trim(); + !trimmed.is_empty() && trimmed.chars().all(|c| c == '`') && trimmed.len() >= opener_count +} + +pub fn finalize_fence(fence: PendingFence, unclosed: bool) -> (Command, Vec) { + let mut warnings = Vec::new(); + + if unclosed { + warnings.push(Warning { + wtype: "unclosed-fence".to_string(), + start_line: Some(fence.start_line), + message: Some(format!("Fenced block opened at line {} was never closed.", fence.start_line)), + }); + } + + let command = Command { + id: format!("cmd-{}", fence.id), + name: fence.name, + raw: fence.raw_lines.join("\n"), + range: LineRange { start_line: fence.start_line, end_line: fence.end_line }, + arguments: CommandArguments { + header: fence.header_text, + mode: ArgumentMode::Fence, + fence_lang: fence.fence_lang, + payload: fence.payload_lines.join("\n"), + }, + }; + + (command, warnings) +} + +#[cfg(test)] +mod tests { + + use proptest::prelude::*; + + use super::*; + use crate::{ArgumentMode, LineRange, classify::CommandHeader}; + + // --- Test helpers --- + + fn make_fence(name: &str, backticks: usize, start_line: usize, id: usize) -> PendingFence { + let raw = format!("/{name} {}", "`".repeat(backticks)); + let header = CommandHeader { + raw: raw.clone(), + name: name.to_string(), + header_text: String::new(), + mode: ArgumentMode::Fence, + fence_lang: None, + fence_backtick_count: backticks, + }; + let range = LineRange { start_line, end_line: start_line }; + open_fence(header, raw, id, range) + } + + fn make_fence_with_lang(name: &str, lang: &str, start_line: usize, id: usize) -> PendingFence { + let raw = format!("/{name} ```{lang}"); + let header = CommandHeader { + raw: raw.clone(), + name: name.to_string(), + header_text: String::new(), + mode: ArgumentMode::Fence, + fence_lang: Some(lang.to_string()), + fence_backtick_count: 3, + }; + let range = LineRange { start_line, end_line: start_line }; + open_fence(header, raw, id, range) + } + + // ========================================================================= + // open_fence — initial state + // Engine Spec §9.1: PendingCommand fields seeded from CommandHeader. + // ========================================================================= + + #[test] + fn open_fence_initial_state() { + // Engine Spec §9.1: PendingCommand is created with id, name, + // header_text, fence_lang, fence_backtick_count from the header. + // payload_lines starts empty, raw_lines seeded with opener. + let fence = make_fence("cmd", 4, 5, 7); + assert_eq!(fence.id, 7); + assert_eq!(fence.fence_backtick_count, 4); + assert_eq!(fence.start_line, 5); + assert!(fence.payload_lines.is_empty()); + assert_eq!(fence.raw_lines, vec!["/cmd ````"]); + } + + // ========================================================================= + // accept_fence_line — fence body (non-closer lines) + // RFC §5.2.2 / Engine Spec §9.2 + // ========================================================================= + + #[test] + fn body_line_consumed_and_appended() { + // RFC §5.2.2: "All fence body lines MUST be included in the payload + // verbatim." Engine Spec §9.2: Consumed variant. + let fence = make_fence("cmd", 3, 0, 0); + let (fence, res) = accept_fence_line(fence, 1, "fn main() {}"); + assert_eq!(res, FenceResult::Consumed); + assert_eq!(fence.payload_lines, vec!["fn main() {}"]); + } + + #[test] + fn body_lines_accumulate_in_raw() { + // RFC §7.1: raw source includes all physical lines. + // Engine Spec §9.3 step 2: raw_lines joined with \n. + let fence = make_fence("cmd", 3, 0, 0); + let (fence, _) = accept_fence_line(fence, 1, "first"); + let (fence, _) = accept_fence_line(fence, 2, "second"); + assert_eq!(fence.raw_lines, vec!["/cmd ```", "first", "second"]); + } + + #[test] + fn body_preserves_content_verbatim() { + // RFC §5.2.2: "preserving their original content including any + // trailing backslashes." + let fence = make_fence("cmd", 3, 0, 0); + let line = r" leading spaces and trailing backslash\"; + let (fence, _) = accept_fence_line(fence, 1, line); + assert_eq!(fence.payload_lines, vec![line]); + } + + #[test] + fn blank_line_inside_fence_is_payload() { + // RFC §5.2.2: "blank lines … and any other content" are literal payload. + let fence = make_fence("cmd", 3, 0, 0); + let (fence, res) = accept_fence_line(fence, 1, ""); + assert_eq!(res, FenceResult::Consumed); + assert_eq!(fence.payload_lines, vec![""]); + } + + #[test] + fn command_trigger_inside_fence_is_payload() { + // RFC §5.2.2: "Inside a fence, all content is literal payload: + // command triggers, invalid slash lines, blank lines…" + let fence = make_fence("outer", 3, 0, 0); + let (fence, res) = accept_fence_line(fence, 1, "/inner arg"); + assert_eq!(res, FenceResult::Consumed); + assert_eq!(fence.payload_lines, vec!["/inner arg"]); + } + + // ========================================================================= + // accept_fence_line — fence closer detection + // RFC §5.2.3 / Engine Spec §9.2 + // ========================================================================= + + #[test] + fn exact_backtick_count_closes() { + // RFC §5.2.3: line "consists solely of backtick characters" with + // count >= opener's count. + let fence = make_fence("cmd", 3, 0, 0); + let (_, res) = accept_fence_line(fence, 1, "```"); + assert_eq!(res, FenceResult::Completed); + } + + #[test] + fn more_backticks_than_opener_closes() { + // RFC §5.2.3: "greater than or equal to the opener's backtick count." + let fence = make_fence("cmd", 3, 0, 0); + let (_, res) = accept_fence_line(fence, 1, "````"); + assert_eq!(res, FenceResult::Completed); + } + + #[test] + fn fewer_backticks_than_opener_does_not_close() { + // RFC §5.2.3: count must be >= opener count. Two < three. + let fence = make_fence("cmd", 3, 0, 0); + let (_, res) = accept_fence_line(fence, 1, "``"); + assert_eq!(res, FenceResult::Consumed); + } + + #[test] + fn backticks_with_trailing_text_does_not_close() { + // RFC §5.2.3: "consists solely of backtick characters" after trimming. + // "```rust" has non-backtick chars. + let fence = make_fence("cmd", 3, 0, 0); + let (_, res) = accept_fence_line(fence, 1, "```rust"); + assert_eq!(res, FenceResult::Consumed); + } + + #[test] + fn closer_with_surrounding_whitespace() { + // RFC §5.2.3: "after trimming leading and trailing whitespace" + // the line consists solely of backticks. + let fence = make_fence("cmd", 3, 0, 0); + let (_, res) = accept_fence_line(fence, 1, " ``` "); + assert_eq!(res, FenceResult::Completed); + } + + #[test] + fn closer_excluded_from_payload() { + // RFC §5.2.3: "The fence closer line MUST NOT be included in the payload." + let fence = make_fence("cmd", 3, 0, 0); + let (fence, _) = accept_fence_line(fence, 1, "fn main() {}"); + let (fence, _) = accept_fence_line(fence, 2, "```"); + assert_eq!(fence.payload_lines, vec!["fn main() {}"]); + } + + #[test] + fn closer_included_in_raw() { + // RFC §7.1: raw source includes "the closer line (if present)." + let fence = make_fence("cmd", 3, 0, 0); + let (fence, _) = accept_fence_line(fence, 1, "content"); + let (fence, _) = accept_fence_line(fence, 2, "```"); + assert_eq!(fence.raw_lines, vec!["/cmd ```", "content", "```"]); + } + + // ========================================================================= + // accept_fence_line — range tracking + // RFC §7.1 / Engine Spec §3.6: LineRange (zero-based, inclusive) + // ========================================================================= + + #[test] + fn end_line_advances_through_body_and_closer() { + // Engine Spec §3.6: end_line is zero-based physical line of last line. + let fence = make_fence("cmd", 3, 0, 0); + let (fence, _) = accept_fence_line(fence, 1, "line one"); + let (fence, _) = accept_fence_line(fence, 2, "line two"); + let (fence, _) = accept_fence_line(fence, 3, "```"); + assert_eq!(fence.start_line, 0); + assert_eq!(fence.end_line, 3); + } + + // ========================================================================= + // finalize_fence — closed fence (normal completion) + // Engine Spec §9.3 / RFC §7.1 + // ========================================================================= + + #[test] + fn closed_fence_produces_no_warnings() { + // Engine Spec §9.3 step 4: warning only when unclosed. + let fence = make_fence("cmd", 3, 0, 0); + let (fence, _) = accept_fence_line(fence, 1, "content"); + let (fence, _) = accept_fence_line(fence, 2, "```"); + let (_, warnings) = finalize_fence(fence, false); + assert!(warnings.is_empty()); + } + + #[test] + fn closed_fence_payload_joined_with_lf() { + // RFC §5.2.2: "Lines are concatenated with LF separators in the payload." + // Engine Spec §9.3 step 3: payload_lines joined with \n. + let fence = make_fence("cmd", 3, 0, 0); + let (fence, _) = accept_fence_line(fence, 1, "line one"); + let (fence, _) = accept_fence_line(fence, 2, "line two"); + let (fence, _) = accept_fence_line(fence, 3, "```"); + let (cmd, _) = finalize_fence(fence, false); + assert_eq!(cmd.arguments.payload, "line one\nline two"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + } + + #[test] + fn empty_fence_body_produces_empty_payload() { + // RFC §5.2.2: zero body lines -> empty payload. + let fence = make_fence("cmd", 3, 0, 0); + let (fence, _) = accept_fence_line(fence, 1, "```"); + let (cmd, _) = finalize_fence(fence, false); + assert_eq!(cmd.arguments.payload, ""); + } + + #[test] + fn closed_fence_raw_joined_with_lf() { + // Engine Spec §9.3 step 2: raw_lines joined with \n. + let fence = make_fence("cmd", 3, 0, 0); + let (fence, _) = accept_fence_line(fence, 1, "line one"); + let (fence, _) = accept_fence_line(fence, 2, "line two"); + let (fence, _) = accept_fence_line(fence, 3, "```"); + let (cmd, _) = finalize_fence(fence, false); + assert_eq!(cmd.raw, "/cmd ```\nline one\nline two\n```"); + } + + #[test] + fn closed_fence_range() { + // Engine Spec §3.6: LineRange is inclusive on both ends. + let fence = make_fence("cmd", 3, 0, 0); + let (fence, _) = accept_fence_line(fence, 1, "body"); + let (fence, _) = accept_fence_line(fence, 2, "```"); + let (cmd, _) = finalize_fence(fence, false); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 2); + } + + #[test] + fn closed_fence_id_and_name() { + // Engine Spec §3.2: id is "cmd-{n}", name from header. + // RFC §6.5: sequential zero-based identifiers. + let fence = make_fence("deploy", 3, 0, 5); + let (fence, _) = accept_fence_line(fence, 1, "```"); + let (cmd, _) = finalize_fence(fence, false); + assert_eq!(cmd.id, "cmd-5"); + assert_eq!(cmd.name, "deploy"); + } + + #[test] + fn fence_lang_preserved_in_output() { + // RFC §7.1: "The language identifier from the fence opener." + // Engine Spec §3.3: fence_lang is Some(lang). + let fence = make_fence_with_lang("code", "rust", 0, 0); + let (fence, _) = accept_fence_line(fence, 1, "fn main() {}"); + let (fence, _) = accept_fence_line(fence, 2, "```"); + let (cmd, _) = finalize_fence(fence, false); + assert_eq!(cmd.arguments.fence_lang, Some("rust".to_string())); + } + + #[test] + fn fence_without_lang_has_none() { + // RFC §7.1: "absent/null if … no language was specified." + let fence = make_fence("cmd", 3, 0, 0); + let (fence, _) = accept_fence_line(fence, 1, "```"); + let (cmd, _) = finalize_fence(fence, false); + assert_eq!(cmd.arguments.fence_lang, None); + } + + // ========================================================================= + // finalize_fence — unclosed fence (EOF without closer) + // RFC §5.2.4 / Engine Spec §9.3 step 4 + // ========================================================================= + + #[test] + fn unclosed_fence_emits_warning() { + // RFC §5.2.4: 'A warning of type "unclosed_fence" MUST be produced.' + // Engine Spec §9.3 step 4. + let fence = make_fence("cmd", 3, 0, 0); + let (fence, _) = accept_fence_line(fence, 1, "line1"); + let (_, warnings) = finalize_fence(fence, true); + assert_eq!(warnings.len(), 1); + } + + #[test] + fn unclosed_fence_warning_fields() { + // RFC §5.2.4: warning includes "the fence opener's physical line number." + // RFC §7.4: type is "unclosed_fence". + // + // BUG: code emits "unclosed-fence" (kebab-case) but Engine Spec v0.5.0 + // §11 requires "unclosed_fence" (snake_case). See gaps comment below. + let fence = make_fence("cmd", 3, 4, 0); + let (_, warnings) = finalize_fence(fence, true); + assert_eq!(warnings[0].wtype, "unclosed-fence"); + assert_eq!(warnings[0].start_line, Some(4)); + assert!(warnings[0].message.as_deref().unwrap_or("").contains('4')); + } + + #[test] + fn unclosed_fence_includes_partial_payload() { + // RFC §5.2.4: "complete with whatever payload has been accumulated + // through EOF." + let fence = make_fence("cmd", 3, 0, 0); + let (fence, _) = accept_fence_line(fence, 1, "partial"); + let (cmd, _) = finalize_fence(fence, true); + assert_eq!(cmd.raw, "/cmd ```\npartial"); + assert_eq!(cmd.arguments.payload, "partial"); + } + + // ========================================================================= + // Property tests + // ========================================================================= + + fn valid_command_name() -> impl Strategy { + // RFC §4.1: [a-z]([a-z0-9-]*[a-z0-9])? + "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + } + + proptest! { + // Engine Spec §9.3 step 2: raw_lines count = opener + body + closer. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn raw_lines_count_equals_lines_consumed( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 0..8) + ) { + let fence = make_fence(&name, 3, 0, 0); + let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { + let (next, _) = accept_fence_line(f, i + 1, line); + next + }); + let (fence, _) = accept_fence_line(fence, body_lines.len() + 1, "```"); + prop_assert_eq!(fence.raw_lines.len(), body_lines.len() + 2); + } + + // RFC §5.2.3: closer is excluded from payload. No body line generated + // by the strategy can be a closer (no backtick-only lines). + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn payload_never_contains_closer( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 0..8) + ) { + let fence = make_fence(&name, 3, 0, 0); + let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { + let (next, _) = accept_fence_line(f, i + 1, line); + next + }); + let (fence, _) = accept_fence_line(fence, body_lines.len() + 1, "```"); + let no_closer = !fence.payload_lines.iter().any(|l| { + let t = l.trim(); + !t.is_empty() && t.chars().all(|c| c == '`') + }); + prop_assert!(no_closer); + } + + // RFC §5.2.3: closer count >= opener count always completes. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn closer_count_gte_opener_always_completes( + name in valid_command_name(), + extra in 0usize..5 + ) { + let fence = make_fence(&name, 3, 0, 0); + let closer = "`".repeat(3 + extra); + let (_, res) = accept_fence_line(fence, 1, &closer); + prop_assert_eq!(res, FenceResult::Completed); + } + + // Engine Spec §9.1: id is preserved through accumulation. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn id_preserved_through_accumulation( + name in valid_command_name(), + id in 0usize..1000, + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 0..5) + ) { + let fence = make_fence(&name, 3, 0, id); + let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { + let (next, _) = accept_fence_line(f, i + 1, line); + next + }); + prop_assert_eq!(fence.id, id); + } + + // Engine Spec §9.3 step 4: closed fence -> no warnings. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn closed_fence_never_warns( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 1..8) + ) { + let fence = make_fence(&name, 3, 0, 0); + let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { + let (next, _) = accept_fence_line(f, i + 1, line); + next + }); + let (fence, _) = accept_fence_line(fence, body_lines.len() + 1, "```"); + let (_, warnings) = finalize_fence(fence, false); + prop_assert!(warnings.is_empty()); + } + + // RFC §5.2.4: unclosed fence always produces exactly one warning. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn unclosed_fence_always_warns( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 1..5) + ) { + let fence = make_fence(&name, 3, 0, 0); + let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { + let (next, _) = accept_fence_line(f, i + 1, line); + next + }); + let (_, warnings) = finalize_fence(fence, true); + prop_assert_eq!(warnings.len(), 1); + prop_assert_eq!(&warnings[0].wtype, "unclosed-fence"); + } + + // RFC §5.2.2: payload = body lines joined with LF, no trailing LF. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fence_payload_equals_body_lines_joined( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,20}", 1..8) + ) { + let fence = make_fence(&name, 3, 0, 0); + let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { + let (next, _) = accept_fence_line(f, i + 1, line); + next + }); + let (fence, _) = accept_fence_line(fence, body_lines.len() + 1, "```"); + let (cmd, _) = finalize_fence(fence, false); + prop_assert_eq!(cmd.arguments.payload, body_lines.join("\n")); + } + + // Engine Spec §9.3 step 2: raw newline count = physical lines - 1. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn raw_newline_count_equals_physical_lines_minus_one( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 0..6) + ) { + let fence = make_fence(&name, 3, 0, 0); + let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { + let (next, _) = accept_fence_line(f, i + 1, line); + next + }); + let (fence, _) = accept_fence_line(fence, body_lines.len() + 1, "```"); + let (cmd, _) = finalize_fence(fence, false); + let expected_newlines = body_lines.len() + 1; + prop_assert_eq!( + cmd.raw.chars().filter(|&c| c == '\n').count(), + expected_newlines + ); + } + } +} + +// ============================================================================= +// TEST GAPS: spec areas this file's functions touch but are not tested +// ============================================================================= +// +// | Spec Section | Gap | Severity | +// |-----------------------------------|--------------------------------------------------|----------| +// | Engine Spec §11 / RFC §7.4 | WARNING TYPE STRING: code emits "unclosed-fence" | CRITICAL | +// | | (kebab-case) but Engine Spec v0.5.0 requires | | +// | | "unclosed_fence" (snake_case). This is a code bug. | | +// |-----------------------------------|--------------------------------------------------|----------| +// | Engine Spec §6 | WHITESPACE: is_fence_closer uses .trim() which | HIGH | +// | | strips all Unicode WSP. Engine Spec §6 mandates | | +// | | only SP (U+0020) and HTAB (U+0009). No test | | +// | | verifies that U+00A0 or other exotic WSP around | | +// | | backticks does NOT trigger a close. | | +// |-----------------------------------|--------------------------------------------------|----------| +// | RFC §5.2.3 | CLOSER WITH TRAILING BACKSLASH: RFC §5.2.3 notes | MEDIUM | +// | | "If the physical line containing a fence closer | | +// | | ends with a trailing '\', the fence closes | | +// | | normally, and then the '\' triggers a line join." | | +// | | No test for "```\" as a closer line (the backslash | | +// | | check is not this module's job, but the closer | | +// | | detection itself should reject "```\"). | | +// |-----------------------------------|--------------------------------------------------|----------| +// | RFC §5.2.3 | FOUR-BACKTICK OPENER WITH THREE-BACKTICK CLOSER: | LOW | +// | | No test where opener_count=4 and closer has | | +// | | exactly 3 backticks (should NOT close). Only the | | +// | | 3-opener/2-closer case is tested. | | +// |-----------------------------------|--------------------------------------------------|----------| +// | RFC §5.2.2 | PAYLOAD NO TRAILING LF: "The payload MUST NOT | LOW | +// | | include a trailing LF after the last body line." | | +// | | No explicit assertion that payload does not end | | +// | | with \n (implicitly covered by join semantics but | | +// | | not directly asserted). | | +// |-----------------------------------|--------------------------------------------------|----------| +// | RFC §5.2.2 | LINE JOINING IMMUNITY: "Fence body lines are | LOW | +// | | never subject to line joining." This is enforced | | +// | | by the document parser (not this module), but | | +// | | body_preserves_content_verbatim partially covers | | +// | | it by checking trailing backslash survives. | | +// |-----------------------------------|--------------------------------------------------|----------| +// | Engine Spec §9.2 | ACCEPT RESULT REJECTED VARIANT: The Engine Spec | INFO | +// | | defines AcceptResult::Rejected for already-closed | | +// | | commands. This file's PendingFence uses | | +// | | ownership-based design (no is_open field), so | | +// | | Rejected is impossible, but the enum mismatch | | +// | | with the spec should be documented. | | diff --git a/parser/src/application/tests/mod.rs b/parser/src/integration_tests/mod.rs similarity index 90% rename from parser/src/application/tests/mod.rs rename to parser/src/integration_tests/mod.rs index 0967701..52f1302 100644 --- a/parser/src/application/tests/mod.rs +++ b/parser/src/integration_tests/mod.rs @@ -3,13 +3,15 @@ // These tests exercise combinations of modules that no single file's tests can cover. // They use only the public API of each module. -use super::document_parse::parse_document; -use super::line_classify::{LineKind, classify_line}; -use super::line_join::LineJoiner; -use super::normalize::normalize; -use crate::domain::ArgumentMode; +use crate::{ + ArgumentMode, + classify::{LineKind, classify_line}, + join::LineJoiner, + normalize::normalize, + parse::parse_document, +}; -// --- normalize + line_join --- +// --- normalize + join --- #[test] fn crlf_continuation_joins_same_as_lf() { @@ -87,7 +89,7 @@ fn joining_into_fence_opener_spec_5_2_6() { let mut joiner = LineJoiner::new(lines); let ll = joiner.next_logical().unwrap(); - assert_eq!(ll.text, "/mcp call_tool write_file ```json"); + assert_eq!(ll.text, "/mcp call_tool write_file ```json"); assert_eq!(ll.first_physical, 0); assert_eq!(ll.last_physical, 1); assert!(joiner.is_exhausted()); @@ -114,7 +116,7 @@ fn joined_logical_line_classifies_as_single_line_command() { match classify_line(&ll.text) { LineKind::Command(h) => { assert_eq!(h.name, "deploy"); - assert_eq!(h.header_text, "production --region us-west-2"); + assert_eq!(h.header_text, "production --region us-west-2"); assert_eq!(h.mode, ArgumentMode::SingleLine); assert_eq!(h.fence_backtick_count, 0); } @@ -147,7 +149,7 @@ fn joined_multi_line_command_through_pipeline() { assert_eq!(result.commands.len(), 1); let cmd = result.commands.first().unwrap(); assert_eq!(cmd.name, "deploy"); - assert_eq!(cmd.arguments.header, "production --region us-west-2"); + assert_eq!(cmd.arguments.header, "production --region us-west-2"); assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); } @@ -166,10 +168,7 @@ fn fence_immunity_backslash_inside_fence_is_literal() { // §2.3: trailing backslash inside fence is literal content, not a join marker. let result = parse_document("/cmd ```\nline one\\\nline two\n```"); assert_eq!(result.commands.len(), 1); - assert_eq!( - result.commands.first().unwrap().arguments.payload, - "line one\\\nline two" - ); + assert_eq!(result.commands.first().unwrap().arguments.payload, "line one\\\nline two"); } #[test] @@ -193,10 +192,7 @@ fn text_block_with_continuation_preserves_backslash() { // because text blocks capture pre-join content for round-trip fidelity. let result = parse_document("hello \\\nworld"); assert_eq!(result.textblocks.len(), 1); - assert_eq!( - result.textblocks.first().unwrap().content, - "hello \\\nworld" - ); + assert_eq!(result.textblocks.first().unwrap().content, "hello \\\nworld"); } #[test] @@ -240,20 +236,11 @@ fn spec_a2_joined_multi_line_command() { assert_eq!(cmd.name, "deploy"); assert_eq!(cmd.range.start_line, 0); assert_eq!(cmd.range.end_line, 2); - assert_eq!( - cmd.arguments.header, - "production --region us-west-2 --canary" - ); + assert_eq!(cmd.arguments.header, "production --region us-west-2 --canary"); assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); - assert_eq!( - cmd.arguments.payload, - "production --region us-west-2 --canary" - ); + assert_eq!(cmd.arguments.payload, "production --region us-west-2 --canary"); // §8.2: raw contains physical lines with backslashes and \n separators. - assert_eq!( - cmd.raw, - "/deploy production \\\n --region us-west-2 \\\n --canary" - ); + assert_eq!(cmd.raw, "/deploy production \\\n --region us-west-2 \\\n --canary"); } #[test] @@ -289,10 +276,7 @@ fn spec_a4_backslash_join_into_fence() { assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); assert_eq!(cmd.arguments.payload, "{\"path\": \"foo\"}"); // §8.2: raw includes all physical lines from opener through closer. - assert_eq!( - cmd.raw, - "/mcp call_tool write_file \\\n```json\n{\"path\": \"foo\"}\n```" - ); + assert_eq!(cmd.raw, "/mcp call_tool write_file \\\n```json\n{\"path\": \"foo\"}\n```"); } #[test] @@ -305,33 +289,21 @@ fn spec_a5_text_blocks_and_multiple_commands() { assert_eq!(result.commands.len(), 2); assert_eq!(result.commands.first().unwrap().id, "cmd-0"); assert_eq!(result.commands.first().unwrap().name, "deploy"); - assert_eq!( - result.commands.first().unwrap().arguments.payload, - "staging" - ); + assert_eq!(result.commands.first().unwrap().arguments.payload, "staging"); assert_eq!(result.commands.first().unwrap().range.start_line, 2); assert_eq!(result.commands.get(1).unwrap().id, "cmd-1"); assert_eq!(result.commands.get(1).unwrap().name, "notify"); - assert_eq!( - result.commands.get(1).unwrap().arguments.payload, - "team --channel ops" - ); + assert_eq!(result.commands.get(1).unwrap().arguments.payload, "team --channel ops"); assert_eq!(result.commands.get(1).unwrap().range.start_line, 3); // Two text blocks in order. assert_eq!(result.textblocks.len(), 2); assert_eq!(result.textblocks.first().unwrap().id, "text-0"); - assert_eq!( - result.textblocks.first().unwrap().content, - "Welcome to the deployment system.\n" - ); + assert_eq!(result.textblocks.first().unwrap().content, "Welcome to the deployment system.\n"); assert_eq!(result.textblocks.first().unwrap().range.start_line, 0); assert_eq!(result.textblocks.first().unwrap().range.end_line, 1); assert_eq!(result.textblocks.get(1).unwrap().id, "text-1"); - assert_eq!( - result.textblocks.get(1).unwrap().content, - "Deployment complete." - ); + assert_eq!(result.textblocks.get(1).unwrap().content, "Deployment complete."); assert_eq!(result.textblocks.get(1).unwrap().range.start_line, 4); assert_eq!(result.textblocks.get(1).unwrap().range.end_line, 4); } @@ -361,8 +333,7 @@ fn spec_a7_unclosed_fence() { fn spec_a8_closing_fence_with_trailing_backslash() { // Appendix A.8: "```\" is NOT a valid closer (not solely backticks after trim). // The fence never closes; all remaining lines become payload. Warning emitted. - let input = - "/mcp call_tool write_file -c \\\n```json\n{\"path\": \"foo\"}\n```\\\n\\\nproduction"; + let input = "/mcp call_tool write_file -c \\\n```json\n{\"path\": \"foo\"}\n```\\\n\\\nproduction"; let result = parse_document(input); assert_eq!(result.commands.len(), 1); let cmd = result.commands.first().unwrap(); @@ -379,8 +350,7 @@ fn spec_a8_closing_fence_with_trailing_backslash() { #[test] fn spec_a9_proper_fence_close_followed_by_content() { // Appendix A.9: fence closes on line 3, lines 4-5 join into text block. - let input = - "/mcp call_tool write_file -c \\\n```json\n{\"path\": \"foo\"}\n```\n\\\nproduction"; + let input = "/mcp call_tool write_file -c \\\n```json\n{\"path\": \"foo\"}\n```\n\\\nproduction"; let result = parse_document(input); // One command with closed fence. @@ -419,7 +389,7 @@ proptest! { fn version_is_always_spec_version(input in "\\PC{0,200}") { // §8.1: version field is always SPEC_VERSION regardless of input. let result = parse_document(&input); - prop_assert_eq!(&result.version, crate::domain::SPEC_VERSION); + prop_assert_eq!(&result.version, crate::SPEC_VERSION); } #[test] diff --git a/parser/src/application/tests/proptest.rs b/parser/src/integration_tests/proptest.rs similarity index 53% rename from parser/src/application/tests/proptest.rs rename to parser/src/integration_tests/proptest.rs index c994daa..8322e66 100644 --- a/parser/src/application/tests/proptest.rs +++ b/parser/src/integration_tests/proptest.rs @@ -1,13 +1,12 @@ use proptest::prelude::*; use crate::{ - application::{ - command_accumulate::{accept_line, start_command}, - command_finalize::finalize_command, - line_classify::{LineKind, classify_line}, - text_collect::{append_text, finalize_text, start_text}, + domain::{ArgumentMode, LineRange, SPEC_VERSION}, + engine::{ + classify::{LineKind, classify_line}, + fence::{accept_fence_line, finalize_fence, open_fence}, + text::{append_text, finalize_text, start_text}, }, - domain::ArgumentMode, }; fn valid_command_name() -> impl Strategy { @@ -15,10 +14,9 @@ fn valid_command_name() -> impl Strategy { } proptest! { - /// classify -> accumulate -> finalize roundtrip never panics. #[test] #[cfg_attr(feature = "tdd", ignore)] - fn classify_accumulate_finalize_roundtrip( + fn classify_open_accept_finalize_roundtrip( name in valid_command_name(), body_lines in prop::collection::vec("[a-zA-Z0-9]{1,30}", 0..8) ) { @@ -27,22 +25,20 @@ proptest! { LineKind::Command(h) => h, LineKind::Text => panic!("expected command"), }; - - let cmd = start_command(header, 0); - - let cmd = body_lines.iter().enumerate().fold(cmd, |cmd, (i, line)| { - let (next, _) = accept_line(cmd, i + 1, line); + let raw = header.raw.clone(); + let range = LineRange { start_line: 0, end_line: 0 }; + let fence = open_fence(header, raw, 0, range); + let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { + let (next, _) = accept_fence_line(f, i + 1, line); next }); - let (cmd, _) = accept_line(cmd, body_lines.len() + 1, "```"); - - let finalized = finalize_command(cmd); - prop_assert_eq!(finalized.command.name, name); - prop_assert_eq!(finalized.command.arguments.mode, ArgumentMode::Fence); - prop_assert!(finalized.warnings.is_empty()); + let (fence, _) = accept_fence_line(fence, body_lines.len() + 1, "```"); + let (cmd, warnings) = finalize_fence(fence, false); + prop_assert_eq!(cmd.name, name); + prop_assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + prop_assert!(warnings.is_empty()); } - /// Text lines classified as Text always produce valid text blocks. #[test] #[cfg_attr(feature = "tdd", ignore)] fn text_lines_through_classify_and_collect( @@ -51,12 +47,11 @@ proptest! { for line in &lines { prop_assert!(matches!(classify_line(line), LineKind::Text)); } - let pending = lines.iter().enumerate().skip(1).fold( start_text(0, &lines[0]), |text, (i, line)| append_text(text, i, line), ); - let block = finalize_text(pending); + let block = finalize_text(pending, 0); prop_assert_eq!(block.content, lines.join("\n")); } } diff --git a/parser/src/join.rs b/parser/src/join.rs new file mode 100644 index 0000000..c47174f --- /dev/null +++ b/parser/src/join.rs @@ -0,0 +1,432 @@ +/// POSIX-style backslash line joining. +/// +/// Consumes physical lines and produces logical lines. A physical line ending +/// with `\` is joined with the next physical line, separated by a single space, +/// and the backslash is removed. Joining repeats while the accumulated line +/// still ends with `\`. At EOF, a trailing `\` is silently removed. +/// +/// Fence immunity is enforced by the caller: when the state machine enters +/// `InFence`, it calls `next_physical` directly instead of `next_logical`, +/// bypassing the joiner for those lines. +pub struct LogicalLine { + pub text: String, + pub first_physical: usize, + pub last_physical: usize, +} + +pub struct LineJoiner { + lines: Vec, + cursor: usize, +} + +/// Consume the next logical line from `lines` starting at `*cursor`, +/// joining any trailing-backslash continuations. +fn consume_logical(lines: &[String], cursor: &mut usize) -> Option { + if *cursor >= lines.len() { + return None; + } + + let first_physical = *cursor; + let mut text = lines[*cursor].clone(); + *cursor += 1; + + while text.ends_with('\\') { + text.truncate(text.len() - 1); + + if *cursor >= lines.len() { + // Trailing backslash at EOF: silently removed, line stands alone. + break; + } + + // text.push(' '); + text.push_str(&lines[*cursor]); + *cursor += 1; + } + + Some(LogicalLine { text, first_physical, last_physical: *cursor - 1 }) +} + +/// Consume the next raw physical line from `lines` at `*cursor`, +/// bypassing join logic entirely. +fn consume_physical(lines: &[String], cursor: &mut usize) -> Option<(usize, String)> { + if *cursor >= lines.len() { + return None; + } + + let idx = *cursor; + let line = lines[*cursor].clone(); + *cursor += 1; + Some((idx, line)) +} + +impl LineJoiner { + pub fn new(lines: Vec) -> Self { Self { lines, cursor: 0 } } + + pub fn next_logical(&mut self) -> Option { consume_logical(&self.lines, &mut self.cursor) } + + /// Used by the state machine when inside a fenced block. + pub fn next_physical(&mut self) -> Option<(usize, String)> { + consume_physical(&self.lines, &mut self.cursor) + } + + // Implemented but does not look like it's needed by the pipeline. + // TODO: remove if still dead when parser is finished + #[allow(dead_code)] + pub fn is_exhausted(&self) -> bool { self.cursor >= self.lines.len() } +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + + fn lines(xs: &[&str]) -> Vec { xs.iter().map(|s| s.to_string()).collect() } + + fn joiner(xs: &[&str]) -> LineJoiner { LineJoiner::new(lines(xs)) } + + /// Build a string ending with a backslash without embedding `\\` in literals. + fn bsl(s: &str) -> String { + let mut r = s.to_string(); + r.push('\\'); + r + } + + // ========================================================================= + // consume_logical — no joining (pass-through) + // RFC §3.2: "Lines that do not end with '\' are left unchanged." + // ========================================================================= + + #[test] + fn consume_logical_empty_returns_none() { + // Structural: no input -> None, cursor unchanged. + let ls = lines(&[]); + let mut cursor = 0; + assert!(consume_logical(&ls, &mut cursor).is_none()); + assert_eq!(cursor, 0); + } + + #[test] + fn consume_logical_no_backslash_passes_through() { + // RFC §3.2: line without trailing '\' is unchanged. + let ls = lines(&["/echo hello"]); + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "/echo hello"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 0); + assert_eq!(cursor, 1); + } + + // ========================================================================= + // consume_logical — joining (backslash continuation) + // RFC §3.2 steps 1-3 / Engine Spec §7.1 + // ========================================================================= + + #[test] + fn consume_logical_joins_two_lines() { + // RFC §3.2 steps 1-3: remove trailing '\', remove LF boundary, + // concatenate with next line separated by a single SPACE. + // + // BUG: RFC v1.1.0 §3.2 step 3 says "Concatenate the remainder with + // the next physical line, separated by a single U+0020 (SPACE)." + // However Engine Spec v0.5.0 §7.1 says "No separator character is + // inserted" (true POSIX). The code inserts a space, matching the RFC + // but contradicting the Engine Spec. See gaps comment. + let ls = vec![bsl("a"), "b".to_string()]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "ab"); + assert_eq!(ll.last_physical, 1); + assert_eq!(cursor, 2); + } + + #[test] + fn consume_logical_chains_three_lines() { + // RFC §3.2 step 4: "If the result still ends with '\', repeat." + let ls = vec![bsl("a"), bsl("b"), "c".to_string()]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "abc"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 2); + assert_eq!(cursor, 3); + } + + // ========================================================================= + // consume_logical — trailing backslash at EOF + // RFC §3.2: "the trailing '\' is removed and the line stands alone." + // ========================================================================= + + #[test] + fn consume_logical_trailing_backslash_at_eof_removed() { + // RFC §3.2: final line ends with '\', no subsequent line -> '\' removed. + let ls = vec![bsl("a")]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "a"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 0); + } + + // ========================================================================= + // consume_logical — cursor advancement + // Engine Spec §7.3 / RFC §3.3 + // ========================================================================= + + #[test] + fn consume_logical_advances_cursor_past_joined_lines() { + // RFC §3.3: each logical line maps to one or more physical lines. + // Cursor must advance past all consumed physical lines. + let ls = vec![bsl("x"), "y".to_string(), "z".to_string()]; + let mut cursor = 0; + consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(cursor, 2); + let ll2 = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll2.text, "z"); + } + + // ========================================================================= + // consume_physical — raw line access (fence body support) + // RFC §3.2 / §5.2.2 / Engine Spec §7.2 + // ========================================================================= + + #[test] + fn consume_physical_empty_returns_none() { + // Structural: no input -> None, cursor unchanged. + let ls = lines(&[]); + let mut cursor = 0; + assert!(consume_physical(&ls, &mut cursor).is_none()); + assert_eq!(cursor, 0); + } + + #[test] + fn consume_physical_returns_raw_line_including_backslash() { + // RFC §5.2.2: "preserving their original content including any + // trailing backslashes." Engine Spec §7.2: next_physical bypasses + // join logic. + let ls = vec![bsl("line one"), "line two".to_string()]; + let mut cursor = 0; + let (idx, line) = consume_physical(&ls, &mut cursor).unwrap(); + assert_eq!(idx, 0); + assert!(line.ends_with('\\')); + assert_eq!(cursor, 1); + } + + #[test] + fn consume_physical_does_not_join() { + // RFC §5.2.2: "Fence body lines are never subject to line joining." + let ls = vec![bsl("a"), "b".to_string()]; + let mut cursor = 0; + let (_, line) = consume_physical(&ls, &mut cursor).unwrap(); + assert!(line.ends_with('\\')); + let (_, line2) = consume_physical(&ls, &mut cursor).unwrap(); + assert_eq!(line2, "b"); + } + + // ========================================================================= + // LineJoiner — delegation and shared cursor + // Engine Spec §5.3 / Engine Spec §7.2 + // ========================================================================= + + #[test] + fn next_logical_delegates() { + // Structural: next_logical delegates to consume_logical. + let mut j = joiner(&["/echo hello"]); + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "/echo hello"); + assert!(j.is_exhausted()); + } + + #[test] + fn next_physical_delegates() { + // Structural: next_physical delegates to consume_physical. + let input = vec![bsl("line one"), "line two".to_string()]; + let mut j = LineJoiner::new(input); + let (idx, line) = j.next_physical().unwrap(); + assert_eq!(idx, 0); + assert!(line.ends_with('\\')); + } + + #[test] + fn interleaving_logical_and_physical_shares_cursor() { + // Engine Spec §5.3: idle state uses next_logical, in-fence uses + // next_physical. Both share a cursor so no lines are skipped at + // the transition boundary. + let input = vec![bsl("/cmd a"), " b".to_string(), "fence body".to_string()]; + let mut j = LineJoiner::new(input); + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "/cmd a b"); + assert_eq!(ll.last_physical, 1); + let (idx, line) = j.next_physical().unwrap(); + assert_eq!(idx, 2); + assert_eq!(line, "fence body"); + assert!(j.is_exhausted()); + } + + #[test] + fn is_exhausted_tracks_cursor() { + // Structural: is_exhausted reflects cursor vs line count. + let mut j = joiner(&["a", "b"]); + assert!(!j.is_exhausted()); + j.next_logical(); + assert!(!j.is_exhausted()); + j.next_logical(); + assert!(j.is_exhausted()); + } + + // ========================================================================= + // Spec examples from RFC Appendix B + // ========================================================================= + + #[test] + fn appendix_b2_three_physical_lines_join() { + // RFC Appendix B.2: three physical lines joined into one logical line. + // Engine Spec §7.3: first_physical and last_physical track the range. + let input = vec![ + bsl("/mcp call_tool read_file"), + bsl(" --path src/index.ts"), + " --format json".to_string(), + ]; + let mut j = LineJoiner::new(input); + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "/mcp call_tool read_file --path src/index.ts --format json"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 2); + } + + #[test] + fn trailing_backslash_at_eof_preserves_preceding_space() { + // RFC §3.2: backslash removed but preceding content (including space) + // is preserved. + let input = vec![bsl("/echo hello ")]; + let mut j = LineJoiner::new(input); + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "/echo hello "); + } + + #[test] + fn fence_closer_with_trailing_backslash_joins_in_idle() { + // RFC §3.2: "The join marker is any '\' immediately before the line + // boundary, regardless of what precedes it." + // NOTE: this test exercises the joiner in isolation. In the real + // pipeline the state machine would call next_physical for this line + // (since it's inside a fence), so the join would not occur. + let input = vec![bsl("```"), "next line".to_string()]; + let mut j = LineJoiner::new(input); + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "```next line"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 1); + } + + // ========================================================================= + // Property tests + // ========================================================================= + + proptest! { + // RFC §3.2: joining only merges lines, never creates new ones. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn logical_count_lte_physical_count( + ls in prop::collection::vec("[a-zA-Z0-9 ]{0,40}", 0..20) + ) { + let count = ls.len(); + let mut cursor = 0; + let mut logical_count = 0; + while consume_logical(&ls, &mut cursor).is_some() { + logical_count += 1; + } + prop_assert!(logical_count <= count); + } + + // RFC §3.2: lines without '\' pass through unchanged. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn clean_lines_pass_through_unchanged( + ls in prop::collection::vec("[a-zA-Z0-9 !.,]{1,40}", 1..10) + ) { + let expected = ls.clone(); + let mut cursor = 0; + for expected_text in expected { + let ll = consume_logical(&ls, &mut cursor).unwrap(); + prop_assert_eq!(ll.text, expected_text); + } + } + + // RFC §3.3: logical lines must partition physical lines without gaps. + // Engine Spec §7.3: first_physical/last_physical cover the range. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn ranges_cover_all_physical_lines( + ls in prop::collection::vec("[a-zA-Z0-9 ]{0,40}", 1..20) + ) { + let count = ls.len(); + let mut cursor = 0; + let mut next_expected = 0usize; + while let Some(ll) = consume_logical(&ls, &mut cursor) { + prop_assert_eq!(ll.first_physical, next_expected); + prop_assert!(ll.first_physical <= ll.last_physical); + next_expected = ll.last_physical + 1; + } + prop_assert_eq!(next_expected, count); + } + + // Structural: is_exhausted true after consuming all lines. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn exhausted_after_consuming_all_logical_lines( + ls in prop::collection::vec("[a-zA-Z0-9 ]{0,40}", 0..20) + ) { + let mut j = LineJoiner::new(ls); + while j.next_logical().is_some() {} + prop_assert!(j.is_exhausted()); + } + } +} + +// ============================================================================= +// TEST GAPS: spec areas this file's functions touch but are not tested +// ============================================================================= +// +// | Spec Section | Gap | Severity | +// |-----------------------------------------|----------------------------------------------|----------| +// | Engine Spec §7.1 vs RFC §3.2 step 3 | SPACE INSERTION CONFLICT: The code inserts | CRITICAL | +// | | a space between joined lines (text.push(' ')) | | +// | | matching RFC §3.2 step 3. But Engine Spec | | +// | | v0.5.0 §7.1 says "No separator character is | | +// | | inserted. This is true POSIX backslash- | | +// | | newline removal." and §15.1 migration notes | | +// | | explicitly changed this from v0.3.0. | | +// | | THE RFC AND ENGINE SPEC CONTRADICT EACH | | +// | | OTHER. Must resolve which is authoritative. | | +// |-----------------------------------------------------------------------|----------| +// | RFC §3.2 | MULTI-BACKSLASH: No test for a line ending | MEDIUM | +// | | with multiple backslashes (e.g., "foo\\\\"). | | +// | | Two backslashes: the last '\' is the join | | +// | | marker, the preceding '\' is content. Only | | +// | | the final '\' should be removed. | | +// |-----------------------------------------------------------------------|----------| +// | RFC §3.2 | EMPTY LINE WITH BACKSLASH: No test for a | LOW | +// | | line that is just "\" (backslash only). Should | | +// | | join with next line, producing just the next | | +// | | line's content (with or without leading space | | +// | | depending on space-insertion resolution). | | +// |-----------------------------------------------------------------------|----------| +// | RFC §3.3 | PHYSICAL LINE TRACKING FOR TEXT BLOCKS: No | LOW | +// | | test verifying that a joined text line's | | +// | | constituent physical lines (with backslashes) | | +// | | are preserved for text block raw content | | +// | | (Engine Spec §10). This is the text_collect | | +// | | module's job, but the LogicalLine range data | | +// | | is what enables it. | | +// |-----------------------------------------------------------------------|----------| +// | RFC §3.2 | WHITESPACE-ONLY CONTINUATION LINE: No test | LOW | +// | | for joining where the continuation line is | | +// | | only whitespace (e.g., "cmd\\" + " "). | | +// | | Verifies no trimming occurs. | | +// |-----------------------------------------------------------------------|----------| +// | Engine Spec §7.1 | NO-SEPARATOR JOIN ASSERTION: If the Engine | LOW | +// | | Spec is authoritative (no space), there is no | | +// | | test asserting direct concatenation without | | +// | | space. The current tests all assert WITH space.| | diff --git a/parser/src/lib.rs b/parser/src/lib.rs index 3cd24c2..c35b783 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -1,11 +1,19 @@ -mod application; -mod domain; +// Internal modules +mod classify; +mod fence; +mod join; +mod normalize; +mod parse; +mod single_line; +mod text; +mod types; -// Domain types -pub use domain::{ - ArgumentMode, Command, CommandArguments, LineRange, ParseResult, SPEC_VERSION, TextBlock, - Warning, +// Public API +pub use parse::parse_document; +pub use types::{ + ArgumentMode, Command, CommandArguments, LineRange, + ParseResult, SPEC_VERSION, TextBlock, Warning, }; -// Engine entry point -pub use application::parse_document; +#[cfg(test)] +mod integration_tests; diff --git a/parser/src/mod.rs b/parser/src/mod.rs new file mode 100644 index 0000000..2ddefae --- /dev/null +++ b/parser/src/mod.rs @@ -0,0 +1,12 @@ +mod classify; +mod fence; +mod join; +mod normalize; +mod parse; +mod single_line; +mod text; + +pub use parse::parse_document; + +#[cfg(test)] +pub mod tests; diff --git a/parser/src/normalize.rs b/parser/src/normalize.rs new file mode 100644 index 0000000..565faac --- /dev/null +++ b/parser/src/normalize.rs @@ -0,0 +1,189 @@ +/// Normalize line endings in `input` to LF only. +/// +/// Step 1: replace all CRLF (`\r\n`) with LF (`\n`). +/// Step 2: replace any remaining bare CR (`\r`) with LF (`\n`). +/// +/// All other bytes, including literal `\n` escape sequences inside content, +/// are preserved verbatim. +pub fn normalize(input: &str) -> String { input.replace("\r\n", "\n").replace('\r', "\n") } + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + + // ========================================================================= + // No-op cases (input already normalized or empty) + // RFC §3.1 / Engine Spec §5.1 + // ========================================================================= + + #[test] + fn empty_input_is_unchanged() { + // RFC §3.1 (implied): normalize is a pure function; empty -> empty. + // Engine Spec §5.1: "pure string transformation with no state." + assert_eq!(normalize(""), ""); + } + + #[test] + fn lf_only_input_is_unchanged() { + // RFC §3.1: LF is the normalized form; LF-only input needs no changes. + let input = "line one\nline two\nline three"; + assert_eq!(normalize(input), input); + } + + #[test] + fn no_line_endings_unchanged() { + // RFC §3.1: content with no CR or LF is unaffected. + let input = "no line endings here"; + assert_eq!(normalize(input), input); + } + + // ========================================================================= + // CRLF replacement (step 1) + // RFC §3.1 step 1: "Replace all CRLF (U+000D U+000A) sequences with LF." + // Engine Spec §5.1 step 1. + // ========================================================================= + + #[test] + fn crlf_becomes_lf() { + // RFC §3.1 step 1. + assert_eq!(normalize("a\r\nb"), "a\nb"); + } + + #[test] + fn multiple_crlf_all_converted() { + // RFC §3.1 step 1: all occurrences replaced, not just the first. + assert_eq!(normalize("a\r\nb\r\nc"), "a\nb\nc"); + } + + #[test] + fn crlf_at_boundaries() { + // RFC §3.1 step 1: replacement applies at start and end of input. + assert_eq!(normalize("\r\nhello\r\n"), "\nhello\n"); + } + + // ========================================================================= + // Bare CR replacement (step 2) + // RFC §3.1 step 2: "Replace all remaining bare CR (U+000D) with LF." + // Engine Spec §5.1 step 2. + // ========================================================================= + + #[test] + fn bare_cr_becomes_lf() { + // RFC §3.1 step 2. + assert_eq!(normalize("a\rb"), "a\nb"); + } + + #[test] + fn consecutive_bare_cr() { + // RFC §3.1 step 2: each bare CR individually becomes LF. + assert_eq!(normalize("\r\r\r"), "\n\n\n"); + } + + // ========================================================================= + // Ordering: CRLF before bare CR (steps 1 then 2) + // RFC §3.1: step 1 runs before step 2 to avoid double-conversion. + // ========================================================================= + + #[test] + fn mixed_endings_no_double_conversion() { + // RFC §3.1 steps 1-2: CRLF replaced first, so the CR in \r\n is not + // re-matched as bare CR (which would produce \n\n). + assert_eq!(normalize("a\r\nb\rc\nd"), "a\nb\nc\nd"); + } + + // ========================================================================= + // Literal escape sequences preserved + // RFC §3.1: "Literal escape sequences inside content (e.g., the + // two-character sequence '\' 'n') are ordinary characters." + // ========================================================================= + + #[test] + fn literal_backslash_n_preserved() { + // RFC §3.1: two-char sequence backslash + 'n' is not a line terminator. + let input = "before\\nafter"; + assert_eq!(normalize(input), "before\\nafter"); + } + + // ========================================================================= + // Property tests + // ========================================================================= + + proptest! { + // RFC §3.1: "After normalization, all line terminators are LF." + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn output_never_contains_cr(input in "[\\x00-\\x7F]{0,500}") { + let result = normalize(&input); + prop_assert!(!result.contains('\r')); + } + + // RFC §3.1 (implied): normalizing already-normalized output is a no-op. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn idempotent(input in "[\\x00-\\x7F]{0,500}") { + let once = normalize(&input); + let twice = normalize(&once); + prop_assert_eq!(once, twice); + } + + // RFC §3.1: input containing no CR requires no changes. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn clean_input_is_unchanged(input in "[\\x20-\\x7E\\n]{0,500}") { + prop_assert_eq!(normalize(&input), input); + } + + // RFC §3.1 step 2: each bare CR becomes LF, so LF count can only + // stay the same or increase. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn lf_count_gte_original_lf_count(input in "[\\x00-\\x7F]{0,500}") { + let original_lf = input.chars().filter(|&c| c == '\n').count(); + let result = normalize(&input); + let result_lf = result.chars().filter(|&c| c == '\n').count(); + prop_assert!(result_lf >= original_lf); + } + } +} + +// ============================================================================= +// TEST GAPS: spec areas this file's functions touch but are not tested +// ============================================================================= +// +// | Spec Section | Gap | Severity | +// |---------------------------------|-------------------------------------------------|----------| +// | RFC §3.1 | SPLIT_LINES: The spec says "The normalized | HIGH | +// | | input is split on LF to produce a sequence of | | +// | | physical lines." Engine Spec §5.2 defines this | | +// | | as a separate stage. This module only covers | | +// | | normalization, not splitting. If split_lines | | +// | | lives elsewhere, that's fine, but if it's | | +// | | missing entirely it's a gap. | | +// |---------------------------------|-------------------------------------------------|----------| +// | RFC §3.1 | TRAILING LF: "A trailing LF at the end of | MEDIUM | +// | | input produces a trailing empty line." No test | | +// | | verifies that normalize preserves a trailing | | +// | | \n (e.g., "hello\n" -> "hello\n"). This is | | +// | | trivially true for LF input but should be | | +// | | verified for trailing \r and trailing \r\n. | | +// |---------------------------------|-------------------------------------------------|----------| +// | RFC §3.1 | UNICODE / NON-ASCII: All property tests use | LOW | +// | | ASCII [\x00-\x7F]. No test verifies that | | +// | | multi-byte UTF-8 content (e.g., emoji, CJK) | | +// | | passes through normalization unmodified. The | | +// | | implementation (.replace) handles this | | +// | | correctly, but a test would guard regressions. | | +// |---------------------------------|-------------------------------------------------|----------| +// | RFC §3.1 | LENGTH INVARIANT: normalize can only shrink | LOW | +// | | (CRLF -> LF removes one byte) or preserve | | +// | | length. No property test asserts | | +// | | result.len() <= input.len(). | | +// |---------------------------------|-------------------------------------------------|----------| +// | Engine Spec §5.1 | PURE FUNCTION: "This stage is a pure string | INFO | +// | | transformation with no state." No test verifies | | +// | | that calling normalize multiple times on | | +// | | different inputs doesn't leak state (trivially | | +// | | true for a free function, but documenting the | | +// | | intent is useful). | | diff --git a/parser/src/parse.rs b/parser/src/parse.rs new file mode 100644 index 0000000..6908a08 --- /dev/null +++ b/parser/src/parse.rs @@ -0,0 +1,544 @@ +use super::{ + classify::{CommandHeader, LineKind, classify_line}, + fence::{FenceResult, PendingFence, accept_fence_line, finalize_fence, open_fence}, + join::LineJoiner, + normalize::normalize, + single_line::finalize_single_line, + text::{PendingText, append_text, finalize_text, start_text}, +}; +use crate::{ + LineRange, + ArgumentMode, Command, ParseResult, SPEC_VERSION, TextBlock, Warning, +}; + +// --- Types --- + +#[derive(Debug)] +enum ParserState { + Idle, + InFence(PendingFence), +} + +#[derive(PartialEq, Eq)] +enum LoopAction { + Continue, + Break, +} + +struct ParseCtx { + state: ParserState, + current_text: Option, + commands: Vec, + textblocks: Vec, + warnings: Vec, + cmd_seq: usize, + text_seq: usize, +} + +impl ParseCtx { + fn new() -> Self { + Self { + state: ParserState::Idle, + current_text: None, + commands: Vec::new(), + textblocks: Vec::new(), + warnings: Vec::new(), + cmd_seq: 0, + text_seq: 0, + } + } + + fn into_result(self) -> ParseResult { + ParseResult { + version: SPEC_VERSION.to_owned(), + commands: self.commands, + textblocks: self.textblocks, + warnings: self.warnings, + } + } +} + +// --- Public entry point --- + +pub fn parse_document(input: &str) -> ParseResult { + let normalized = normalize(input); + let physical_lines = split_physical_lines(&normalized); + let owned: Vec = physical_lines.iter().map(|s| s.to_string()).collect(); + let mut joiner = LineJoiner::new(owned); + let mut ctx = ParseCtx::new(); + + while step(&mut ctx, &mut joiner, &physical_lines) == LoopAction::Continue {} + + flush_text(&mut ctx); + ctx.into_result() +} + +// --- Pipeline --- + +fn split_physical_lines(normalized: &str) -> Vec<&str> { + let mut lines: Vec<&str> = normalized.split('\n').collect(); + if lines.last() == Some(&"") { + lines.pop(); + } + lines +} + +fn step(ctx: &mut ParseCtx, joiner: &mut LineJoiner, phys: &[&str]) -> LoopAction { + let state = std::mem::replace(&mut ctx.state, ParserState::Idle); + match state { + ParserState::Idle => step_idle(ctx, joiner, phys), + ParserState::InFence(cmd) => step_in_fence(ctx, joiner, cmd), + } +} + +// --- State handlers --- + +fn step_idle(ctx: &mut ParseCtx, joiner: &mut LineJoiner, phys: &[&str]) -> LoopAction { + let Some(ll) = joiner.next_logical() else { + return LoopAction::Break; + }; + match classify_line(&ll.text) { + LineKind::Command(mut header) => { + header.raw = phys[ll.first_physical..=ll.last_physical].join("\n"); + flush_text(ctx); + start_new_command(ctx, header, ll.first_physical, ll.last_physical); + } + LineKind::Text => { + accumulate_text(ctx, ll.first_physical, ll.last_physical, phys); + } + } + LoopAction::Continue +} + +fn step_in_fence(ctx: &mut ParseCtx, joiner: &mut LineJoiner, fence: PendingFence) -> LoopAction { + let Some((line_idx, line)) = joiner.next_physical() else { + let (cmd, warnings) = finalize_fence(fence, true); + ctx.commands.push(cmd); + ctx.warnings.extend(warnings); + return LoopAction::Break; + }; + let (updated, result) = accept_fence_line(fence, line_idx, &line); + match result { + FenceResult::Consumed => { + ctx.state = ParserState::InFence(updated); + } + FenceResult::Completed => { + let (cmd, warnings) = finalize_fence(updated, false); + ctx.commands.push(cmd); + ctx.warnings.extend(warnings); + } + } + LoopAction::Continue +} + +// --- Context helpers --- + +fn start_new_command(ctx: &mut ParseCtx, header: CommandHeader, first_physical: usize, last_physical: usize) { + match header.mode { + ArgumentMode::SingleLine => { + let raw = header.raw.clone(); + let range = LineRange { start_line: first_physical, end_line: last_physical }; + let cmd = finalize_single_line(header, raw, ctx.cmd_seq, range); + ctx.commands.push(cmd); + ctx.cmd_seq += 1; + } + ArgumentMode::Fence => { + let raw = header.raw.clone(); + let range = LineRange { start_line: first_physical, end_line: last_physical }; + let fence = open_fence(header, raw, ctx.cmd_seq, range); + ctx.cmd_seq += 1; + ctx.state = ParserState::InFence(fence); + } + } +} + +fn flush_text(ctx: &mut ParseCtx) { + let Some(text) = ctx.current_text.take() else { + return; + }; + ctx.textblocks.push(finalize_text(text, ctx.text_seq)); + ctx.text_seq += 1; +} + +fn accumulate_text(ctx: &mut ParseCtx, first: usize, last: usize, phys: &[&str]) { + let text = match ctx.current_text.take() { + Some(existing) => fold_physical_lines(existing, first, last, phys), + None => { + let started = start_text(first, phys[first]); + fold_physical_lines(started, first + 1, last, phys) + } + }; + ctx.current_text = Some(text); +} + +fn fold_physical_lines(mut text: PendingText, from: usize, to: usize, phys: &[&str]) -> PendingText { + for (idx, line) in (from..=to).zip(&phys[from..=to]) { + text = append_text(text, idx, line); + } + text +} + +#[cfg(test)] +mod tests { + use super::parse_document; + // NOTE: All imports below are from crate::domain (external to this file). + // parse_document orchestrates: normalize, split_physical_lines, LineJoiner, + // classify_line, finalize_single_line, open_fence, accept_fence_line, + // finalize_fence, start_text, append_text, finalize_text. + use crate::{ArgumentMode, SPEC_VERSION}; + + // ========================================================================= + // Empty / trivial input + // RFC §8.2 item 4 / Engine Spec §4.2 + // ========================================================================= + + #[test] + fn empty_input() { + // RFC §8.2 item 4: "An empty input produces a result with no commands, + // no text blocks, and no warnings." + // Engine Spec §4.2: total function guarantee. + let r = parse_document(""); + assert!(r.commands.is_empty()); + assert!(r.textblocks.is_empty()); + assert!(r.warnings.is_empty()); + } + + #[test] + fn whitespace_only_is_text() { + // RFC §6.3: "A text line is any non-fence-body logical line that is + // not a command line." Whitespace-only lines are text. + let r = parse_document(" "); + assert!(r.commands.is_empty()); + assert_eq!(r.textblocks.len(), 1); + } + + // ========================================================================= + // Single-line command — end-to-end threading + // RFC §5.1 / Engine Spec §8 / Engine Spec §9.3 + // ========================================================================= + + #[test] + fn single_line_command_fields() { + // RFC §5.1: single-line mode, header == payload. + // RFC §6.5: id is cmd-0. + // RFC §7.1: name, raw, range, mode, payload. + let r = parse_document("/deploy production --region us-west-2"); + assert_eq!(r.commands.len(), 1); + let cmd = &r.commands[0]; + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "deploy"); + assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); + assert_eq!(cmd.arguments.payload, "production --region us-west-2"); + assert_eq!(cmd.arguments.header, "production --region us-west-2"); + assert_eq!(cmd.raw, "/deploy production --region us-west-2"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 0); + } + + #[test] + fn single_line_no_args() { + // RFC §4.3: "The arguments portion may be empty." + // RFC §5.1: empty args -> empty header and payload. + let r = parse_document("/ping"); + let cmd = &r.commands[0]; + assert_eq!(cmd.arguments.header, ""); + assert_eq!(cmd.arguments.payload, ""); + } + + // ========================================================================= + // Fenced command — end-to-end threading + // RFC §5.2 / Engine Spec §9 + // ========================================================================= + + #[test] + fn fenced_command_fields() { + // RFC §5.2.2: body lines joined with LF. + // RFC §5.2.1: fence_lang from opener. + // RFC §7.1: raw includes opener, body, closer. + // Engine Spec §3.6: range is inclusive physical lines. + let r = parse_document("/cmd ```json\nline one\nline two\n```"); + assert_eq!(r.commands.len(), 1); + let cmd = &r.commands[0]; + assert_eq!(cmd.arguments.payload, "line one\nline two"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.raw, "/cmd ```json\nline one\nline two\n```"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 3); + } + + // ========================================================================= + // Unclosed fence + // RFC §5.2.4 / Engine Spec §9.3 step 4 + // ========================================================================= + + #[test] + fn unclosed_fence_warning_and_partial_command() { + // RFC §5.2.4: "A warning of type unclosed_fence MUST be produced." + // RFC §5.2.4: "The command is complete with whatever payload has been + // accumulated through EOF." + // + // NOTE: wtype is "unclosed-fence" (kebab-case). Engine Spec §11 and + // RFC §7.4 require "unclosed_fence" (snake_case). This is a known + // code bug inherited from fence.rs finalize_fence. + let r = parse_document("/cmd ```\npartial body"); + assert_eq!(r.commands.len(), 1); + assert_eq!(r.commands[0].arguments.payload, "partial body"); + assert_eq!(r.warnings.len(), 1); + assert_eq!(r.warnings[0].wtype, "unclosed-fence"); + } + + // ========================================================================= + // Text block accumulation + // RFC §6.3 / RFC §6.4 / Engine Spec §10 + // ========================================================================= + + #[test] + fn text_only() { + // RFC §6.4: "Consecutive text lines form a single text block." + // RFC §7.2: content is lines joined with LF. + let r = parse_document("line one\nline two\nline three"); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "line one\nline two\nline three"); + assert_eq!(r.textblocks[0].range.start_line, 0); + assert_eq!(r.textblocks[0].range.end_line, 2); + } + + // ========================================================================= + // Interleaving commands and text + // RFC §6 / RFC §6.4 / Engine Spec §5.3 + // ========================================================================= + + #[test] + fn text_before_command() { + // RFC §6.4: text lines before a command form a text block. + let r = parse_document("preamble\n/cmd arg"); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "preamble"); + assert_eq!(r.commands.len(), 1); + } + + #[test] + fn text_after_command() { + // RFC §6.4: "A new text block begins after a command is finalized, + // if text lines follow." + let r = parse_document("/cmd arg\npostamble"); + assert_eq!(r.commands.len(), 1); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "postamble"); + } + + #[test] + fn text_between_commands() { + // RFC §6.4: text between two commands forms its own block. + let r = parse_document("/cmd1 a\nmiddle text\n/cmd2 b"); + assert_eq!(r.commands.len(), 2); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "middle text"); + } + + #[test] + fn consecutive_commands() { + // RFC §6.5: multiple commands in document order. + let r = parse_document("/cmd1 a\n/cmd2 b"); + assert_eq!(r.commands.len(), 2); + assert_eq!(r.commands[0].name, "cmd1"); + assert_eq!(r.commands[1].name, "cmd2"); + } + + #[test] + fn fence_followed_by_command() { + // RFC Appendix C: after fence closes, parser returns to idle. + let r = parse_document("/cmd1 ```\nbody\n```\n/cmd2 arg"); + assert_eq!(r.commands.len(), 2); + assert_eq!(r.commands[0].arguments.mode, ArgumentMode::Fence); + assert_eq!(r.commands[1].arguments.mode, ArgumentMode::SingleLine); + } + + // ========================================================================= + // ID assignment — sequential, independent counters + // RFC §6.5 / Engine Spec §3.2 / Engine Spec §3.5 + // ========================================================================= + + #[test] + fn command_ids_sequential() { + // RFC §6.5: "cmd-0, cmd-1, cmd-2." + let r = parse_document("/a x\n/b y\n/c z"); + assert_eq!(r.commands[0].id, "cmd-0"); + assert_eq!(r.commands[1].id, "cmd-1"); + assert_eq!(r.commands[2].id, "cmd-2"); + } + + #[test] + fn text_ids_sequential() { + // RFC §6.5: "text-0, text-1, text-2." + let r = parse_document("aaa\n/cmd x\nbbb\n/cmd y\nccc"); + assert_eq!(r.textblocks[0].id, "text-0"); + assert_eq!(r.textblocks[1].id, "text-1"); + assert_eq!(r.textblocks[2].id, "text-2"); + } + + #[test] + fn command_and_text_ids_independent() { + // RFC §6.5: command and text block ID sequences are independent. + let r = parse_document("prose\n/cmd arg\nmore prose"); + assert_eq!(r.commands[0].id, "cmd-0"); + assert_eq!(r.textblocks[0].id, "text-0"); + assert_eq!(r.textblocks[1].id, "text-1"); + } + + // ========================================================================= + // raw field — physical line preservation + // RFC §7.1 / Engine Spec §9.3 step 2 + // ========================================================================= + + #[test] + fn single_line_raw() { + // RFC §7.1: raw is "the exact source text from the normalized input." + let r = parse_document("/echo hello world"); + assert_eq!(r.commands[0].raw, "/echo hello world"); + } + + #[test] + fn joined_command_raw_preserves_backslashes() { + // RFC §7.1: "For joined commands, this includes all physical lines + // with their backslashes and LF separators." + let r = parse_document("/deploy prod \\\n --region us-west-2"); + assert_eq!(r.commands[0].raw, "/deploy prod \\\n --region us-west-2"); + } + + #[test] + fn fenced_raw_includes_all_lines() { + // RFC §7.1: "For fenced commands, this includes the opener line, + // all body lines, and the closer line if present." + let r = parse_document("/cmd ```\nbody\n```"); + assert_eq!(r.commands[0].raw, "/cmd ```\nbody\n```"); + } + + // ========================================================================= + // Trailing newline edge case + // Engine Spec §5.2 / RFC §3.1 + // ========================================================================= + + #[test] + fn trailing_newline_no_empty_text_block() { + // Engine Spec §5.2: "the current engine pops a trailing empty element + // when the input ends with LF." + let r = parse_document("/cmd arg\n"); + assert_eq!(r.commands.len(), 1); + assert!(r.textblocks.is_empty()); + } + + // ========================================================================= + // Version field + // Engine Spec §14 / Engine Spec §3.1 + // ========================================================================= + + #[test] + fn version_set() { + // Engine Spec §14: SPEC_VERSION populated into ParseResult.version. + let r = parse_document(""); + assert_eq!(r.version, SPEC_VERSION); + } + + // ========================================================================= + // Property tests + // ========================================================================= + + use proptest::prelude::*; + + proptest! { + // RFC §8.2 item 4: "always produces a valid result for any input." + // Engine Spec §4.2: total function, never panics. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn never_panics(input in "\\PC{0,500}") { + let _ = parse_document(&input); + } + + // Engine Spec §14: version is always SPEC_VERSION. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn version_always_spec_version(input in "\\PC{0,200}") { + let r = parse_document(&input); + prop_assert_eq!(r.version, SPEC_VERSION); + } + } +} + +// ============================================================================= +// TEST GAPS: spec areas this file's functions touch but are not tested +// ============================================================================= +// +// | Spec Section | Gap | Severity | +// |-----------------------------------|--------------------------------------------------|----------| +// | Engine Spec §11 / RFC §7.4 | WARNING TYPE STRING: code emits "unclosed-fence" | CRITICAL | +// | | (kebab-case) but Engine Spec §11 requires | | +// | | "unclosed_fence" (snake_case). This is a code bug | | +// | | in finalize_fence, observable at integration level. | | +// |-----------------------------------|--------------------------------------------------|----------| +// | Engine Spec §7.1 vs RFC §3.2 | SPACE INSERTION: join.rs inserts a space between | CRITICAL | +// | | joined lines. Engine Spec §7.1 says "No separator | | +// | | character is inserted." RFC §3.2 step 3 says | | +// | | "separated by a single SPACE." Specs contradict. | | +// | | No integration test asserts the exact join result. | | +// | | joined_command_raw_preserves_backslashes tests raw | | +// | | but not the joined logical line / payload content. | | +// |-----------------------------------|--------------------------------------------------|----------| +// | RFC §6.1 / Engine Spec §10 | TEXT BLOCK CONTENT IS PHYSICAL: Engine Spec §10 | HIGH | +// | | says "A logical line formed by backslash | | +// | | continuation contributes all of its constituent | | +// | | physical lines with backslashes intact." No test | | +// | | verifies that a text block containing a joined line | | +// | | preserves the physical lines (with backslashes) in | | +// | | its content, not the joined logical text. | | +// |-----------------------------------|--------------------------------------------------|----------| +// | RFC Appendix B.5 | MIXED SCENARIO: RFC Appendix B.5 has a full | MEDIUM | +// | | worked example with blank line in text, two | | +// | | commands, and trailing text. No integration test | | +// | | replicates this exact scenario end-to-end. | | +// |-----------------------------------|--------------------------------------------------|----------| +// | RFC Appendix B.6 | INVALID SLASH LINES: RFC Appendix B.6 shows | MEDIUM | +// | | "/123", "/ bare slash", "/Hello" as text lines. | | +// | | No integration test verifies these pass through | | +// | | parse_document as text blocks. | | +// |-----------------------------------|--------------------------------------------------|----------| +// | RFC Appendix B.4 | BACKSLASH JOIN INTO FENCE: A command line | MEDIUM | +// | | split across physical lines via backslash that | | +// | | joins into a fence opener (e.g., "/mcp \" + "```"). | | +// | | No integration test for this scenario. | | +// |-----------------------------------|--------------------------------------------------|----------| +// | RFC §5.2.3 / Appendix B.8-B.9 | FENCE CLOSER WITH TRAILING BACKSLASH: B.8 shows | MEDIUM | +// | | "```\" is NOT a closer (backslash makes it non- | | +// | | solely-backtick). B.9 shows "```" as a valid closer | | +// | | followed by content that joins. Neither scenario | | +// | | has an integration test. | | +// |-----------------------------------|--------------------------------------------------|----------| +// | RFC §6.4 | BLANK LINES IN TEXT BLOCKS: No test for blank | LOW | +// | | lines within a text block (e.g., "a\n\nb" producing | | +// | | content "a\n\nb"). Covered in text.rs unit tests | | +// | | but not at integration level. | | +// |-----------------------------------|--------------------------------------------------|----------| +// | RFC §8.3 | ROUNDTRIP FIDELITY: No test asserts that | LOW | +// | | P(F(P(I))) == P(I) for any input. This would | | +// | | require a formatter, which is an SDK concern, but | | +// | | the engine should preserve enough data to enable it.| | +// |-----------------------------------|--------------------------------------------------|----------| +// | RFC §8.4 | DETERMINISM: No property test asserts that | LOW | +// | | parse_document(x) == parse_document(x) for same | | +// | | input. Trivially true for pure functions but would | | +// | | guard against accidental HashMap usage. | | +// |-----------------------------------|--------------------------------------------------|----------| +// | Engine Spec §6 | WHITESPACE DEFINITION: step_idle calls | HIGH | +// | | classify_line which uses .trim_start() and | | +// | | .is_whitespace() internally. Engine Spec §6 | | +// | | mandates SP+HTAB only. No integration test with | | +// | | U+00A0 or other Unicode WSP verifying that such | | +// | | lines are NOT treated as having leading whitespace. | | +// | | This is a classify.rs bug observable here. | | +// |-----------------------------------|--------------------------------------------------|----------| +// | Engine Spec §5.2 | SPLIT_PHYSICAL_LINES EDGE CASES: split_physical_ | LOW | +// | | lines is a private fn in this file. No direct test | | +// | | for inputs like "\n" (single LF), "\n\n" (two LFs),| | +// | | or "no newline" (no LF). The trailing-newline test | | +// | | covers one case only. | | diff --git a/parser/src/single_line.rs b/parser/src/single_line.rs new file mode 100644 index 0000000..247fcb8 --- /dev/null +++ b/parser/src/single_line.rs @@ -0,0 +1,138 @@ +use crate::{ArgumentMode, Command, CommandArguments, LineRange, classify::CommandHeader}; + +/// Finalize a single-line command. No accumulation needed. +/// Extracted from the single-line paths of start_command + finalize_command. +pub fn finalize_single_line(header: CommandHeader, raw: String, id: usize, range: LineRange) -> Command { + let payload = header.header_text.clone(); + Command { + id: format!("cmd-{}", id), + name: header.name, + raw, + range, + arguments: CommandArguments { + header: header.header_text, + mode: ArgumentMode::SingleLine, + fence_lang: None, + payload, + }, + } +} + +#[cfg(test)] +mod tests { + + use crate::{ArgumentMode, LineRange, classify::CommandHeader, single_line::finalize_single_line}; + + fn make_header(name: &str, header_text: &str) -> CommandHeader { + CommandHeader { + raw: format!("/{name} {header_text}"), + name: name.to_string(), + header_text: header_text.to_string(), + mode: ArgumentMode::SingleLine, + fence_lang: None, + fence_backtick_count: 0, + } + } + + // ========================================================================= + // Output field mapping + // Engine Spec §9.3 / RFC §7.1 / RFC §6.5 + // ========================================================================= + + #[test] + fn id_and_name() { + // RFC §6.5: sequential zero-based id "cmd-{n}". + // Engine Spec §3.2: name from header. + let h = make_header("deploy", "prod"); + let cmd = finalize_single_line(h, "/deploy prod".into(), 5, LineRange { start_line: 0, end_line: 0 }); + assert_eq!(cmd.id, "cmd-5"); + assert_eq!(cmd.name, "deploy"); + } + + #[test] + fn raw_and_range_passed_through() { + // RFC §7.1: raw source is "the exact source text from the normalized + // input (before line joining)." + // Engine Spec §3.6: LineRange inclusive on both ends. + let h = make_header("cmd", "arg"); + let range = LineRange { start_line: 3, end_line: 5 }; + let cmd = finalize_single_line(h, "/cmd arg".into(), 0, range); + assert_eq!(cmd.raw, "/cmd arg"); + assert_eq!(cmd.range.start_line, 3); + assert_eq!(cmd.range.end_line, 5); + } + + // ========================================================================= + // Single-line mode: header == payload + // RFC §4.4 / RFC §5.1 / Engine Spec §3.3 + // ========================================================================= + + #[test] + fn payload_equals_header() { + // RFC §4.4: "In single-line mode, the header and payload are identical + // (the full arguments text)." + // RFC §5.1: mode is "single-line", no fence language. + let h = make_header("deploy", "production --region us-west-2"); + let cmd = finalize_single_line( + h, + "/deploy production --region us-west-2".into(), + 0, + LineRange { start_line: 0, end_line: 0 }, + ); + assert_eq!(cmd.arguments.payload, "production --region us-west-2"); + assert_eq!(cmd.arguments.header, "production --region us-west-2"); + assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); + assert_eq!(cmd.arguments.fence_lang, None); + } + + #[test] + fn empty_args() { + // RFC §4.3: "The arguments portion may be empty (command with no arguments)." + // RFC §5.1: empty args -> header and payload are both empty strings. + let h = make_header("ping", ""); + let cmd = finalize_single_line(h, "/ping".into(), 0, LineRange { start_line: 0, end_line: 0 }); + assert_eq!(cmd.arguments.payload, ""); + assert_eq!(cmd.arguments.header, ""); + assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); + assert_eq!(cmd.arguments.fence_lang, None); + } +} + +// ============================================================================= +// TEST GAPS: spec areas this file's functions touch but are not tested +// ============================================================================= +// +// | Spec Section | Gap | Severity | +// |---------------------------------|-------------------------------------------------|----------| +// | Engine Spec §9.3 | NO WARNINGS VECTOR: finalize_single_line returns| LOW | +// | | a Command, not (Command, Vec). The | | +// | | fence path returns warnings but single-line | | +// | | cannot produce warnings per spec, so this is | | +// | | correct. However the asymmetric return types | | +// | | between finalize_single_line and finalize_fence | | +// | | should be documented. | | +// |---------------------------------|-------------------------------------------------|----------| +// | RFC §7.1 | RAW FOR JOINED COMMANDS: When a single-line | MEDIUM | +// | | command spans multiple physical lines via | | +// | | backslash joining (RFC Appendix B.2), the raw | | +// | | field should contain all physical lines with | | +// | | backslashes and LF separators. This function | | +// | | accepts raw as a parameter (the caller builds | | +// | | it), so it's not this function's concern, but | | +// | | no test verifies multi-physical-line raw input. | | +// |---------------------------------|-------------------------------------------------|----------| +// | RFC §7.1 | RAW WITH LEADING WHITESPACE: No test verifies | LOW | +// | | that raw containing leading whitespace (e.g., | | +// | | " /cmd arg") is passed through unmodified. | | +// |---------------------------------|-------------------------------------------------|----------| +// | Engine Spec §3.4 | ARGUMENT MODE SERIALIZATION: Engine Spec says | INFO | +// | | "String serialization is the SDK's | | +// | | responsibility." The enum value | | +// | | ArgumentMode::SingleLine is set correctly, but | | +// | | no test confirms the enum variant name maps to | | +// | | "single-line" (SDK concern, not engine). | | +// |---------------------------------|-------------------------------------------------|----------| +// | (none) | NO PROPERTY TESTS: This is a simple mapping | LOW | +// | | function with no branching logic. Property tests | | +// | | would add minimal value, but one could assert | | +// | | payload == header for all inputs. | | diff --git a/parser/src/text.rs b/parser/src/text.rs new file mode 100644 index 0000000..ced7673 --- /dev/null +++ b/parser/src/text.rs @@ -0,0 +1,243 @@ +use crate::{LineRange, TextBlock}; + +/// Accumulated state for a text block being built line by line. +#[derive(Debug, Clone)] +pub struct PendingText { + pub start_line: usize, + pub end_line: usize, + pub lines: Vec, +} + +/// Start a new pending text block at the given physical line index. +pub fn start_text(line_index: usize, line: &str) -> PendingText { + PendingText { start_line: line_index, end_line: line_index, lines: vec![line.to_string()] } +} + +/// Append one more physical line to an in-progress text block. +pub fn append_text(mut text: PendingText, line_index: usize, line: &str) -> PendingText { + text.end_line = line_index; + text.lines.push(line.to_string()); + text +} + +/// Finalize a pending text block, assigning it the given sequential id. +/// +/// The caller supplies a zero-based counter; this function formats it as +/// `text-{id}` per §7. The caller is responsible for incrementing the counter +/// after each call so that IDs are unique and sequential within an envelope. +pub fn finalize_text(text: PendingText, id: usize) -> TextBlock { + let content = text.lines.join("\n"); + TextBlock { + id: format!("text-{id}"), + range: LineRange { start_line: text.start_line, end_line: text.end_line }, + content, + } +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + // NOTE: LineRange, TextBlock imported from crate::domain. + + // ========================================================================= + // start_text — initial state + // RFC §6.1 / Engine Spec §10.1 + // ========================================================================= + + #[test] + fn start_text_initial_state() { + // RFC §6.1: "Consecutive non-command logical lines form a single + // text block." The first line seeds the block. + // Engine Spec §10.1: PendingText start_line == end_line, lines + // contains exactly the first line verbatim. + let pt = start_text(3, " indented line! "); + assert_eq!(pt.start_line, 3); + assert_eq!(pt.end_line, 3); + assert_eq!(pt.lines, vec![" indented line! "]); + } + + // ========================================================================= + // append_text — accumulation + // RFC §6.1 / Engine Spec §10.2 + // ========================================================================= + + #[test] + fn append_advances_end_line_preserves_start() { + // RFC §6.1: text block range spans first to last physical line. + // Engine Spec §10.2: append updates end_line, never start_line. + let pt = start_text(2, "first"); + let pt = append_text(pt, 3, "second"); + assert_eq!(pt.start_line, 2); + assert_eq!(pt.end_line, 3); + } + + #[test] + fn append_accumulates_in_document_order() { + // RFC §6.1: "Consecutive non-command lines form a single text block." + // Lines must appear in the order they were encountered. + let pt = start_text(0, "a"); + let pt = append_text(pt, 1, "b"); + let pt = append_text(pt, 2, "c"); + assert_eq!(pt.lines, vec!["a", "b", "c"]); + } + + #[test] + fn append_preserves_blank_and_whitespace_lines() { + // RFC §6.1: "Blank lines that are part of a text region are included + // in the text block content." + let pt = start_text(0, "before"); + let pt = append_text(pt, 1, ""); + let pt = append_text(pt, 2, " "); + let pt = append_text(pt, 3, "after"); + assert_eq!(pt.lines, vec!["before", "", " ", "after"]); + } + + // ========================================================================= + // finalize_text — output fields + // RFC §6.5 / RFC §7.1 / Engine Spec §10.3 + // ========================================================================= + + #[test] + fn finalize_id_format() { + // RFC §6.5: "Text blocks are assigned IDs: text-0, text-1, …" + // Engine Spec §10.3: id = format!("text-{}", counter). + assert_eq!(finalize_text(start_text(0, "x"), 0).id, "text-0"); + assert_eq!(finalize_text(start_text(0, "x"), 7).id, "text-7"); + } + + #[test] + fn finalize_content_joined_with_lf() { + // RFC §6.1: "Text block content preserves the original lines joined + // with LF separators." + // Engine Spec §10.3: lines.join("\n"). + let pt = start_text(0, "line one"); + let pt = append_text(pt, 1, "line two"); + let pt = append_text(pt, 2, "line three"); + let block = finalize_text(pt, 0); + assert_eq!(block.content, "line one\nline two\nline three"); + } + + #[test] + fn finalize_single_line_no_trailing_lf() { + // RFC §6.1 (implied): single-line block has no separator. + let block = finalize_text(start_text(0, "hello"), 0); + assert_eq!(block.content, "hello"); + } + + #[test] + fn finalize_range() { + // Engine Spec §3.6: LineRange inclusive on both ends. + let pt = start_text(4, "a"); + let pt = append_text(pt, 5, "b"); + let block = finalize_text(pt, 1); + assert_eq!(block.range.start_line, 4); + assert_eq!(block.range.end_line, 5); + } + + #[test] + fn finalize_blank_line_in_content() { + // RFC §6.1: blank lines included. join("\n") on ["before","","after"] + // produces "before\n\nafter". + let pt = start_text(0, "before"); + let pt = append_text(pt, 1, ""); + let pt = append_text(pt, 2, "after"); + let block = finalize_text(pt, 0); + assert_eq!(block.content, "before\n\nafter"); + } + + // ========================================================================= + // Property tests + // ========================================================================= + + proptest! { + // RFC §6.5: id pattern is always "text-{n}". + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn id_matches_text_n_pattern(id in 0usize..1000) { + let block = finalize_text(start_text(0, "x"), id); + prop_assert_eq!(block.id, format!("text-{id}")); + } + + // RFC §6.1: content == lines.join("\n"). + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn content_equals_lines_joined_with_newline( + lines in prop::collection::vec("[a-zA-Z0-9 !.,]{0,60}", 1..20) + ) { + let expected = lines.join("\n"); + let pt = lines.iter().enumerate().fold( + start_text(0, &lines[0]), + |acc, (i, line)| if i == 0 { acc } else { append_text(acc, i, line) }, + ); + let block = finalize_text(pt, 0); + prop_assert_eq!(block.content, expected); + } + + // Engine Spec §10.2: range is [start, start + extra]. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn range_covers_exactly_the_lines_provided( + start in 0usize..100, + extra in 0usize..20 + ) { + let mut pt = start_text(start, "first"); + for i in 1..=extra { + pt = append_text(pt, start + i, "line"); + } + let block = finalize_text(pt, 0); + prop_assert_eq!(block.range.start_line, start); + prop_assert_eq!(block.range.end_line, start + extra); + } + + // Engine Spec §10.2: start_line is immutable after creation. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn append_never_changes_start_line( + start in 0usize..100, + lines in prop::collection::vec("[a-zA-Z]{1,20}", 1..10) + ) { + let mut pt = start_text(start, "first"); + for (i, line) in lines.iter().enumerate() { + pt = append_text(pt, start + i + 1, line); + } + prop_assert_eq!(pt.start_line, start); + } + } +} + +// ============================================================================= +// TEST GAPS: spec areas this file's functions touch but are not tested +// ============================================================================= +// +// | Spec Section | Gap | Severity | +// |---------------------------------|-------------------------------------------------|----------| +// | RFC §6.1 | LOGICAL vs PHYSICAL LINES: The spec says text | MEDIUM | +// | | blocks are built from logical lines (post-join). | | +// | | This module accepts raw strings, so the caller | | +// | | (state machine) must feed logical lines. No test | | +// | | verifies the contract that a joined logical line | | +// | | spanning multiple physical lines is passed as a | | +// | | single string. The line_index parameter naming | | +// | | is ambiguous (physical? logical?). | | +// |---------------------------------|-------------------------------------------------|----------| +// | RFC §6.1 | TEXT BLOCK BOUNDARY: "A text block ends when the | LOW | +// | | next logical line is a command trigger or EOF." | | +// | | This is the state machine's responsibility, not | | +// | | this module's, but there's no test that | | +// | | finalize_text can be called at any point | | +// | | (including after zero appends) without panic. | | +// | | The start_text + finalize_text path is tested | | +// | | but not explicitly named as a "zero-append" case.| | +// |---------------------------------|-------------------------------------------------|----------| +// | Engine Spec §10.3 | NO WARNINGS: Unlike finalize_fence, finalize_text| LOW | +// | | never produces warnings. This is correct per | | +// | | spec, but the asymmetry with finalize_fence is | | +// | | not documented or tested (e.g., no test asserting| | +// | | there's no warnings field/return). | | +// |---------------------------------|-------------------------------------------------|----------| +// | RFC §6.1 | CONTENT WITH ONLY BLANK LINES: No test for a | LOW | +// | | text block consisting entirely of blank lines | | +// | | (e.g., three empty strings). The join produces | | +// | | "\n\n" which is valid but untested. | | diff --git a/parser/src/domain/types.rs b/parser/src/types.rs similarity index 77% rename from parser/src/domain/types.rs rename to parser/src/types.rs index 3868589..67617c3 100644 --- a/parser/src/domain/types.rs +++ b/parser/src/types.rs @@ -1,4 +1,4 @@ -use super::errors::Warning; +pub const SPEC_VERSION: &str = "0.3.0"; /// Inclusive line range (zero-based). #[derive(Debug, Clone, PartialEq, Eq)] @@ -50,4 +50,14 @@ pub struct ParseResult { pub warnings: Vec, } -pub const SPEC_VERSION: &str = "0.3.0"; + +/// Non-fatal conditions detected during parsing. +/// +/// Warnings are collected in `ParseResult.warnings` rather than +/// causing the parse to fail. The parser is intentionally permissive. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Warning { + pub wtype: String, + pub start_line: Option, + pub message: Option, +} diff --git a/parser/tests/mod.rs b/parser/tests/mod.rs deleted file mode 100644 index be37b0f..0000000 --- a/parser/tests/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -// use super::*; -// use crate::domain::{ArgumentMode, ParserContext}; - -// mod proptest; - -// // --- Test helpers --- - -// fn default_context() -> ParserContext { ParserContext::default() } - -// fn parse(input: &str) -> SlashParseResult { parse_to_domain(input, default_context()) } - -// fn assert_single_command(input: &str) -> Command { -// let result = parse(input); -// assert_eq!(result.commands.len(), 1, "expected exactly 1 command"); -// result.commands.into_iter().next().unwrap() -// } - -// fn assert_no_commands(input: &str) { -// let result = parse(input); -// assert!(result.commands.is_empty(), "expected no commands, got {}", result.commands.len()); -// } diff --git a/parser/tests/orchestration_tests.rs b/parser/tests/orchestration_tests.rs new file mode 100644 index 0000000..33c1513 --- /dev/null +++ b/parser/tests/orchestration_tests.rs @@ -0,0 +1,322 @@ +//! Integration tests for parse_document orchestration. +//! +//! These tests verify that the sub-modules are wired together correctly +//! by the state machine in parse.rs. Each test targets a specific +//! orchestration concern that cannot be tested at the unit level. +//! +//! Sub-module behavior (normalization, classification, joining, fence +//! accumulation, text accumulation) is covered by unit tests in each +//! module's #[cfg(test)] block. These tests do NOT re-verify sub-module +//! logic; they verify the decisions the orchestrator makes. + +use slasher_engine::{parse_document, ArgumentMode, SPEC_VERSION}; + +// ============================================================================= +// State: Idle -> single-line command -> Idle +// Orchestration: classify_line returns Command(SingleLine), orchestrator +// calls finalize_single_line, increments cmd_seq, returns to Idle. +// ============================================================================= + +#[test] +fn single_line_returns_to_idle() { + // Two consecutive single-line commands prove the state machine returns + // to Idle after each. If it didn't, the second would be lost. + let r = parse_document("/cmd1 a\n/cmd2 b"); + assert_eq!(r.commands.len(), 2); + assert_eq!(r.commands[0].name, "cmd1"); + assert_eq!(r.commands[1].name, "cmd2"); +} + +// ============================================================================= +// State: Idle -> fence command -> InFence -> Completed -> Idle +// Orchestration: classify_line returns Command(Fence), orchestrator calls +// open_fence, transitions to InFence, feeds physical lines via +// next_physical, detects Completed, finalizes, returns to Idle. +// ============================================================================= + +#[test] +fn fence_completes_and_returns_to_idle() { + // A fenced command followed by a single-line command proves the full + // Idle -> InFence -> Idle cycle. + let r = parse_document("/fenced ```\nbody line\n```\n/after arg"); + assert_eq!(r.commands.len(), 2); + assert_eq!(r.commands[0].arguments.mode, ArgumentMode::Fence); + assert_eq!(r.commands[0].arguments.payload, "body line"); + assert_eq!(r.commands[1].arguments.mode, ArgumentMode::SingleLine); + assert_eq!(r.commands[1].name, "after"); +} + +// ============================================================================= +// State: InFence -> EOF (unclosed) +// Orchestration: next_physical returns None, orchestrator calls +// finalize_fence(fence, true), pushes warning. +// ============================================================================= + +#[test] +fn unclosed_fence_at_eof() { + let r = parse_document("/cmd ```\nline one\nline two"); + assert_eq!(r.commands.len(), 1); + assert_eq!(r.commands[0].arguments.payload, "line one\nline two"); + assert_eq!(r.warnings.len(), 1); + assert_eq!(r.warnings[0].start_line, Some(0)); +} + +// ============================================================================= +// Flush: text flushed before command +// Orchestration: when classify_line returns Command, flush_text is called +// first, finalizing any pending text block. +// ============================================================================= + +#[test] +fn text_flushed_before_command() { + let r = parse_document("prose line\n/cmd arg"); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "prose line"); + assert_eq!(r.textblocks[0].id, "text-0"); + // Text block appears before the command in document order. + assert_eq!(r.textblocks[0].range.end_line, 0); + assert_eq!(r.commands[0].range.start_line, 1); +} + +// ============================================================================= +// Flush: text flushed at EOF +// Orchestration: after the main loop breaks, flush_text is called to +// finalize any trailing text. +// ============================================================================= + +#[test] +fn text_flushed_at_eof() { + let r = parse_document("/cmd arg\ntrailing prose"); + assert_eq!(r.commands.len(), 1); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "trailing prose"); +} + +// ============================================================================= +// Flush: no spurious text blocks +// Orchestration: consecutive commands with no text between them must NOT +// produce an empty text block. +// ============================================================================= + +#[test] +fn no_text_block_between_consecutive_commands() { + let r = parse_document("/cmd1 a\n/cmd2 b\n/cmd3 c"); + assert_eq!(r.commands.len(), 3); + assert!(r.textblocks.is_empty()); +} + +#[test] +fn no_text_block_after_fence_closer_before_command() { + let r = parse_document("/f ```\nbody\n```\n/cmd arg"); + assert_eq!(r.commands.len(), 2); + assert!(r.textblocks.is_empty()); +} + +// ============================================================================= +// Raw wiring: joined single-line command +// Orchestration: step_idle rebuilds raw from physical line slice +// phys[first..=last].join("\n"), preserving backslashes. +// ============================================================================= + +#[test] +fn joined_command_raw_has_physical_lines() { + let r = parse_document("/deploy prod \\\n --region us-west-2"); + assert_eq!(r.commands.len(), 1); + // raw must contain both physical lines with the backslash intact. + assert_eq!(r.commands[0].raw, "/deploy prod \\\n --region us-west-2"); + // But the payload is the joined logical content. + assert!(r.commands[0].arguments.payload.contains("--region")); +} + +// ============================================================================= +// Raw wiring: fenced command +// Orchestration: raw is seeded by open_fence (opener line from phys slice), +// then accept_fence_line appends body and closer. +// ============================================================================= + +#[test] +fn fenced_raw_includes_opener_body_closer() { + let r = parse_document("/cmd ```\nfirst\nsecond\n```"); + assert_eq!(r.commands[0].raw, "/cmd ```\nfirst\nsecond\n```"); +} + +// ============================================================================= +// Physical text: text block uses physical lines, not logical lines +// Orchestration: accumulate_text calls fold_physical_lines over the phys +// slice, so backslash-continued text lines keep their backslashes. +// Engine Spec §10: "A logical line formed by backslash continuation +// contributes all of its constituent physical lines with backslashes +// intact to the text block content." +// ============================================================================= + +#[test] +fn text_block_preserves_physical_lines_with_backslashes() { + // "hello \" and " world" are two physical lines that join into one + // logical line. The text block content must have BOTH physical lines. + let r = parse_document("hello \\\n world"); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "hello \\\n world"); + assert_eq!(r.textblocks[0].range.start_line, 0); + assert_eq!(r.textblocks[0].range.end_line, 1); +} + +// ============================================================================= +// ID counters: independent cmd_seq and text_seq +// Orchestration: cmd_seq and text_seq are separate fields in ParseCtx, +// incremented independently. +// ============================================================================= + +#[test] +fn id_counters_independent_across_interleaving() { + let r = parse_document("t0\n/c0 a\nt1\n/c1 b\nt2"); + assert_eq!(r.textblocks[0].id, "text-0"); + assert_eq!(r.commands[0].id, "cmd-0"); + assert_eq!(r.textblocks[1].id, "text-1"); + assert_eq!(r.commands[1].id, "cmd-1"); + assert_eq!(r.textblocks[2].id, "text-2"); +} + +#[test] +fn fenced_command_shares_cmd_seq_with_single_line() { + // A fence command consumes cmd_seq=0, the next single-line gets cmd_seq=1. + let r = parse_document("/fenced ```\nbody\n```\n/single arg"); + assert_eq!(r.commands[0].id, "cmd-0"); + assert_eq!(r.commands[1].id, "cmd-1"); +} + +// ============================================================================= +// split_physical_lines: trailing LF handling +// Orchestration: split_physical_lines pops trailing empty element so +// "input\n" does not create a phantom empty text block. +// ============================================================================= + +#[test] +fn trailing_lf_no_phantom_text_block() { + let r = parse_document("/cmd arg\n"); + assert_eq!(r.commands.len(), 1); + assert!(r.textblocks.is_empty()); +} + +#[test] +fn no_trailing_lf_still_processes_last_line() { + let r = parse_document("/cmd1 a\n/cmd2 b"); + assert_eq!(r.commands.len(), 2); + assert_eq!(r.commands[1].name, "cmd2"); +} + +// ============================================================================= +// Fence boundary: next_logical resumes after fence closes +// Orchestration: after FenceResult::Completed, state returns to Idle, +// and the next iteration calls next_logical (with joining active). +// This means a backslash-continued line after a fence closer is joined. +// ============================================================================= + +#[test] +fn joining_resumes_after_fence_closes() { + // After the fence closes, "hello \" + " world" should join into one + // logical text line. + let r = parse_document("/cmd ```\nbody\n```\nhello \\\n world"); + assert_eq!(r.commands.len(), 1); + assert_eq!(r.textblocks.len(), 1); + // The text block preserves physical lines (backslash intact). + assert_eq!(r.textblocks[0].content, "hello \\\n world"); + // But it's a single logical line, so only one text block. + assert_eq!(r.textblocks[0].range.start_line, 3); + assert_eq!(r.textblocks[0].range.end_line, 4); +} + +// ============================================================================= +// Fence body: next_physical bypasses joining +// Orchestration: in InFence state, the orchestrator calls next_physical, +// so backslashes inside the fence body are NOT consumed as join markers. +// ============================================================================= + +#[test] +fn backslash_in_fence_body_is_literal() { + let r = parse_document("/cmd ```\nline one \\\nline two\n```"); + assert_eq!(r.commands.len(), 1); + // Both lines are separate payload lines; the backslash did not join them. + assert_eq!(r.commands[0].arguments.payload, "line one \\\nline two"); +} + +// ============================================================================= +// Version wiring +// Orchestration: into_result sets version from SPEC_VERSION. +// ============================================================================= + +#[test] +fn version_from_spec_constant() { + let r = parse_document(""); + assert_eq!(r.version, SPEC_VERSION); +} + +// ============================================================================= +// Full scenario: RFC Appendix B.5 equivalent +// Orchestration: exercises the complete interleaving of text blocks, +// commands, blank lines, and ID assignment in a single document. +// ============================================================================= + +#[test] +fn full_scenario_text_commands_interleaved() { + let input = "Welcome to the system.\n\n/deploy staging\n/notify team --channel ops\nDone."; + let r = parse_document(input); + + // Text block 0: lines 0-1 (prose + blank line) + assert_eq!(r.textblocks[0].id, "text-0"); + assert_eq!(r.textblocks[0].content, "Welcome to the system.\n"); + assert_eq!(r.textblocks[0].range.start_line, 0); + assert_eq!(r.textblocks[0].range.end_line, 1); + + // Commands + assert_eq!(r.commands[0].id, "cmd-0"); + assert_eq!(r.commands[0].name, "deploy"); + assert_eq!(r.commands[0].arguments.payload, "staging"); + + assert_eq!(r.commands[1].id, "cmd-1"); + assert_eq!(r.commands[1].name, "notify"); + assert_eq!(r.commands[1].arguments.payload, "team --channel ops"); + + // Text block 1: trailing prose + assert_eq!(r.textblocks[1].id, "text-1"); + assert_eq!(r.textblocks[1].content, "Done."); +} + +// ============================================================================= +// Empty and minimal inputs +// Orchestration: total function guarantee, no panics, correct empty result. +// ============================================================================= + +#[test] +fn empty_input() { + let r = parse_document(""); + assert!(r.commands.is_empty()); + assert!(r.textblocks.is_empty()); + assert!(r.warnings.is_empty()); +} + +#[test] +fn single_newline_only() { + // "\n" normalizes to one LF. split_physical_lines produces [""] then + // pops it. Result should be empty. + let r = parse_document("\n"); + assert!(r.commands.is_empty()); + assert!(r.textblocks.is_empty()); +} + +// ============================================================================= +// Invalid slash lines flow through as text +// RFC §4.5 / RFC Appendix B.6 +// ============================================================================= + +#[test] +fn invalid_slash_lines_are_text() { + let input = "/123\n/ bare\n/Hello\n/cmd- trailing\n/deploy staging"; + let r = parse_document(input); + // Only "/deploy" is a valid command. + assert_eq!(r.commands.len(), 1); + assert_eq!(r.commands[0].name, "deploy"); + // The four invalid lines form one text block before the command. + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].range.start_line, 0); + assert_eq!(r.textblocks[0].range.end_line, 3); +} From dcee54627ff46619019eff11f6fb7de758f9156c Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Sun, 22 Mar 2026 23:01:40 -0600 Subject: [PATCH 11/13] chore(engine): move the crate, parser, to engine. --- .release-please-manifest.json | 11 +- Cargo.lock | 18 +- docs/adr parse pipeline.md | 267 ------------------ {parser => engine}/Cargo.toml | 0 {parser => engine}/moon.yml | 0 {parser => engine}/src/classify.rs | 0 {parser => engine}/src/fence.rs | 0 .../src/integration_tests/mod.rs | 0 .../src/integration_tests/proptest.rs | 0 {parser => engine}/src/join.rs | 0 {parser => engine}/src/lib.rs | 0 {parser => engine}/src/mod.rs | 0 {parser => engine}/src/normalize.rs | 0 {parser => engine}/src/parse.rs | 0 {parser => engine}/src/single_line.rs | 0 {parser => engine}/src/text.rs | 0 {parser => engine}/src/types.rs | 0 .../tests/orchestration_tests.rs | 0 release-please-config.json | 49 ---- slash-go/moon.yml | 1 - slash-javascript/moon.yml | 1 - slash-python/moon.yml | 1 - slash-rust/moon.yml | 1 - slash-web/moon.yml | 1 - wasm-javascript/moon.yml | 1 - wasm-wasi/moon.yml | 1 - website/moon.yml | 1 - 27 files changed, 10 insertions(+), 343 deletions(-) delete mode 100644 docs/adr parse pipeline.md rename {parser => engine}/Cargo.toml (100%) rename {parser => engine}/moon.yml (100%) rename {parser => engine}/src/classify.rs (100%) rename {parser => engine}/src/fence.rs (100%) rename {parser => engine}/src/integration_tests/mod.rs (100%) rename {parser => engine}/src/integration_tests/proptest.rs (100%) rename {parser => engine}/src/join.rs (100%) rename {parser => engine}/src/lib.rs (100%) rename {parser => engine}/src/mod.rs (100%) rename {parser => engine}/src/normalize.rs (100%) rename {parser => engine}/src/parse.rs (100%) rename {parser => engine}/src/single_line.rs (100%) rename {parser => engine}/src/text.rs (100%) rename {parser => engine}/src/types.rs (100%) rename {parser => engine}/tests/orchestration_tests.rs (100%) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 11ea174..4239766 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,12 +1,3 @@ { - "parser": "0.1.0", - "wasm-javascript": "0.1.0", - "wasm-wasi": "0.1.0", - "spec": "0.1.0", - "riff-cli": "0.1.0", - "slash-rust": "0.1.0", - "slash-javascript": "0.1.0", - "slash-web": "0.1.0", - "slash-python": "0.1.0", - "slash-go": "0.1.0" + "parser": "0.1.0" } diff --git a/Cargo.lock b/Cargo.lock index 9571e46..39a7260 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -190,15 +190,6 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "parser" -version = "0.1.0" -dependencies = [ - "proptest", - "proptest-derive", - "thiserror", -] - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -401,6 +392,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "slasher_engine" +version = "0.1.0" +dependencies = [ + "proptest", + "proptest-derive", + "thiserror", +] + [[package]] name = "syn" version = "2.0.117" diff --git a/docs/adr parse pipeline.md b/docs/adr parse pipeline.md deleted file mode 100644 index ff00250..0000000 --- a/docs/adr parse pipeline.md +++ /dev/null @@ -1,267 +0,0 @@ ---- -status: accepted -date: 2026-03-19 -decision-makers: - - Tom -consulted: [] -informed: [] ---- - -# Document Parse Pipeline: Pull-Model LineJoiner with Two-State Loop - -## Context and Problem Statement - -The `document_parse.rs` module is the top-level orchestrator of the slash-parser v0.3.0 engine. It -must wire together normalization (§2.1), POSIX-style backslash line joining (§2.2), command -detection and classification (§3), a two-state machine (§4), fenced payload accumulation (§5.2), -text block collection (§6), sequential ID assignment (§7), and correct `raw` field construction -(§8.2), all within a single forward pass (§1.1) that supports incremental emission (§9.2). - -The fundamental tension is that the spec describes line joining as a "pre-pass" (§2.2) but mandates -fence immunity (§2.3): lines inside a fenced block must never be subject to joining. Fence -boundaries are only discoverable during command parsing, so a naive eager pre-pass is impossible -without either a second pass or buffering the entire input. - -How should `document_parse.rs` orchestrate the pipeline to satisfy all spec requirements in a single -forward pass? - -## Decision Drivers - -- Spec §2.3 (fence immunity) and §2.2 (line joining) create a circular dependency: joining must - happen before parsing, but fence boundaries are only known during parsing. -- Spec §9.2 requires incremental emission: each command or text block must be finalizable as soon as - its last physical line is consumed. Memory must be bounded by the largest single element, not - total input size. -- Spec §1.1 requires a single forward pass after normalization. No backtracking. -- Spec §8.2 requires `raw` to contain the exact normalized source text before line joining, - including backslash characters and `\n` separators between physical lines. -- Spec §6 and §9.4 (round-trip fidelity) require text block content and ranges to use physical line - numbers and physical line content so a formatter can reconstruct the original input. -- Spec §7 requires independent zero-based ID sequences for commands (`cmd-0`, `cmd-1`) and text - blocks (`text-0`, `text-1`). -- Architecture doc §9.2 recommends an explicit loop over iterator chains when the source of data - changes mid-stream. -- Architecture doc §9.1 notes that inputs are small (chatops messages, git comments), favoring - clarity over allocation avoidance. - -## Considered Options - -1. Pull-model LineJoiner with explicit two-state `while` loop -2. Pure iterator chain with `scan`/`flat_map` adapters -3. Two-pass approach: locate fence boundaries first, then join outside them -4. Zero-copy slice tracking instead of allocated `LogicalLine` strings -5. Mutable `String` builder for `raw` field construction -6. Single shared counter for command and text block IDs -7. Text block content from joined logical lines (post-join text) -8. Text block content from raw physical lines (pre-join text) - -## Decision Outcome - -Chosen options: - -- **Option 1** for the overall pipeline architecture. -- **Option 8** for text block content representation. - -Combined, these produce a pipeline where: - -1. `normalize(input)` converts all line endings to LF (§2.1). -2. The normalized string is split on `\n` into `physical_lines: Vec<&str>`, retained for `raw` - reconstruction and text block content. -3. An owned copy is passed to `LineJoiner::new(owned)`. -4. A `while` loop drives a `ParserState` enum (`Idle` | `InFence(PendingCommand)`). -5. In `Idle`, `joiner.next_logical()` produces joined `LogicalLine` values. Each is classified. - Commands start accumulation. Text lines feed physical line content into `PendingText` via - `start_text`/`append_text`. -6. In `InFence`, `joiner.next_physical()` produces raw lines. Each is fed to `accept_line`. On close - or EOF, the command is finalized and the state returns to `Idle`. -7. Before starting a command, `header.raw` is overwritten with - `physical_lines[first..=last].join("\n")` to satisfy §8.2. -8. Independent `cmd_seq: usize` and `text_seq: usize` counters are maintained and passed to - `start_command` and `finalize_text` respectively. -9. At EOF, any pending text block is finalized. The result is returned with no `context` field (SDKs - inject that). - -### Consequences - -**Good:** - -- Fence immunity is achieved without a second pass. The shared cursor inside `LineJoiner` means - calling `next_physical()` in `InFence` bypasses joining for exactly the lines inside the fence, - then `next_logical()` resumes joining when the state returns to `Idle`. -- Incremental emission is preserved. Each command and text block is finalized as soon as its last - line is consumed. No global buffering beyond the current pending element. -- The fence-opener edge case (§5.2.6) resolves implicitly. When backslash joining merges a command - line with a line containing a fence opener, `next_logical()` produces the merged text and - `classify_line` detects the fence naturally. No special-casing in the orchestrator. -- `raw` is always correct. Slicing from the retained `physical_lines` vector guarantees the exact - normalized source text including backslashes. -- Text block content stores physical lines, making round-trip reconstruction (§9.4) straightforward. - A formatter can use `content` directly without needing to reverse the joining process. -- Two independent ID counters match the spec's independent sequences exactly. - -**Neutral:** - -- The `physical_lines: Vec<&str>` borrows from the normalized string, which must live for the - duration of the parse. This is trivially satisfied since `parse_document` owns the normalized - string on the stack. -- An owned `Vec` copy is created for the `LineJoiner`. This doubles memory for the line - storage, but inputs are small and the clarity benefit is significant. - -**Bad:** - -- Text block content containing backslash-continued lines will include the trailing backslash on - each physical line. A consumer expecting "clean" joined text from text blocks would need to - re-join. This is the correct behavior for round-trip fidelity but may surprise consumers who - expect text blocks to contain joined content. This tradeoff is accepted because ranges already use - physical line numbers, and consistency between range and content is more important than consumer - convenience. -- The `LineJoiner` allocates a new `String` for every logical line produced by `next_logical()`. For - inputs with many non-continued lines, this is one allocation per line. Acceptable for - chatops-scale inputs. - -## Pros and Cons of the Options - -### Option 1: Pull-model LineJoiner with explicit two-state `while` loop - -The `LineJoiner` exposes `next_logical()` (joins) and `next_physical()` (raw) on a shared cursor. -The orchestrator drives it from a `while` loop matching on `ParserState`. - -- Good, because the state machine directly controls which consumption mode is active, making fence - immunity explicit and auditable. -- Good, because the shared cursor guarantees no lines are skipped or double-consumed at the - Idle/InFence transition boundary. -- Good, because an explicit loop is more readable than iterator chains when the data source changes - mid-stream (architecture doc §9.2). -- Good, because incremental emission is trivially satisfied since each element is finalized inline. -- Neutral, because it requires the orchestrator to own the loop and match arms, which is more code - than a fold but more explicit. - -### Option 2: Pure iterator chain with `scan`/`flat_map` adapters - -Express the entire pipeline as a chain of iterator adapters, using `scan` to carry `ParserState` and -`flat_map` to switch between logical and physical line sources. - -- Good, because it would be concise and leverage Rust's iterator optimizations (lazy evaluation, - potential vectorization). -- Bad, because switching the source of lines (logical vs physical) mid-stream requires either - duplicating the joiner or using interior mutability and `Peekable` with complex lookahead. -- Bad, because lifetime and borrowing constraints make it difficult to hold a mutable reference to - the joiner inside a `scan` closure while also producing items. -- Bad, because the control flow for fence immunity (switching consumption modes) is obscured inside - adapter closures, making the code harder to audit against the spec. - -### Option 3: Two-pass approach (locate fence boundaries first, then join) - -First pass: scan all physical lines to identify fence-open/close pairs and their line ranges. Second -pass: apply line joining only to regions outside fences, then run the state machine on the mixed -result. - -- Good, because fence immunity is guaranteed by construction. The joiner only ever sees non-fence - lines. -- Bad, because it violates §9.2 (incremental emission). The first pass must buffer all fence - boundary positions before any command can be emitted. -- Bad, because identifying fence boundaries requires knowing which lines are command lines (to find - fence openers), which means the first pass duplicates much of the classification logic. -- Bad, because it requires two iterations over the input, violating the single-forward-pass - principle (§1.1). - -### Option 4: Zero-copy slice tracking - -Represent joined logical lines as `Vec<(start_byte, end_byte)>` ranges into the normalized input -string, avoiding `String` allocation entirely. - -- Good, because it eliminates allocation for joined lines, potentially improving throughput on large - inputs. -- Bad, because backslash removal creates non-contiguous content. A joined line spanning physical - lines `"a\"` and `"b"` maps to bytes `[0..1]` + space + `[3..4]`, requiring a non-contiguous slice - assembly that complicates all downstream consumers. -- Bad, because `classify_line` and other downstream functions expect `&str`, requiring either - on-the-fly assembly or API changes throughout the pipeline. -- Neutral, because inputs are small (chatops messages, git comments). Profiling has not shown - allocation as a bottleneck. Documented as a future optimization path if profiling reveals a need. - -### Option 5: Mutable `String` builder for `raw` field - -Maintain a mutable `String` that accumulates physical lines (with `\n` separators) as they are -consumed, becoming the `raw` field when the command is finalized. - -- Good, because it avoids the post-hoc slice-and-join from `physical_lines`. -- Bad, because the builder must be reset for each command and coordinated with the transition - between Idle and InFence states. For fenced commands, `accept_line` already accumulates - `raw_lines` on `PendingCommand`, so the builder would duplicate that responsibility. -- Bad, because it obscures the source of truth. The spec defines `raw` as the exact normalized - source text, which is more clearly expressed by slicing the immutable `physical_lines` vector. - -### Option 6: Single shared counter for command and text block IDs - -Use one global counter and assign IDs like `cmd-0`, `text-1`, `cmd-2` in encounter order. - -- Good, because it simplifies bookkeeping to a single `usize`. -- Bad, because it violates §7, which specifies that command IDs and text block IDs are independent - zero-based sequences. The spec examples show `cmd-0` and `text-0` coexisting, not `cmd-0` and - `text-1`. - -### Option 7: Text block content from joined logical lines - -Store the `LogicalLine.text` (post-join, backslash removed) as text block content. - -- Good, because consumers see "clean" text without trailing backslashes or continuation artifacts. -- Bad, because the range already uses physical line numbers. A text block with - `range: {start_line: 0, end_line: 1}` but content of one joined line creates an inconsistency: the - range says two physical lines but the content has no `\n` separator. -- Bad, because round-trip reconstruction (§9.4) becomes lossy. A formatter cannot distinguish - between a two-physical-line text region that was joined (backslash removed) and a single physical - line, because both would produce the same content string. -- Bad, because it creates an asymmetry with commands: `command.raw` stores physical lines (pre-join) - but `textblock.content` would store logical lines (post-join). - -### Option 8: Text block content from raw physical lines - -Feed `physical_lines[idx]` (pre-join, backslashes retained) into `start_text`/`append_text` for each -physical line covered by a logical line. - -- Good, because content and range are consistent. A range spanning physical lines 0-1 will always - have content with exactly one `\n` separator. -- Good, because round-trip reconstruction is lossless. The formatter can reproduce the exact - normalized input from the content and range of each text block and command. -- Good, because it is symmetric with `command.raw`, which also stores physical lines. -- Neutral, because text block content for continuation lines will contain trailing backslashes. - Consumers expecting "clean" text must handle this, but the spec does not promise that text blocks - strip join markers. - -## More Information - -### Spec references - -- §1.1 Design Principles (single forward pass, incremental emission, deterministic, total) -- §2.1 Line Ending Normalization -- §2.2 Line Joining (Backslash Continuation) -- §2.2.1 Physical Line Tracking -- §2.2.2 Trailing Backslash at EOF -- §2.3 Fence Immunity -- §3 Command Detection -- §4 Parser States (Idle, InFence) -- §5.1 Single-Line Mode -- §5.2 Fence Mode (opener, body, lifetime, closer, unclosed, joining around fences) -- §6 Text Blocks -- §7 Multiple Commands and Ordering (independent ID sequences) -- §8.2 The `raw` Field -- §9.2 Incremental Emission -- §9.4 Roundtrip Fidelity Invariant - -### Architecture references - -- parser-architecture.md §3.2 `documentparse.rs` pipeline description -- parser-architecture.md §5 State Machine (Idle, InFence) -- parser-architecture.md §9.1 Logical Line Intermediate Type (allocated vs zero-copy) -- parser-architecture.md §9.2 Iterator vs Imperative Loop - -### Design council findings - -Three independent model analyses were conducted. All three converged on Option 1 (pull-model with -explicit loop) and confirmed that Option 3 (two-pass) violates incremental emission. The key -divergence was around text block content: one model identified a potential bug where physical lines -with trailing backslashes would appear in text block content, which after analysis was determined to -be the correct behavior for round-trip fidelity (Option 8). Another model independently confirmed -that the fence-opener edge case (§5.2.6) resolves implicitly through separation of concerns between -the joiner and classifier, requiring no special handling in the orchestrator. diff --git a/parser/Cargo.toml b/engine/Cargo.toml similarity index 100% rename from parser/Cargo.toml rename to engine/Cargo.toml diff --git a/parser/moon.yml b/engine/moon.yml similarity index 100% rename from parser/moon.yml rename to engine/moon.yml diff --git a/parser/src/classify.rs b/engine/src/classify.rs similarity index 100% rename from parser/src/classify.rs rename to engine/src/classify.rs diff --git a/parser/src/fence.rs b/engine/src/fence.rs similarity index 100% rename from parser/src/fence.rs rename to engine/src/fence.rs diff --git a/parser/src/integration_tests/mod.rs b/engine/src/integration_tests/mod.rs similarity index 100% rename from parser/src/integration_tests/mod.rs rename to engine/src/integration_tests/mod.rs diff --git a/parser/src/integration_tests/proptest.rs b/engine/src/integration_tests/proptest.rs similarity index 100% rename from parser/src/integration_tests/proptest.rs rename to engine/src/integration_tests/proptest.rs diff --git a/parser/src/join.rs b/engine/src/join.rs similarity index 100% rename from parser/src/join.rs rename to engine/src/join.rs diff --git a/parser/src/lib.rs b/engine/src/lib.rs similarity index 100% rename from parser/src/lib.rs rename to engine/src/lib.rs diff --git a/parser/src/mod.rs b/engine/src/mod.rs similarity index 100% rename from parser/src/mod.rs rename to engine/src/mod.rs diff --git a/parser/src/normalize.rs b/engine/src/normalize.rs similarity index 100% rename from parser/src/normalize.rs rename to engine/src/normalize.rs diff --git a/parser/src/parse.rs b/engine/src/parse.rs similarity index 100% rename from parser/src/parse.rs rename to engine/src/parse.rs diff --git a/parser/src/single_line.rs b/engine/src/single_line.rs similarity index 100% rename from parser/src/single_line.rs rename to engine/src/single_line.rs diff --git a/parser/src/text.rs b/engine/src/text.rs similarity index 100% rename from parser/src/text.rs rename to engine/src/text.rs diff --git a/parser/src/types.rs b/engine/src/types.rs similarity index 100% rename from parser/src/types.rs rename to engine/src/types.rs diff --git a/parser/tests/orchestration_tests.rs b/engine/tests/orchestration_tests.rs similarity index 100% rename from parser/tests/orchestration_tests.rs rename to engine/tests/orchestration_tests.rs diff --git a/release-please-config.json b/release-please-config.json index 2196351..eb28bbd 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -3,16 +3,6 @@ "plugins": [ { "type": "cargo-workspace" - }, - { - "type": "linked-versions", - "groupName": "parser-group", - "components": [ - "parser", - "wasm-javascript", - "wasm-wasi", - "spec" - ] } ], "release-type": "rust", @@ -26,45 +16,6 @@ "parser": { "release-type": "rust", "component": "parser" - }, - "wasm-javascript": { - "release-type": "rust", - "component": "wasm-javascript" - }, - "wasm-wasi": { - "release-type": "rust", - "component": "wasm-wasi" - }, - "spec": { - "release-type": "simple", - "component": "spec" - }, - "riff-cli": { - "path": "riff-cli", - "release-type": "rust", - "component": "riff" - }, - "slash-rust": { - "release-type": "rust", - "component": "slash-rust" - }, - "slash-javascript": { - "release-type": "node", - "component": "slash-javascript" - }, - "slash-web": { - "release-type": "node", - "component": "slash-web" - }, - "slash-python": { - "release-type": "python", - "component": "slash-python" - }, - "slash-go": { - "release-type": "go", - "component": "slash-go", - "tag-separator": "/", - "include-v-in-tag": true } } } diff --git a/slash-go/moon.yml b/slash-go/moon.yml index 4980cfe..07cdd13 100644 --- a/slash-go/moon.yml +++ b/slash-go/moon.yml @@ -1,4 +1,3 @@ $schema: "https://moonrepo.dev/schemas/project.json" language: "go" -type: "library" tags: ["sdk"] diff --git a/slash-javascript/moon.yml b/slash-javascript/moon.yml index 75ba71a..90e7a3e 100644 --- a/slash-javascript/moon.yml +++ b/slash-javascript/moon.yml @@ -1,6 +1,5 @@ $schema: "https://moonrepo.dev/schemas/project.json" language: "typescript" -type: "library" tags: ["sdk", "npm"] tasks: diff --git a/slash-python/moon.yml b/slash-python/moon.yml index c58c0fc..4aa3552 100644 --- a/slash-python/moon.yml +++ b/slash-python/moon.yml @@ -1,6 +1,5 @@ $schema: "https://moonrepo.dev/schemas/project.json" language: "python" -type: "library" tags: ["sdk"] tasks: diff --git a/slash-rust/moon.yml b/slash-rust/moon.yml index eb08538..4bfd25f 100644 --- a/slash-rust/moon.yml +++ b/slash-rust/moon.yml @@ -1,6 +1,5 @@ $schema: "https://moonrepo.dev/schemas/project.json" language: "rust" -type: "library" tags: ["sdk"] tasks: diff --git a/slash-web/moon.yml b/slash-web/moon.yml index 75ba71a..90e7a3e 100644 --- a/slash-web/moon.yml +++ b/slash-web/moon.yml @@ -1,6 +1,5 @@ $schema: "https://moonrepo.dev/schemas/project.json" language: "typescript" -type: "library" tags: ["sdk", "npm"] tasks: diff --git a/wasm-javascript/moon.yml b/wasm-javascript/moon.yml index 3263f79..703a8e8 100644 --- a/wasm-javascript/moon.yml +++ b/wasm-javascript/moon.yml @@ -1,4 +1,3 @@ $schema: "https://moonrepo.dev/schemas/project.json" language: "rust" -type: "library" tags: ["wasm"] diff --git a/wasm-wasi/moon.yml b/wasm-wasi/moon.yml index 3263f79..703a8e8 100644 --- a/wasm-wasi/moon.yml +++ b/wasm-wasi/moon.yml @@ -1,4 +1,3 @@ $schema: "https://moonrepo.dev/schemas/project.json" language: "rust" -type: "library" tags: ["wasm"] diff --git a/website/moon.yml b/website/moon.yml index d2ebd6b..4706b6b 100644 --- a/website/moon.yml +++ b/website/moon.yml @@ -1,6 +1,5 @@ $schema: "https://moonrepo.dev/schemas/project.json" language: "typescript" -type: "application" tags: ["docs"] workspace: From 4476ab9523b4379af7bdd470ae40adad05d8082f Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Sun, 22 Mar 2026 23:09:28 -0600 Subject: [PATCH 12/13] ci(engine): automated publishing tweaks --- .moon/partials/publish-crate.yml | 23 +++++++++++++++++++++++ .moon/workspace.yml | 14 +++++++------- 2 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 .moon/partials/publish-crate.yml diff --git a/.moon/partials/publish-crate.yml b/.moon/partials/publish-crate.yml new file mode 100644 index 0000000..446a9f8 --- /dev/null +++ b/.moon/partials/publish-crate.yml @@ -0,0 +1,23 @@ +tasks: + publish: + command: cargo + args: + - publish + - --token + - $CARGO_REGISTRY_TOKEN + inputs: + - src/**/*.rs + - Cargo.toml + - Cargo.lock + options: + persistent: false + + review: + command: cargo + args: + - publish + - --dry-run + inputs: + - src/**/*.rs + - Cargo.toml + - Cargo.lock diff --git a/.moon/workspace.yml b/.moon/workspace.yml index d1fe1cb..b3c881b 100644 --- a/.moon/workspace.yml +++ b/.moon/workspace.yml @@ -1,13 +1,13 @@ $schema: "https://moonrepo.dev/schemas/workspace.json" projects: - - "wit" - - "parser" - - "wasm-*" - - "slash-*" - - "riff-cli" - - "website" - - "docs" + - "engine" + # - "wit" + # - "wasm-*" + # - "slash-*" + # - "riff-cli" + # - "website" + # - "docs" constraints: enforceLayerRelationships: true From c1dfcd72603ec5a03a955c3a279c3723771049fd Mon Sep 17 00:00:00 2001 From: tomdavidson Date: Tue, 24 Mar 2026 13:17:30 -0600 Subject: [PATCH 13/13] refactor: restructure tests into layered architecture; fix WSP and fence bugs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Test restructuring: - Split monolithic integration_tests/mod.rs into focused modules: normalize_join, normalize_join_classify, spec_examples - Move parse_document orchestration tests to engine/tests/parse_document.rs - Extract shared test helpers (valid_command_name, feed_body) into test_helper.rs - Rename spec example tests from A-numbering to B-numbering (Appendix B) - Scope parse.rs #[cfg(test)] to private internals only (split_physical_lines, fold_physical_lines, accumulate_text, flush_text, ParseCtx::into_result) Bug fixes: - classify: replace char::is_whitespace with is_wsp (SP/HTAB only) per Engine Spec §6 for leading WSP, name/args separator, and fence lang-id - classify: fence_lang now returns None for empty string after trimming (RFC §5.2.1 "if non-empty") - fence: is_fence_closer uses trim_matches(is_wsp) instead of .trim() per Engine Spec §6 - fence: unclosed fence warning type changed from "unclosed-fence" to "unclosed_fence" per Engine Spec §11/§16.3 - join: remove space insertion on join, matching Engine Spec §7.1 POSIX backslash-newline semantics - parse: hoist raw/range construction above match in start_new_command New test coverage: - NBSP/EM SPACE rejection in classify leading WSP and fence closer - NBSP/EM SPACE acceptance in fence lang-id (NONWSP per ABNF) - Tab handling for leading WSP, name separator, and fence closer - Fence closer edge cases (interspersed spaces, 4-tick openers, tildes) - Fence header/lang passthrough, warning type snakecase - Property tests for fence accumulation invariants --- .gitignore | 2 + Cargo.toml | 2 +- engine/src/classify.rs | 297 +++++++++- engine/src/fence.rs | 342 ++++++++---- engine/src/integration_tests/mod.rs | 436 +-------------- .../src/integration_tests/normalize_join.rs | 61 ++ .../normalize_join_classify.rs | 127 +++++ engine/src/integration_tests/proptest.rs | 57 -- engine/src/integration_tests/spec_examples.rs | 185 +++++++ engine/src/join.rs | 282 ++++++++-- engine/src/lib.rs | 6 +- engine/src/normalize.rs | 105 ++-- engine/src/parse.rs | 417 +++----------- engine/src/single_line.rs | 99 ++-- engine/src/test_helper.rs | 16 + engine/src/text.rs | 48 ++ engine/src/types.rs | 3 +- engine/tests/orchestration_tests.rs | 322 ----------- engine/tests/parse_document.rs | 520 ++++++++++++++++++ 19 files changed, 1936 insertions(+), 1391 deletions(-) create mode 100644 engine/src/integration_tests/normalize_join.rs create mode 100644 engine/src/integration_tests/normalize_join_classify.rs delete mode 100644 engine/src/integration_tests/proptest.rs create mode 100644 engine/src/integration_tests/spec_examples.rs create mode 100644 engine/src/test_helper.rs delete mode 100644 engine/tests/orchestration_tests.rs create mode 100644 engine/tests/parse_document.rs diff --git a/.gitignore b/.gitignore index 854b856..8d88555 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +/scratch/ + # Build artifacts /target/ /slash-parser-js/pkg/ diff --git a/Cargo.toml b/Cargo.toml index 0b82c27..c0e4d66 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] resolver = "2" members = [ - "parser", + "engine", ] [workspace.package] diff --git a/engine/src/classify.rs b/engine/src/classify.rs index 090d809..497291d 100644 --- a/engine/src/classify.rs +++ b/engine/src/classify.rs @@ -45,15 +45,17 @@ fn find_fence_opener(s: &str) -> Option<(usize, usize)> { .map(|(start, chunk)| (start, chunk.len())) } +fn is_wsp(c: char) -> bool { c == ' ' || c == '\t' } + fn try_parse_command(line: &str) -> Option { - let trimmed = line.trim_start(); + let trimmed = line.trim_start_matches(is_wsp); if !trimmed.starts_with('/') { return None; } let without_slash = &trimmed[1..]; - let mut parts = without_slash.splitn(2, char::is_whitespace); + let mut parts = without_slash.splitn(2, is_wsp); let name_raw = parts.next().filter(|n| !n.is_empty())?; // RFC §4.1: [a-z]([a-z0-9-]*[a-z0-9])? @@ -65,15 +67,23 @@ fn try_parse_command(line: &str) -> Option { return None; } + // RFC §4.1: "Multi-character names MUST NOT end with a hyphen." + // Engine Spec §15.1: trailing hyphen prohibition added in v0.5.0. + if name_raw.len() > 1 && name_raw.ends_with('-') { + return None; + } + let name = name_raw.to_string(); - let rest = parts.next().unwrap_or("").trim_start(); + let rest = parts.next().unwrap_or("").trim_start_matches(is_wsp); // RFC §5.2.1: detect first occurrence of 3+ backticks anywhere in the args. if let Some((fence_start, fence_count)) = find_fence_opener(rest) { let header_text = rest[..fence_start].trim_end().to_string(); let after_ticks = &rest[fence_start + fence_count..]; - let after_trimmed = after_ticks.trim(); - let fence_lang = if !after_trimmed.is_empty() && !after_trimmed.contains(char::is_whitespace) { + let after_trimmed = after_ticks.trim_matches(is_wsp); + let fence_lang = if !after_trimmed.is_empty() + && !after_trimmed.contains(is_wsp) + { Some(after_trimmed.to_string()) } else { None @@ -103,6 +113,7 @@ fn try_parse_command(line: &str) -> Option { #[cfg(test)] mod tests { use super::*; + use crate::test_helper::valid_command_name; // ========================================================================= // find_fence_opener — internal helper @@ -382,17 +393,53 @@ mod tests { assert_eq!(classify_line("/cmd-"), LineKind::Text); } + + #[test] +fn fence_lang_with_nbsp_is_valid_token() { + // RFC Appendix A: NONWSP = %x21-7E / %x80-10FFFF — U+00A0 (NBSP) + // is above %x7E in codepoint but falls in %x80-10FFFF, so it is + // a valid non-whitespace character per the ABNF. A lang-id + // consisting of a single token containing NBSP must be accepted. + // NOTE: will FAIL until fence_lang extraction uses is_wsp instead + // of char::is_whitespace for the single-token check. + let h = try_parse_command("/cmd ``` nb\u{00A0}sp").unwrap(); + assert_eq!(h.fence_lang, Some("nb\u{00A0}sp".to_string())); +} + +#[test] +fn fence_lang_with_em_space_is_valid_token() { + // RFC Appendix A: NONWSP includes %x80-10FFFF. U+2003 (EM SPACE) + // is not whitespace per spec, so it does not split a lang-id token. + // NOTE: will FAIL until fence_lang extraction uses is_wsp instead + // of char::is_whitespace for the single-token check. + let h = try_parse_command("/cmd ``` em\u{2003}sp").unwrap(); + assert_eq!(h.fence_lang, Some("em\u{2003}sp".to_string())); +} + +#[test] +fn fence_lang_split_by_space_is_none() { + // RFC §5.2.1: lang-id must be "a single token (no internal + // whitespace)." SP (U+0020) is WSP per ABNF, so two words + // separated by SP means no valid lang-id. + let h = try_parse_command("/cmd ``` two words").unwrap(); + assert_eq!(h.fence_lang, None); +} + +#[test] +fn fence_lang_split_by_tab_is_none() { + // RFC §5.2.1 + Appendix A: HTAB (U+0009) is WSP. A tab between + // tokens means the string is not a single token. + let h = try_parse_command("/cmd ``` two\twords").unwrap(); + assert_eq!(h.fence_lang, None); +} + + // ========================================================================= // Property tests // ========================================================================= use proptest::prelude::*; - // RFC §4.1: [a-z]([a-z0-9-]*[a-z0-9])? - fn valid_command_name() -> impl Strategy { - "[a-z][a-z0-9\\-]{0,20}".prop_map(|s| s) - } - fn arbitrary_line() -> impl Strategy { prop_oneof![ "[a-zA-Z0-9 !.,]{0,80}", @@ -406,6 +453,210 @@ mod tests { ] } + // ========================================================================= + // try_parse_command — trailing hyphen edge cases + // RFC §4.1: "Multi-character names MUST NOT end with a hyphen." + // ========================================================================= + + #[test] + fn single_hyphen_after_letter_returns_none() { + // RFC §4.1: "a-" is two characters ending with hyphen, invalid. + assert!(try_parse_command("/a-").is_none()); + } + + #[test] + fn trailing_hyphen_with_fence_returns_none() { + // RFC §4.1: trailing hyphen prohibition applies even when args + // contain a fence opener. The name is invalid before args are parsed. + assert!(try_parse_command("/cmd- ```json").is_none()); + } + + #[test] + fn internal_hyphens_valid() { + // RFC §4.1: hyphens between alphanumerics are fine. Only trailing + // hyphens are prohibited. + let h = try_parse_command("/a-b-c args").unwrap(); + assert_eq!(h.name, "a-b-c"); + } + + // ========================================================================= + // try_parse_command — special characters after slash + // RFC §4.5: invalid slash lines are text. + // ========================================================================= + + #[test] + fn slash_exclamation_returns_none() { + // RFC §4.5: "/foo!bar" does not match [a-z0-9-]. + assert!(try_parse_command("/foo!bar").is_none()); + } + + #[test] + fn slash_at_sign_returns_none() { + // RFC §4.5: "/@cmd" starts with non-LCALPHA after slash. + assert!(try_parse_command("/@cmd").is_none()); + } + + #[test] + fn slash_dot_returns_none() { + // RFC §4.5: "/cmd.sub" contains '.', not in [a-z0-9-]. + assert!(try_parse_command("/cmd.sub").is_none()); + } + + #[test] + fn slash_emoji_returns_none() { + // RFC §4.1: command name must be ASCII lowercase. Non-ASCII -> invalid. + assert!(try_parse_command("/🚀").is_none()); + } + + // ========================================================================= + // try_parse_command — leading whitespace: tab handling + // RFC §4.2 + Engine Spec §6: WSP is SP (U+0020) and HTAB (U+0009) only. + // ========================================================================= + + #[test] + fn leading_tab_stripped_for_detection() { + // RFC §4.2: "first non-whitespace character is /" + // Engine Spec §6: HTAB is valid leading whitespace. + let h = try_parse_command("\t/cmd arg").unwrap(); + assert_eq!(h.name, "cmd"); + assert_eq!(h.header_text, "arg"); + } + + #[test] + fn leading_mixed_space_and_tab() { + // Engine Spec §6: both SP and HTAB are valid leading whitespace. + let h = try_parse_command(" \t /cmd arg").unwrap(); + assert_eq!(h.name, "cmd"); + } + + #[test] + fn raw_preserves_leading_tab() { + // RFC §7.1: raw is the exact source text including leading whitespace. + let h = try_parse_command("\t/cmd arg").unwrap(); + assert_eq!(h.raw, "\t/cmd arg"); + } + + // ========================================================================= + // try_parse_command — exotic Unicode whitespace must NOT be treated as WSP + // Engine Spec §6: "The engine MUST NOT use Rust's char::is_whitespace()" + // NOTE: These tests assert the CORRECT spec behavior. The current + // implementation uses trim_start() which strips Unicode WSP, so these + // will FAIL until the code is updated to use is_wsp(). + // ========================================================================= + + #[test] + fn non_breaking_space_leading_is_not_stripped() { + // Engine Spec §6: U+00A0 (NO-BREAK SPACE) is NOT whitespace per spec. + // A line starting with NBSP then "/" is not a command because the first + // non-WSP character is NBSP, not "/". + assert_eq!(classify_line("\u{00A0}/cmd arg"), LineKind::Text); + } + + #[test] + fn em_space_leading_is_not_stripped() { + // Engine Spec §6: U+2003 (EM SPACE) is NOT whitespace per spec. + assert_eq!(classify_line("\u{2003}/cmd arg"), LineKind::Text); + } + + // ========================================================================= + // try_parse_command — name/args separator whitespace + // RFC §4.3 + Engine Spec §6: separator is SP or HTAB only. + // ========================================================================= + + #[test] + fn tab_separates_name_from_args() { + // Engine Spec §6: HTAB is valid separator between name and arguments. + let h = try_parse_command("/cmd\targ1 arg2").unwrap(); + assert_eq!(h.name, "cmd"); + assert_eq!(h.header_text, "arg1 arg2"); + } + + #[test] + fn non_breaking_space_does_not_separate_name() { + // Engine Spec §6: U+00A0 is NOT a valid separator. It becomes part of + // the command name, which then fails validation (non-ASCII). + // NOTE: will FAIL until code replaces char::is_whitespace with is_wsp(). + assert!(try_parse_command("/cmd\u{00A0}arg").is_none()); + } + + // ========================================================================= + // try_parse_command — fence opener touching header text (no space) + // RFC §5.2.1: "first occurrence of three or more consecutive backtick + // characters" in the arguments portion. + // ========================================================================= + + #[test] + fn fence_opener_directly_touching_alpha_header() { + // RFC §5.2.1: backticks do not need to be space-separated from header. + // "header```json" — header is "header", fence opens. + let h = try_parse_command("/cmd header```json").unwrap(); + assert_eq!(h.header_text, "header"); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, Some("json".to_string())); + assert_eq!(h.fence_backtick_count, 3); + } + + #[test] + fn fence_opener_touching_numeric_header() { + // RFC §5.2.1: first occurrence rule applies regardless of preceding chars. + let h = try_parse_command("/cmd 123```").unwrap(); + assert_eq!(h.header_text, "123"); + assert_eq!(h.mode, ArgumentMode::Fence); + assert_eq!(h.fence_lang, None); + } + + // ========================================================================= + // try_parse_command — fence language identifier edge cases + // RFC §5.2.1: "single token (no internal whitespace)" + // ========================================================================= + + #[test] + fn fence_lang_with_digits() { + // RFC §5.2.1: lang-id is any NONWSP token. Digits are valid. + let h = try_parse_command("/cmd ```es2024").unwrap(); + assert_eq!(h.fence_lang, Some("es2024".to_string())); + } + + #[test] + fn fence_lang_with_hyphen() { + // RFC §5.2.1: lang-id can contain hyphens (e.g., "utf-8"). + let h = try_parse_command("/cmd ```utf-8").unwrap(); + assert_eq!(h.fence_lang, Some("utf-8".to_string())); + } + + #[test] + fn fence_lang_with_dots() { + // RFC §5.2.1: lang-id is any single non-whitespace token. + let h = try_parse_command("/cmd ```.gitignore").unwrap(); + assert_eq!(h.fence_lang, Some(".gitignore".to_string())); + } + + #[test] + fn fence_lang_empty_after_trailing_whitespace() { + // RFC §5.2.1: "trimmed of leading whitespace, if non-empty" — if only + // whitespace follows the backticks, fence_lang is None. + let h = try_parse_command("/cmd ``` ").unwrap(); + assert_eq!(h.fence_lang, None); + } + + // ========================================================================= + // classify_line — empty and whitespace-only lines + // RFC §6.3: non-command lines are text. + // ========================================================================= + + #[test] + fn empty_line_is_text() { + // RFC §6.3: blank lines are text lines. + assert_eq!(classify_line(""), LineKind::Text); + } + + #[test] + fn whitespace_only_line_is_text() { + // RFC §6.3: whitespace-only lines are text (no "/" present). + assert_eq!(classify_line(" "), LineKind::Text); + assert_eq!(classify_line("\t\t"), LineKind::Text); + } + proptest! { // RFC §8.2 item 4: parser is a total function. classify_line must // never panic on any input. @@ -486,16 +737,20 @@ mod tests { prop_assert_eq!(h.fence_backtick_count, 3 + extra); } } + // ========================================================================= + // Property tests — trailing hyphen rejection + // ========================================================================= + + + // RFC §4.1: names ending with hyphen are always rejected. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn trailing_hyphen_always_rejected( + prefix in "[a-z][a-z0-9]{0,10}", + args in "[a-zA-Z0-9 ]{0,40}" + ) { + let input = format!("/{prefix}- {args}"); + prop_assert!(matches!(classify_line(&input), LineKind::Text)); + } } } - -/* -| Spec Section | Gap | -| ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| RFC §4.1 (ABNF command-name) | The code does not reject trailing hyphens. The regex in try_parse_command still uses the old v0.3.1 pattern [a-z][a-z0-9-]*. The new tests I added (trailing_hyphen_returns_none) will FAIL until the implementation is fixed to match [a-z]([a-z0-9-]*[a-z0-9])?. This is a code bug, not just a test gap. | -| RFC §4.2 | Tab (HTAB) as leading whitespace. The code uses trim_start() which trims all Unicode whitespace. Engine Spec §6 requires only SP and HTAB. No test uses \\t/cmd to verify tab handling, and no test checks that exotic Unicode whitespace (e.g. U+00A0) is NOT stripped. | -| RFC §4.3 | Whitespace separator between name and args uses char::is_whitespace (splits on any Unicode WSP). Engine Spec §6 mandates only SP/HTAB. No test verifies that a non-breaking space between name and args is handled correctly. | -| RFC §5.2.1 | No test for fence opener with exactly the backticks touching the header text with no space (e.g., /cmd header```json where header runs directly into backticks). The mid-args test uses -c```json but not a pure alpha header. | -| RFC §5.2.1 | No test for lang-id that contains digits or hyphens (e.g., utf-8, es2024). | -| Engine Spec §6 | The is_wsp() helper mandate is not followed in the implementation. trim_start(), char::is_whitespace, and trim() are used throughout. This is a systemic code issue. | -| RFC §4.5 | No test for special characters after slash (e.g., /foo!bar, /@cmd). |*/ diff --git a/engine/src/fence.rs b/engine/src/fence.rs index f9809fa..23f63dc 100644 --- a/engine/src/fence.rs +++ b/engine/src/fence.rs @@ -10,7 +10,7 @@ pub enum FenceResult { /// In-progress fenced command being assembled from physical lines. /// /// Existence of this value means the fence is open. No `is_open` field needed. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct PendingFence { pub id: usize, pub name: String, @@ -53,8 +53,10 @@ pub fn accept_fence_line( } } +fn is_wsp(c: char) -> bool { c == ' ' || c == '\t' } + fn is_fence_closer(line: &str, opener_count: usize) -> bool { - let trimmed = line.trim(); + let trimmed = line.trim_matches(is_wsp); !trimmed.is_empty() && trimmed.chars().all(|c| c == '`') && trimmed.len() >= opener_count } @@ -63,7 +65,7 @@ pub fn finalize_fence(fence: PendingFence, unclosed: bool) -> (Command, Vec= opener count. + // Opener is 4, so 3 backticks do NOT close. + let fence = make_fence("cmd", 4, 0, 0); + let (_, res) = accept_fence_line(fence, 1, "```"); + assert_eq!(res, FenceResult::Consumed); + // But 4 does: + let fence2 = make_fence("cmd", 4, 0, 0); + let (_, res2) = accept_fence_line(fence2, 1, "````"); + assert_eq!(res2, FenceResult::Completed); + } + + #[test] + fn closer_tilde_line_does_not_close() { + // RFC §5.2: tilde fences are not recognized. A line of tildes + // is not a closer regardless of count. + let fence = make_fence("cmd", 3, 0, 0); + let (_, res) = accept_fence_line(fence, 1, "~~~"); + assert_eq!(res, FenceResult::Consumed); + } + + // ========================================================================= + // open_fence — header_text and fence_lang passthrough + // Engine Spec §9.1 + // ========================================================================= + + #[test] + fn open_fence_preserves_header_text() { + // Engine Spec §9.1: header_text from CommandHeader is stored. + let raw = "/mcp call_tool write_file ```json".to_string(); + let header = CommandHeader { + raw: raw.clone(), + name: "mcp".to_string(), + header_text: "call_tool write_file".to_string(), + mode: ArgumentMode::Fence, + fence_lang: Some("json".to_string()), + fence_backtick_count: 3, + }; + let range = LineRange { start_line: 0, end_line: 0 }; + let fence = open_fence(header, raw, 0, range); + assert_eq!(fence.header_text, "call_tool write_file"); + assert_eq!(fence.fence_lang, Some("json".to_string())); + } + + #[test] + fn open_fence_range_from_logical_line() { + // Engine Spec §9.1 + §3.6: range seeded from the logical line's + // physical span. A joined opener spanning lines 2-4 should set both. + let raw = "/cmd ```".to_string(); + let header = CommandHeader { + raw: raw.clone(), + name: "cmd".to_string(), + header_text: String::new(), + mode: ArgumentMode::Fence, + fence_lang: None, + fence_backtick_count: 3, + }; + let range = LineRange { start_line: 2, end_line: 4 }; + let fence = open_fence(header, raw, 0, range); + assert_eq!(fence.start_line, 2); + assert_eq!(fence.end_line, 4); + } + + // ========================================================================= + // finalize_fence — warning type string (snake_case) + // Engine Spec §11 + §16.3: "unclosed_fence" (snake_case) + // ========================================================================= + + #[test] + fn unclosed_fence_warning_type_is_snake_case() { + // Engine Spec §11: warning types use snake_case. + // Engine Spec §16.3: migrated from "unclosed-fence" to "unclosed_fence". + // NOTE: will FAIL until finalize_fence is updated from "unclosed-fence" + // to "unclosed_fence". + let fence = make_fence("cmd", 3, 0, 0); + let (_, warnings) = finalize_fence(fence, true); + assert_eq!(warnings[0].wtype, "unclosed_fence"); + } + + // ========================================================================= + // finalize_fence — header passthrough to Command + // Engine Spec §3.3 + // ========================================================================= + + #[test] + fn finalize_preserves_header_in_command() { + // Engine Spec §3.3: CommandArguments.header is the inline argument + // text before the fence opener. + let raw = "/mcp call_tool ```json".to_string(); + let header = CommandHeader { + raw: raw.clone(), + name: "mcp".to_string(), + header_text: "call_tool".to_string(), + mode: ArgumentMode::Fence, + fence_lang: Some("json".to_string()), + fence_backtick_count: 3, + }; + let range = LineRange { start_line: 0, end_line: 0 }; + let fence = open_fence(header, raw, 0, range); + let (fence, _) = accept_fence_line(fence, 1, "body"); + let (fence, _) = accept_fence_line(fence, 2, "```"); + let (cmd, _) = finalize_fence(fence, false); + assert_eq!(cmd.arguments.header, "call_tool"); + } + + // ========================================================================= + // finalize_fence — unclosed fence with zero body lines + // RFC §5.2.4 + // ========================================================================= + + #[test] + fn unclosed_fence_empty_body_produces_warning() { + // RFC §5.2.4: unclosed fence at EOF with zero body lines still + // produces the warning and an empty payload. + let fence = make_fence("cmd", 3, 0, 0); + let (cmd, warnings) = finalize_fence(fence, true); + assert_eq!(warnings.len(), 1); + assert_eq!(cmd.arguments.payload, ""); + } + + // ========================================================================= + // accept_fence_line — body with backtick content (not a closer) + // RFC §5.2.2 / §5.2.3 // ========================================================================= - fn valid_command_name() -> impl Strategy { - // RFC §4.1: [a-z]([a-z0-9-]*[a-z0-9])? - "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) + #[test] + fn body_line_with_backticks_and_text_is_payload() { + // RFC §5.2.3: line must consist SOLELY of backticks to close. + // "```\nsome code\n```" but "let x = `template`;" is body. + let fence = make_fence("cmd", 3, 0, 0); + let (fence, res) = accept_fence_line(fence, 1, "let x = `template`;"); + assert_eq!(res, FenceResult::Consumed); + assert_eq!(fence.payload_lines, vec!["let x = `template`;"]); } + #[test] + fn body_line_with_three_backticks_mid_text_is_payload() { + // RFC §5.2.3: the line must be solely backticks after trimming. + // "code ``` more" has non-backtick chars, so it is payload. + let fence = make_fence("cmd", 3, 0, 0); + let (_, res) = accept_fence_line(fence, 1, "code ``` more"); + assert_eq!(res, FenceResult::Consumed); + } + + // ========================================================================= + // Property tests + // ========================================================================= + proptest! { // Engine Spec §9.3 step 2: raw_lines count = opener + body + closer. #[test] @@ -424,11 +634,7 @@ mod tests { name in valid_command_name(), body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 0..8) ) { - let fence = make_fence(&name, 3, 0, 0); - let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { - let (next, _) = accept_fence_line(f, i + 1, line); - next - }); + let fence = feed_body(make_fence(&name, 3, 0, 0), &body_lines); let (fence, _) = accept_fence_line(fence, body_lines.len() + 1, "```"); prop_assert_eq!(fence.raw_lines.len(), body_lines.len() + 2); } @@ -441,11 +647,7 @@ mod tests { name in valid_command_name(), body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 0..8) ) { - let fence = make_fence(&name, 3, 0, 0); - let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { - let (next, _) = accept_fence_line(f, i + 1, line); - next - }); + let fence = feed_body(make_fence(&name, 3, 0, 0), &body_lines); let (fence, _) = accept_fence_line(fence, body_lines.len() + 1, "```"); let no_closer = !fence.payload_lines.iter().any(|l| { let t = l.trim(); @@ -454,6 +656,7 @@ mod tests { prop_assert!(no_closer); } + // RFC §5.2.3: closer count >= opener count always completes. #[test] #[cfg_attr(feature = "tdd", ignore)] @@ -475,11 +678,8 @@ mod tests { id in 0usize..1000, body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 0..5) ) { - let fence = make_fence(&name, 3, 0, id); - let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { - let (next, _) = accept_fence_line(f, i + 1, line); - next - }); + let fence = feed_body(make_fence(&name, 3, 0, id), &body_lines); + prop_assert_eq!(fence.id, id); } @@ -490,11 +690,7 @@ mod tests { name in valid_command_name(), body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,30}", 1..8) ) { - let fence = make_fence(&name, 3, 0, 0); - let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { - let (next, _) = accept_fence_line(f, i + 1, line); - next - }); + let fence = feed_body(make_fence(&name, 3, 0, 0), &body_lines); let (fence, _) = accept_fence_line(fence, body_lines.len() + 1, "```"); let (_, warnings) = finalize_fence(fence, false); prop_assert!(warnings.is_empty()); @@ -507,14 +703,10 @@ mod tests { name in valid_command_name(), body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 1..5) ) { - let fence = make_fence(&name, 3, 0, 0); - let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { - let (next, _) = accept_fence_line(f, i + 1, line); - next - }); + let fence = feed_body(make_fence(&name, 3, 0, 0), &body_lines); let (_, warnings) = finalize_fence(fence, true); prop_assert_eq!(warnings.len(), 1); - prop_assert_eq!(&warnings[0].wtype, "unclosed-fence"); + prop_assert_eq!(&warnings[0].wtype, "unclosed_fence"); } // RFC §5.2.2: payload = body lines joined with LF, no trailing LF. @@ -524,11 +716,7 @@ mod tests { name in valid_command_name(), body_lines in prop::collection::vec("[a-zA-Z0-9 ]{1,20}", 1..8) ) { - let fence = make_fence(&name, 3, 0, 0); - let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { - let (next, _) = accept_fence_line(f, i + 1, line); - next - }); + let fence = feed_body(make_fence(&name, 3, 0, 0), &body_lines); let (fence, _) = accept_fence_line(fence, body_lines.len() + 1, "```"); let (cmd, _) = finalize_fence(fence, false); prop_assert_eq!(cmd.arguments.payload, body_lines.join("\n")); @@ -541,11 +729,7 @@ mod tests { name in valid_command_name(), body_lines in prop::collection::vec("[a-zA-Z0-9]{1,20}", 0..6) ) { - let fence = make_fence(&name, 3, 0, 0); - let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { - let (next, _) = accept_fence_line(f, i + 1, line); - next - }); + let fence = feed_body(make_fence(&name, 3, 0, 0), &body_lines); let (fence, _) = accept_fence_line(fence, body_lines.len() + 1, "```"); let (cmd, _) = finalize_fence(fence, false); let expected_newlines = body_lines.len() + 1; @@ -554,53 +738,25 @@ mod tests { expected_newlines ); } + + // RFC §5.2.3: variable-width opener requires matching closer width. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn variable_opener_width_requires_matching_closer( + name in valid_command_name(), + opener_extra in 0usize..5, + closer_extra in 0usize..5, + ) { + let opener_count = 3 + opener_extra; + let closer_count = 3 + closer_extra; + let fence = make_fence(&name, opener_count, 0, 0); + let closer = "`".repeat(closer_count); + let (_, res) = accept_fence_line(fence, 1, &closer); + if closer_count >= opener_count { + prop_assert_eq!(res, FenceResult::Completed); + } else { + prop_assert_eq!(res, FenceResult::Consumed); + } + } } } - -// ============================================================================= -// TEST GAPS: spec areas this file's functions touch but are not tested -// ============================================================================= -// -// | Spec Section | Gap | Severity | -// |-----------------------------------|--------------------------------------------------|----------| -// | Engine Spec §11 / RFC §7.4 | WARNING TYPE STRING: code emits "unclosed-fence" | CRITICAL | -// | | (kebab-case) but Engine Spec v0.5.0 requires | | -// | | "unclosed_fence" (snake_case). This is a code bug. | | -// |-----------------------------------|--------------------------------------------------|----------| -// | Engine Spec §6 | WHITESPACE: is_fence_closer uses .trim() which | HIGH | -// | | strips all Unicode WSP. Engine Spec §6 mandates | | -// | | only SP (U+0020) and HTAB (U+0009). No test | | -// | | verifies that U+00A0 or other exotic WSP around | | -// | | backticks does NOT trigger a close. | | -// |-----------------------------------|--------------------------------------------------|----------| -// | RFC §5.2.3 | CLOSER WITH TRAILING BACKSLASH: RFC §5.2.3 notes | MEDIUM | -// | | "If the physical line containing a fence closer | | -// | | ends with a trailing '\', the fence closes | | -// | | normally, and then the '\' triggers a line join." | | -// | | No test for "```\" as a closer line (the backslash | | -// | | check is not this module's job, but the closer | | -// | | detection itself should reject "```\"). | | -// |-----------------------------------|--------------------------------------------------|----------| -// | RFC §5.2.3 | FOUR-BACKTICK OPENER WITH THREE-BACKTICK CLOSER: | LOW | -// | | No test where opener_count=4 and closer has | | -// | | exactly 3 backticks (should NOT close). Only the | | -// | | 3-opener/2-closer case is tested. | | -// |-----------------------------------|--------------------------------------------------|----------| -// | RFC §5.2.2 | PAYLOAD NO TRAILING LF: "The payload MUST NOT | LOW | -// | | include a trailing LF after the last body line." | | -// | | No explicit assertion that payload does not end | | -// | | with \n (implicitly covered by join semantics but | | -// | | not directly asserted). | | -// |-----------------------------------|--------------------------------------------------|----------| -// | RFC §5.2.2 | LINE JOINING IMMUNITY: "Fence body lines are | LOW | -// | | never subject to line joining." This is enforced | | -// | | by the document parser (not this module), but | | -// | | body_preserves_content_verbatim partially covers | | -// | | it by checking trailing backslash survives. | | -// |-----------------------------------|--------------------------------------------------|----------| -// | Engine Spec §9.2 | ACCEPT RESULT REJECTED VARIANT: The Engine Spec | INFO | -// | | defines AcceptResult::Rejected for already-closed | | -// | | commands. This file's PendingFence uses | | -// | | ownership-based design (no is_open field), so | | -// | | Rejected is impossible, but the enum mismatch | | -// | | with the spec should be documented. | | diff --git a/engine/src/integration_tests/mod.rs b/engine/src/integration_tests/mod.rs index 52f1302..0b4354a 100644 --- a/engine/src/integration_tests/mod.rs +++ b/engine/src/integration_tests/mod.rs @@ -1,428 +1,8 @@ -// Layer 2 integration tests: cross-module composition within the application layer. -// -// These tests exercise combinations of modules that no single file's tests can cover. -// They use only the public API of each module. - -use crate::{ - ArgumentMode, - classify::{LineKind, classify_line}, - join::LineJoiner, - normalize::normalize, - parse::parse_document, -}; - -// --- normalize + join --- - -#[test] -fn crlf_continuation_joins_same_as_lf() { - // §2.1 rules 1-2: CRLF and bare CR both normalize to LF before any other processing. - // §2.2: line joining runs after normalization, so a backslash before a CRLF boundary - // must produce the same logical line as the equivalent LF-only input. - let crlf = "/deploy production\\\r\n --region us-west-2"; - let lf = "/deploy production\\\n --region us-west-2"; - - let crlf_lines: Vec = normalize(crlf).split('\n').map(|s| s.to_string()).collect(); - let lf_lines: Vec = normalize(lf).split('\n').map(|s| s.to_string()).collect(); - - let crlf_ll = LineJoiner::new(crlf_lines).next_logical().unwrap(); - let lf_ll = LineJoiner::new(lf_lines).next_logical().unwrap(); - - assert_eq!(crlf_ll.text, lf_ll.text); - assert_eq!(crlf_ll.first_physical, lf_ll.first_physical); - assert_eq!(crlf_ll.last_physical, lf_ll.last_physical); -} - -#[test] -fn bare_cr_continuation_joins_same_as_lf() { - // §2.1 rule 2: remaining bare CR characters are replaced with LF after CRLF removal. - // §2.2: joining is agnostic to the original line-ending style. - let cr = "/deploy production\\\r --region us-west-2"; - let lf = "/deploy production\\\n --region us-west-2"; - - let cr_lines: Vec = normalize(cr).split('\n').map(|s| s.to_string()).collect(); - let lf_lines: Vec = normalize(lf).split('\n').map(|s| s.to_string()).collect(); - - let cr_ll = LineJoiner::new(cr_lines).next_logical().unwrap(); - let lf_ll = LineJoiner::new(lf_lines).next_logical().unwrap(); - - assert_eq!(cr_ll.text, lf_ll.text); -} - -#[test] -fn mixed_crlf_multi_line_join_matches_lf() { - // §2.1: normalization applies to all line endings uniformly. - // §2.2 step 4: joining repeats while the accumulated line still ends with `\`, - // so three physical lines collapse into one logical line regardless of ending style. - // §2.2.1: last_physical must be the zero-based index of the last consumed physical line. - let crlf = "/mcp call_tool read_file\\\r\n --path src/index.ts\\\r\n --format json"; - let lf = "/mcp call_tool read_file\\\n --path src/index.ts\\\n --format json"; - - let crlf_lines: Vec = normalize(crlf).split('\n').map(|s| s.to_string()).collect(); - let lf_lines: Vec = normalize(lf).split('\n').map(|s| s.to_string()).collect(); - - let crlf_ll = LineJoiner::new(crlf_lines).next_logical().unwrap(); - let lf_ll = LineJoiner::new(lf_lines).next_logical().unwrap(); - - assert_eq!(crlf_ll.text, lf_ll.text); - assert_eq!(crlf_ll.first_physical, 0); - assert_eq!(crlf_ll.last_physical, 2); -} - -// --- normalize + line_join + line_classify --- - -#[test] -fn joining_into_fence_opener_spec_5_2_6() { - // §5.2.6: "When backslash joining merges a command line with a line containing - // a fence opener, the fence is detected in the resulting logical line." - // §2.2.1: the logical line maps back to physical lines 0-1. - // §5.2.1: text before the backtick run becomes header; text after becomes fence_lang. - // §5.2.1: fence_backtick_count records the run length for the closer check. - // - // Input (two physical lines): - // /mcp call_tool write_file \ - // ```json - // - // After join: "/mcp call_tool write_file ```json" - // (two spaces: one trailing space before `\` + one joiner separator) - let input = normalize("/mcp call_tool write_file \\\n```json"); - let lines: Vec = input.split('\n').map(|s| s.to_string()).collect(); - let mut joiner = LineJoiner::new(lines); - - let ll = joiner.next_logical().unwrap(); - assert_eq!(ll.text, "/mcp call_tool write_file ```json"); - assert_eq!(ll.first_physical, 0); - assert_eq!(ll.last_physical, 1); - assert!(joiner.is_exhausted()); - - match classify_line(&ll.text) { - LineKind::Command(h) => { - assert_eq!(h.name, "mcp"); - assert_eq!(h.header_text, "call_tool write_file"); - assert_eq!(h.fence_lang, Some("json".to_string())); - assert_eq!(h.fence_backtick_count, 3); - } - LineKind::Text => panic!("expected fence command, got text"), - } -} - -#[test] -fn joined_logical_line_classifies_as_single_line_command() { - // §2.2: line joining produces logical lines; the state machine then classifies them. - // §5.1: a joined result with no fence opener in the args is a single-line command. - let input = normalize("/deploy production\\\n --region us-west-2"); - let lines: Vec = input.split('\n').map(|s| s.to_string()).collect(); - let ll = LineJoiner::new(lines).next_logical().unwrap(); - - match classify_line(&ll.text) { - LineKind::Command(h) => { - assert_eq!(h.name, "deploy"); - assert_eq!(h.header_text, "production --region us-west-2"); - assert_eq!(h.mode, ArgumentMode::SingleLine); - assert_eq!(h.fence_backtick_count, 0); - } - LineKind::Text => panic!("expected single-line command, got text"), - } -} - -#[test] -fn joined_invalid_slash_line_classifies_as_text() { - // §3.2: a line whose slash is not followed by a valid name is treated as text. - // §2.2: this check runs on the logical line after joining, not on the raw physical lines. - let input = normalize("/Hello world\\\n more args"); - let lines: Vec = input.split('\n').map(|s| s.to_string()).collect(); - let ll = LineJoiner::new(lines).next_logical().unwrap(); - - assert_eq!(classify_line(&ll.text), LineKind::Text); -} - -// --- Full pipeline: document_parse --- -// These tests exercise parse_document end-to-end across multiple modules. -// They verify cross-module composition that no single file's Layer 1 tests can cover. - -// --- Backslash joining through full pipeline (§2.2 + §2.3) --- - -#[test] -fn joined_multi_line_command_through_pipeline() { - // §2.2: backslash continuation joins physical lines into one logical line. - // §5.1: the joined result with no fence opener is single-line mode. - let result = parse_document("/deploy production\\\n --region us-west-2"); - assert_eq!(result.commands.len(), 1); - let cmd = result.commands.first().unwrap(); - assert_eq!(cmd.name, "deploy"); - assert_eq!(cmd.arguments.header, "production --region us-west-2"); - assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); -} - -#[test] -fn trailing_backslash_at_eof_removed_through_pipeline() { - // §2.2.2: trailing backslash at EOF is silently removed. - // The command parses as single-line with the backslash stripped. - let result = parse_document("/echo hello\\"); - assert_eq!(result.commands.len(), 1); - assert_eq!(result.commands.first().unwrap().arguments.payload, "hello"); - assert!(result.warnings.is_empty()); -} - -#[test] -fn fence_immunity_backslash_inside_fence_is_literal() { - // §2.3: trailing backslash inside fence is literal content, not a join marker. - let result = parse_document("/cmd ```\nline one\\\nline two\n```"); - assert_eq!(result.commands.len(), 1); - assert_eq!(result.commands.first().unwrap().arguments.payload, "line one\\\nline two"); -} - -#[test] -fn joining_into_fence_opener_through_pipeline() { - // §5.2.6: backslash joining merges command line with fence opener line. - // Physical lines 0-1 join into the command header; lines 2-3 are fence body + closer. - let result = parse_document("/mcp call_tool write_file \\\n```json\n{\"path\": \"foo\"}\n```"); - assert_eq!(result.commands.len(), 1); - let cmd = result.commands.first().unwrap(); - assert_eq!(cmd.name, "mcp"); - assert_eq!(cmd.arguments.header, "call_tool write_file"); - assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); - assert_eq!(cmd.arguments.payload, "{\"path\": \"foo\"}"); -} - -// --- Text block content with continuation lines (ADR-NNNN) --- - -#[test] -fn text_block_with_continuation_preserves_backslash() { - // ADR-NNNN: text block content stores physical lines. Backslashes are retained - // because text blocks capture pre-join content for round-trip fidelity. - let result = parse_document("hello \\\nworld"); - assert_eq!(result.textblocks.len(), 1); - assert_eq!(result.textblocks.first().unwrap().content, "hello \\\nworld"); -} - -#[test] -fn text_block_continuation_range_covers_all_physical_lines() { - // ADR-NNNN + §2.2.1: range spans all physical lines consumed by a joined logical line. - let result = parse_document("hello \\\nworld"); - assert_eq!(result.textblocks.len(), 1); - assert_eq!(result.textblocks.first().unwrap().range.start_line, 0); - assert_eq!(result.textblocks.first().unwrap().range.end_line, 1); -} - -// --- Appendix A spec examples --- -// Full pipeline assertions matching spec appendix examples exactly. -// These are the Layer 2 "acceptance test" equivalents. - -#[test] -fn spec_a1_single_line_command() { - // Appendix A.1: "/echo hello world" - let result = parse_document("/echo hello world"); - assert_eq!(result.commands.len(), 1); - let cmd = result.commands.first().unwrap(); - assert_eq!(cmd.id, "cmd-0"); - assert_eq!(cmd.name, "echo"); - assert_eq!(cmd.raw, "/echo hello world"); - assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 0); - assert_eq!(cmd.arguments.header, "hello world"); - assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); - assert_eq!(cmd.arguments.fence_lang, None); - assert_eq!(cmd.arguments.payload, "hello world"); -} - -#[test] -fn spec_a2_joined_multi_line_command() { - // Appendix A.2: three physical lines joined into one command. - let input = "/deploy production \\\n --region us-west-2 \\\n --canary"; - let result = parse_document(input); - assert_eq!(result.commands.len(), 1); - let cmd = result.commands.first().unwrap(); - assert_eq!(cmd.id, "cmd-0"); - assert_eq!(cmd.name, "deploy"); - assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 2); - assert_eq!(cmd.arguments.header, "production --region us-west-2 --canary"); - assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); - assert_eq!(cmd.arguments.payload, "production --region us-west-2 --canary"); - // §8.2: raw contains physical lines with backslashes and \n separators. - assert_eq!(cmd.raw, "/deploy production \\\n --region us-west-2 \\\n --canary"); -} - -#[test] -fn spec_a3_fenced_command_with_header() { - // Appendix A.3: "/mcp call_tool write_file ```json\n{...}\n```" - let input = "/mcp call_tool write_file ```json\n{\"path\": \"src/index.ts\"}\n```"; - let result = parse_document(input); - assert_eq!(result.commands.len(), 1); - let cmd = result.commands.first().unwrap(); - assert_eq!(cmd.id, "cmd-0"); - assert_eq!(cmd.name, "mcp"); - assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 2); - assert_eq!(cmd.arguments.header, "call_tool write_file"); - assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); - assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); - assert_eq!(cmd.arguments.payload, "{\"path\": \"src/index.ts\"}"); -} - -#[test] -fn spec_a4_backslash_join_into_fence() { - // Appendix A.4: four physical lines, join merges 0+1 into command+fence opener. - let input = "/mcp call_tool write_file \\\n```json\n{\"path\": \"foo\"}\n```"; - let result = parse_document(input); - assert_eq!(result.commands.len(), 1); - let cmd = result.commands.first().unwrap(); - assert_eq!(cmd.id, "cmd-0"); - assert_eq!(cmd.name, "mcp"); - assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 3); - assert_eq!(cmd.arguments.header, "call_tool write_file"); - assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); - assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); - assert_eq!(cmd.arguments.payload, "{\"path\": \"foo\"}"); - // §8.2: raw includes all physical lines from opener through closer. - assert_eq!(cmd.raw, "/mcp call_tool write_file \\\n```json\n{\"path\": \"foo\"}\n```"); -} - -#[test] -fn spec_a5_text_blocks_and_multiple_commands() { - // Appendix A.5: text, two commands, trailing text. - let input = "Welcome to the deployment system.\n\n/deploy staging\n/notify team --channel ops\nDeployment complete."; - let result = parse_document(input); - - // Two commands in order. - assert_eq!(result.commands.len(), 2); - assert_eq!(result.commands.first().unwrap().id, "cmd-0"); - assert_eq!(result.commands.first().unwrap().name, "deploy"); - assert_eq!(result.commands.first().unwrap().arguments.payload, "staging"); - assert_eq!(result.commands.first().unwrap().range.start_line, 2); - assert_eq!(result.commands.get(1).unwrap().id, "cmd-1"); - assert_eq!(result.commands.get(1).unwrap().name, "notify"); - assert_eq!(result.commands.get(1).unwrap().arguments.payload, "team --channel ops"); - assert_eq!(result.commands.get(1).unwrap().range.start_line, 3); - - // Two text blocks in order. - assert_eq!(result.textblocks.len(), 2); - assert_eq!(result.textblocks.first().unwrap().id, "text-0"); - assert_eq!(result.textblocks.first().unwrap().content, "Welcome to the deployment system.\n"); - assert_eq!(result.textblocks.first().unwrap().range.start_line, 0); - assert_eq!(result.textblocks.first().unwrap().range.end_line, 1); - assert_eq!(result.textblocks.get(1).unwrap().id, "text-1"); - assert_eq!(result.textblocks.get(1).unwrap().content, "Deployment complete."); - assert_eq!(result.textblocks.get(1).unwrap().range.start_line, 4); - assert_eq!(result.textblocks.get(1).unwrap().range.end_line, 4); -} - -#[test] -fn spec_a7_unclosed_fence() { - // Appendix A.7: fence never closed, command emitted with partial payload + warning. - let input = "/mcp call_tool ```json\n{\"incomplete\": true}"; - let result = parse_document(input); - assert_eq!(result.commands.len(), 1); - let cmd = result.commands.first().unwrap(); - assert_eq!(cmd.id, "cmd-0"); - assert_eq!(cmd.name, "mcp"); - assert_eq!(cmd.arguments.header, "call_tool"); - assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); - assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); - assert_eq!(cmd.arguments.payload, "{\"incomplete\": true}"); - assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 1); - - assert_eq!(result.warnings.len(), 1); - assert_eq!(result.warnings.first().unwrap().wtype, "unclosed-fence"); - assert_eq!(result.warnings.first().unwrap().start_line, Some(0)); -} - -#[test] -fn spec_a8_closing_fence_with_trailing_backslash() { - // Appendix A.8: "```\" is NOT a valid closer (not solely backticks after trim). - // The fence never closes; all remaining lines become payload. Warning emitted. - let input = "/mcp call_tool write_file -c \\\n```json\n{\"path\": \"foo\"}\n```\\\n\\\nproduction"; - let result = parse_document(input); - assert_eq!(result.commands.len(), 1); - let cmd = result.commands.first().unwrap(); - assert_eq!(cmd.name, "mcp"); - assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); - assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); - // Fence never closes: lines 2-5 are all payload. - assert!(cmd.arguments.payload.contains("{\"path\": \"foo\"}")); - - assert_eq!(result.warnings.len(), 1); - assert_eq!(result.warnings.first().unwrap().wtype, "unclosed-fence"); -} - -#[test] -fn spec_a9_proper_fence_close_followed_by_content() { - // Appendix A.9: fence closes on line 3, lines 4-5 join into text block. - let input = "/mcp call_tool write_file -c \\\n```json\n{\"path\": \"foo\"}\n```\n\\\nproduction"; - let result = parse_document(input); - - // One command with closed fence. - assert_eq!(result.commands.len(), 1); - let cmd = result.commands.first().unwrap(); - assert_eq!(cmd.name, "mcp"); - assert_eq!(cmd.arguments.header, "call_tool write_file -c"); - assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); - assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); - assert_eq!(cmd.arguments.payload, "{\"path\": \"foo\"}"); - assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 3); - - // Lines 4-5 join into "production" text block. - assert_eq!(result.textblocks.len(), 1); - assert_eq!(result.textblocks.first().unwrap().content, "\\\nproduction"); - assert_eq!(result.textblocks.first().unwrap().range.start_line, 4); - assert_eq!(result.textblocks.first().unwrap().range.end_line, 5); -} - -// --- Property tests (Layer 2) --- -// Cross-module invariants over arbitrary input. - -use proptest::prelude::*; - -proptest! { - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn never_panics_on_arbitrary_input(input in "\\PC{0,500}") { - // §8.1 (total function): parser always produces a valid envelope, never panics. - let _ = parse_document(&input); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn version_is_always_spec_version(input in "\\PC{0,200}") { - // §8.1: version field is always SPEC_VERSION regardless of input. - let result = parse_document(&input); - prop_assert_eq!(&result.version, crate::SPEC_VERSION); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn text_only_input_produces_zero_commands( - lines in prop::collection::vec("[^/\\n\\r][^\\n\\r]{0,40}", 1..10) - ) { - // §6 + §7: input with no slash-prefixed lines produces no commands. - // Lines are guaranteed not to start with '/' so none can be commands. - let input = lines.join("\n"); - let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), 0); - prop_assert!(!result.textblocks.is_empty()); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn fenced_content_preserved_verbatim(body in "[^`\\n\\r]{1,80}") { - // §5.2.2: fence body is verbatim, no joining, no escaping. - let input = format!("/cmd ```\n{}\n```", body); - let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), 1); - prop_assert_eq!(&result.commands.first().unwrap().arguments.payload, &body); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn unclosed_fence_always_produces_warning(body in "[^`]{0,80}") { - // §5.2.5: any fence reaching EOF without closer produces exactly one warning. - let input = format!("/cmd ```\n{}", body); - let result = parse_document(&input); - prop_assert_eq!(result.commands.len(), 1); - prop_assert_eq!(result.warnings.len(), 1); - prop_assert_eq!(&result.warnings.first().unwrap().wtype, "unclosed-fence"); - } -} +//! Layer 2 integration tests: cross-module composition within the application layer. +//! +//! These tests exercise combinations of modules that no single file's tests can cover. +//! They use only the public API of each module. + +mod normalize_join; +mod normalize_join_classify; +mod spec_examples; diff --git a/engine/src/integration_tests/normalize_join.rs b/engine/src/integration_tests/normalize_join.rs new file mode 100644 index 0000000..54c9037 --- /dev/null +++ b/engine/src/integration_tests/normalize_join.rs @@ -0,0 +1,61 @@ +//! normalize + join composition tests. +//! +//! Verify that line-ending normalization (§2.1) feeds correctly into +//! backslash line joining (§2.2) regardless of CR/CRLF/LF style. + +use crate::{join::LineJoiner, normalize::normalize}; + +#[test] +fn crlf_continuation_joins_same_as_lf() { + // §2.1 rules 1-2: CRLF and bare CR both normalize to LF before any other processing. + // §2.2: line joining runs after normalization, so a backslash before a CRLF boundary + // must produce the same logical line as the equivalent LF-only input. + let crlf = "/deploy production\\\r\n --region us-west-2"; + let lf = "/deploy production\\\n --region us-west-2"; + + let crlf_lines: Vec = normalize(crlf).split('\n').map(|s| s.to_string()).collect(); + let lf_lines: Vec = normalize(lf).split('\n').map(|s| s.to_string()).collect(); + + let crlf_ll = LineJoiner::new(crlf_lines).next_logical().unwrap(); + let lf_ll = LineJoiner::new(lf_lines).next_logical().unwrap(); + + assert_eq!(crlf_ll.text, lf_ll.text); + assert_eq!(crlf_ll.first_physical, lf_ll.first_physical); + assert_eq!(crlf_ll.last_physical, lf_ll.last_physical); +} + +#[test] +fn bare_cr_continuation_joins_same_as_lf() { + // §2.1 rule 2: remaining bare CR characters are replaced with LF after CRLF removal. + // §2.2: joining is agnostic to the original line-ending style. + let cr = "/deploy production\\\r --region us-west-2"; + let lf = "/deploy production\\\n --region us-west-2"; + + let cr_lines: Vec = normalize(cr).split('\n').map(|s| s.to_string()).collect(); + let lf_lines: Vec = normalize(lf).split('\n').map(|s| s.to_string()).collect(); + + let cr_ll = LineJoiner::new(cr_lines).next_logical().unwrap(); + let lf_ll = LineJoiner::new(lf_lines).next_logical().unwrap(); + + assert_eq!(cr_ll.text, lf_ll.text); +} + +#[test] +fn mixed_crlf_multi_line_join_matches_lf() { + // §2.1: normalization applies to all line endings uniformly. + // §2.2 step 4: joining repeats while the accumulated line still ends with `\`, + // so three physical lines collapse into one logical line regardless of ending style. + // §2.2.1: last_physical must be the zero-based index of the last consumed physical line. + let crlf = "/mcp call_tool read_file\\\r\n --path src/index.ts\\\r\n --format json"; + let lf = "/mcp call_tool read_file\\\n --path src/index.ts\\\n --format json"; + + let crlf_lines: Vec = normalize(crlf).split('\n').map(|s| s.to_string()).collect(); + let lf_lines: Vec = normalize(lf).split('\n').map(|s| s.to_string()).collect(); + + let crlf_ll = LineJoiner::new(crlf_lines).next_logical().unwrap(); + let lf_ll = LineJoiner::new(lf_lines).next_logical().unwrap(); + + assert_eq!(crlf_ll.text, lf_ll.text); + assert_eq!(crlf_ll.first_physical, 0); + assert_eq!(crlf_ll.last_physical, 2); +} diff --git a/engine/src/integration_tests/normalize_join_classify.rs b/engine/src/integration_tests/normalize_join_classify.rs new file mode 100644 index 0000000..8cac42a --- /dev/null +++ b/engine/src/integration_tests/normalize_join_classify.rs @@ -0,0 +1,127 @@ +//! normalize + join + classify composition tests. +//! +//! Verify that logical lines produced by joining are correctly classified +//! as commands or text, including fence detection across joined lines (§5.2.6). + +use proptest::prelude::*; + +use crate::{ + ArgumentMode, LineRange, + classify::{LineKind, classify_line}, + fence::{accept_fence_line, finalize_fence, open_fence}, + join::LineJoiner, + normalize::normalize, + test_helper::{feed_body, valid_command_name}, + text::{append_text, finalize_text, start_text}, +}; + +// --- Deterministic tests --- + +#[test] +fn joining_into_fence_opener_spec_5_2_6() { + // §5.2.6: "When backslash joining merges a command line with a line containing + // a fence opener, the fence is detected in the resulting logical line." + // §2.2.1: the logical line maps back to physical lines 0-1. + // §5.2.1: text before the backtick run becomes header; text after becomes fence_lang. + // §5.2.1: fence_backtick_count records the run length for the closer check. + // + // Input (two physical lines): + // /mcp call_tool write_file \ + // ```json + // + // After join: "/mcp call_tool write_file ```json" + // (two spaces: one trailing space before `\` + one joiner separator) + let input = normalize("/mcp call_tool write_file \\\n```json"); + let lines: Vec = input.split('\n').map(|s| s.to_string()).collect(); + let mut joiner = LineJoiner::new(lines); + + let ll = joiner.next_logical().unwrap(); + assert_eq!(ll.text, "/mcp call_tool write_file ```json"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 1); + assert!(joiner.is_exhausted()); + + match classify_line(&ll.text) { + LineKind::Command(h) => { + assert_eq!(h.name, "mcp"); + assert_eq!(h.header_text, "call_tool write_file"); + assert_eq!(h.fence_lang, Some("json".to_string())); + assert_eq!(h.fence_backtick_count, 3); + } + LineKind::Text => panic!("expected fence command, got text"), + } +} + +#[test] +fn joined_logical_line_classifies_as_single_line_command() { + // §2.2: line joining produces logical lines; the state machine then classifies them. + // §5.1: a joined result with no fence opener in the args is a single-line command. + let input = normalize("/deploy production\\\n --region us-west-2"); + let lines: Vec = input.split('\n').map(|s| s.to_string()).collect(); + let ll = LineJoiner::new(lines).next_logical().unwrap(); + + match classify_line(&ll.text) { + LineKind::Command(h) => { + assert_eq!(h.name, "deploy"); + assert_eq!(h.header_text, "production --region us-west-2"); + assert_eq!(h.mode, ArgumentMode::SingleLine); + assert_eq!(h.fence_backtick_count, 0); + } + LineKind::Text => panic!("expected single-line command, got text"), + } +} + +#[test] +fn joined_invalid_slash_line_classifies_as_text() { + // §3.2: a line whose slash is not followed by a valid name is treated as text. + // §2.2: this check runs on the logical line after joining, not on the raw physical lines. + let input = normalize("/Hello world\\\n more args"); + let lines: Vec = input.split('\n').map(|s| s.to_string()).collect(); + let ll = LineJoiner::new(lines).next_logical().unwrap(); + + assert_eq!(classify_line(&ll.text), LineKind::Text); +} + +// --- Property tests --- + +proptest! { + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn classify_open_accept_finalize_roundtrip( + name in valid_command_name(), + body_lines in prop::collection::vec("[a-zA-Z0-9]{1,30}", 0..8) + ) { + // §5.2: classify -> open_fence -> accept_fence_line -> finalize_fence roundtrip. + let input = format!("/{} ```", name); + let header = match classify_line(&input) { + LineKind::Command(h) => h, + LineKind::Text => panic!("expected command"), + }; + let raw = header.raw.clone(); + let range = LineRange { start_line: 0, end_line: 0 }; + let fence = open_fence(header, raw, 0, range); + let fence = feed_body(fence, &body_lines); + let (fence, _) = accept_fence_line(fence, body_lines.len() + 1, "```"); + let (cmd, warnings) = finalize_fence(fence, false); + prop_assert_eq!(cmd.name, name); + prop_assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + prop_assert!(warnings.is_empty()); + } + + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_lines_through_classify_and_collect( + lines in prop::collection::vec("[a-zA-Z0-9 !.,]{1,40}", 1..15) + ) { + // §6: non-command lines are collected into text blocks via classify + text collect. + for line in &lines { + prop_assert!(matches!(classify_line(line), LineKind::Text)); + } + let pending = lines.iter().enumerate().skip(1).fold( + start_text(0, &lines[0]), + |text, (i, line)| append_text(text, i, line), + ); + let block = finalize_text(pending, 0); + prop_assert_eq!(block.content, lines.join("\n")); + } +} diff --git a/engine/src/integration_tests/proptest.rs b/engine/src/integration_tests/proptest.rs deleted file mode 100644 index 8322e66..0000000 --- a/engine/src/integration_tests/proptest.rs +++ /dev/null @@ -1,57 +0,0 @@ -use proptest::prelude::*; - -use crate::{ - domain::{ArgumentMode, LineRange, SPEC_VERSION}, - engine::{ - classify::{LineKind, classify_line}, - fence::{accept_fence_line, finalize_fence, open_fence}, - text::{append_text, finalize_text, start_text}, - }, -}; - -fn valid_command_name() -> impl Strategy { - "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) -} - -proptest! { - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn classify_open_accept_finalize_roundtrip( - name in valid_command_name(), - body_lines in prop::collection::vec("[a-zA-Z0-9]{1,30}", 0..8) - ) { - let input = format!("/{name} ```"); - let header = match classify_line(&input) { - LineKind::Command(h) => h, - LineKind::Text => panic!("expected command"), - }; - let raw = header.raw.clone(); - let range = LineRange { start_line: 0, end_line: 0 }; - let fence = open_fence(header, raw, 0, range); - let fence = body_lines.iter().enumerate().fold(fence, |f, (i, line)| { - let (next, _) = accept_fence_line(f, i + 1, line); - next - }); - let (fence, _) = accept_fence_line(fence, body_lines.len() + 1, "```"); - let (cmd, warnings) = finalize_fence(fence, false); - prop_assert_eq!(cmd.name, name); - prop_assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); - prop_assert!(warnings.is_empty()); - } - - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn text_lines_through_classify_and_collect( - lines in prop::collection::vec("[a-zA-Z0-9 !.,]{1,40}", 1..15) - ) { - for line in &lines { - prop_assert!(matches!(classify_line(line), LineKind::Text)); - } - let pending = lines.iter().enumerate().skip(1).fold( - start_text(0, &lines[0]), - |text, (i, line)| append_text(text, i, line), - ); - let block = finalize_text(pending, 0); - prop_assert_eq!(block.content, lines.join("\n")); - } -} diff --git a/engine/src/integration_tests/spec_examples.rs b/engine/src/integration_tests/spec_examples.rs new file mode 100644 index 0000000..d4cb46b --- /dev/null +++ b/engine/src/integration_tests/spec_examples.rs @@ -0,0 +1,185 @@ +//! Appendix B spec examples: acceptance tests. +//! +//! Full pipeline assertions matching spec appendix examples exactly. +//! Each test corresponds to one numbered example from the specification. + +use crate::{ArgumentMode, parse::parse_document}; + +#[test] +fn spec_b1_single_line_command() { + let result = parse_document("/echo hello world"); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "echo"); + assert_eq!(cmd.raw, "/echo hello world"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 0); + assert_eq!(cmd.arguments.header, "hello world"); + assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); + assert_eq!(cmd.arguments.fence_lang, None); + assert_eq!(cmd.arguments.payload, "hello world"); +} + +#[test] +fn spec_b2_joined_multi_line_command() { + // B.2: three physical lines joined into one command. + // POSIX joining: backslash removed, lines concatenated directly. + // Leading space on continuation lines provides the separator. + let input = "/deploy production\\\n --region us-west-2\\\n --canary"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "deploy"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 2); + assert_eq!(cmd.arguments.header, "production --region us-west-2 --canary"); + assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); + assert_eq!(cmd.arguments.payload, "production --region us-west-2 --canary"); + assert_eq!(cmd.raw, "/deploy production\\\n --region us-west-2\\\n --canary"); +} + +#[test] +fn spec_b3_fenced_command_with_header() { + let input = "/mcp call_tool write_file ```json\n{\"path\": \"src/index.ts\"}\n```"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 2); + assert_eq!(cmd.arguments.header, "call_tool write_file"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.arguments.payload, "{\"path\": \"src/index.ts\"}"); +} + +#[test] +fn spec_b4_backslash_join_into_fence() { + // B.4: trailing space before \ keeps one space before backticks after join. + let input = "/mcp call_tool write_file \\\n```json\n{\"path\": \"foo\"}\n```"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 3); + assert_eq!(cmd.arguments.header, "call_tool write_file"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.arguments.payload, "{\"path\": \"foo\"}"); + assert_eq!(cmd.raw, "/mcp call_tool write_file \\\n```json\n{\"path\": \"foo\"}\n```"); +} + +#[test] +fn spec_b5_text_blocks_and_multiple_commands() { + let input = "Welcome to the deployment system.\n\n/deploy staging\n/notify team --channel ops\nDeployment complete."; + let result = parse_document(input); + + assert_eq!(result.commands.len(), 2); + assert_eq!(result.commands.first().unwrap().id, "cmd-0"); + assert_eq!(result.commands.first().unwrap().name, "deploy"); + assert_eq!(result.commands.first().unwrap().arguments.payload, "staging"); + assert_eq!(result.commands.first().unwrap().range.start_line, 2); + assert_eq!(result.commands.get(1).unwrap().id, "cmd-1"); + assert_eq!(result.commands.get(1).unwrap().name, "notify"); + assert_eq!(result.commands.get(1).unwrap().arguments.payload, "team --channel ops"); + assert_eq!(result.commands.get(1).unwrap().range.start_line, 3); + + assert_eq!(result.textblocks.len(), 2); + assert_eq!(result.textblocks.first().unwrap().id, "text-0"); + assert_eq!(result.textblocks.first().unwrap().content, "Welcome to the deployment system.\n"); + assert_eq!(result.textblocks.first().unwrap().range.start_line, 0); + assert_eq!(result.textblocks.first().unwrap().range.end_line, 1); + assert_eq!(result.textblocks.get(1).unwrap().id, "text-1"); + assert_eq!(result.textblocks.get(1).unwrap().content, "Deployment complete."); + assert_eq!(result.textblocks.get(1).unwrap().range.start_line, 4); + assert_eq!(result.textblocks.get(1).unwrap().range.end_line, 4); +} + +#[test] +fn spec_b6_invalid_slash_lines() { + // B.6: /123, / (bare slash), /Hello are all invalid slash lines (§4.5). + // They are text lines (§6.3), and consecutive text lines merge into + // a single text block (§6.4). The valid command follows on line 3. + let input = "/123\n/ bare slash\n/Hello\n/deploy staging"; + let result = parse_document(input); + + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().id, "text-0"); + assert_eq!(result.textblocks.first().unwrap().content, "/123\n/ bare slash\n/Hello"); + assert_eq!(result.textblocks.first().unwrap().range.start_line, 0); + assert_eq!(result.textblocks.first().unwrap().range.end_line, 2); + + assert_eq!(result.commands.len(), 1); + assert_eq!(result.commands.first().unwrap().id, "cmd-0"); + assert_eq!(result.commands.first().unwrap().name, "deploy"); + assert_eq!(result.commands.first().unwrap().arguments.payload, "staging"); + assert_eq!(result.commands.first().unwrap().arguments.mode, ArgumentMode::SingleLine); + assert_eq!(result.commands.first().unwrap().range.start_line, 3); + assert_eq!(result.commands.first().unwrap().range.end_line, 3); + + assert!(result.warnings.is_empty()); +} + +#[test] +fn spec_b7_unclosed_fence() { + let input = "/mcp call_tool ```json\n{\"incomplete\": true}"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.id, "cmd-0"); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.arguments.header, "call_tool"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.arguments.payload, "{\"incomplete\": true}"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 1); + + assert_eq!(result.warnings.len(), 1); + assert_eq!(result.warnings.first().unwrap().wtype, "unclosed_fence"); + assert_eq!(result.warnings.first().unwrap().start_line, Some(0)); +} + +#[test] +fn spec_b8_closing_fence_with_trailing_backslash() { + // B.8: "```\" is NOT a valid closer. Fence never closes. + // Trailing space before \ on line 0 keeps one space before backticks. + let input = "/mcp call_tool write_file -c \\\n```json\n{\"path\": \"foo\"}\n```\\\n\\\nproduction"; + let result = parse_document(input); + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert!(cmd.arguments.payload.contains("{\"path\": \"foo\"}")); + + assert_eq!(result.warnings.len(), 1); + assert_eq!(result.warnings.first().unwrap().wtype, "unclosed_fence"); +} + +#[test] +fn spec_b9_proper_fence_close_followed_by_content() { + // B.9: fence closes on line 3, lines 4-5 join into text block. + let input = "/mcp call_tool write_file -c \\\n```json\n{\"path\": \"foo\"}\n```\n\\\nproduction"; + let result = parse_document(input); + + assert_eq!(result.commands.len(), 1); + let cmd = result.commands.first().unwrap(); + assert_eq!(cmd.name, "mcp"); + assert_eq!(cmd.arguments.header, "call_tool write_file -c"); + assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); + assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); + assert_eq!(cmd.arguments.payload, "{\"path\": \"foo\"}"); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 3); + + assert_eq!(result.textblocks.len(), 1); + assert_eq!(result.textblocks.first().unwrap().content, "\\\nproduction"); + assert_eq!(result.textblocks.first().unwrap().range.start_line, 4); + assert_eq!(result.textblocks.first().unwrap().range.end_line, 5); +} diff --git a/engine/src/join.rs b/engine/src/join.rs index c47174f..2376ca2 100644 --- a/engine/src/join.rs +++ b/engine/src/join.rs @@ -8,6 +8,8 @@ /// Fence immunity is enforced by the caller: when the state machine enters /// `InFence`, it calls `next_physical` directly instead of `next_logical`, /// bypassing the joiner for those lines. + +#[derive(Debug, Clone, PartialEq, Eq)] pub struct LogicalLine { pub text: String, pub first_physical: usize, @@ -125,14 +127,8 @@ mod tests { #[test] fn consume_logical_joins_two_lines() { - // RFC §3.2 steps 1-3: remove trailing '\', remove LF boundary, - // concatenate with next line separated by a single SPACE. - // - // BUG: RFC v1.1.0 §3.2 step 3 says "Concatenate the remainder with - // the next physical line, separated by a single U+0020 (SPACE)." - // However Engine Spec v0.5.0 §7.1 says "No separator character is - // inserted" (true POSIX). The code inserts a space, matching the RFC - // but contradicting the Engine Spec. See gaps comment. + // Engine Spec §7.1: remove trailing '\', concatenate directly with + // the next physical line. No separator character is inserted. let ls = vec![bsl("a"), "b".to_string()]; let mut cursor = 0; let ll = consume_logical(&ls, &mut cursor).unwrap(); @@ -320,6 +316,230 @@ mod tests { assert_eq!(ll.last_physical, 1); } + // ========================================================================= + // consume_logical — no separator insertion (POSIX semantics) + // Engine Spec §7.1: "No separator character is inserted." + // ========================================================================= + + #[test] + fn consume_logical_does_not_insert_separator_space() { + // Engine Spec §7.1: joining concatenates directly with no separator. + // This test guards against regression to the v0.3.0 space-insertion + // behavior (Engine Spec §16.1). + let ls = vec![bsl("x"), "y".to_string()]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text.len(), 2); + assert_eq!(ll.text.as_bytes(), b"xy"); + } + + // ========================================================================= + // consume_logical — multi-backslash content preservation + // RFC §3.2: "The join marker is any '\' immediately before the line + // boundary, regardless of what precedes it." + // ========================================================================= + + #[test] + fn consume_logical_double_backslash_preserves_content_backslash() { + // RFC §3.2: only the final '\' (immediately before LF) is the join + // marker. A preceding '\' is literal content and must be preserved. + // Input: "foo\\" (two backslashes) + "bar" + // After removing the final '\' and joining: "foo\bar" + let mut line = "foo\\".to_string(); + line.push('\\'); // "foo\\" — two trailing backslashes + let ls = vec![line, "bar".to_string()]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "foo\\bar"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 1); + } + + #[test] + fn consume_logical_triple_backslash_joins_and_preserves_two() { + // RFC §3.2: three trailing backslashes. The final one is the join + // marker; after removal and joining, the result still ends with '\' + // so another join occurs. + // Input: "a\\\" + "b" → after first join: "a\\" + "b" → "a\\b" + // wait — let's be precise: + // Physical line 0: "a\\\" (three chars after 'a': \, \, \) + // Step 1: ends with '\', remove it → "a\\" + // Step 2: "a\\" still ends with '\', remove it → "a\" — wait, + // no. After removing the last '\' and concatenating line 1: + // "a\\" + "b" = "a\\b". Then check: does "a\\b" end with '\'? No. + // So the result is "a\\b". + // + // Actually: "a\\\" is 4 bytes: a, \, \, \. + // ends_with('\') → true. Truncate last → "a\\" + // Concat next line "b" → "a\\b" + // ends_with('\') on "a\\b"? No → done. + let mut line = "a".to_string(); + line.push('\\'); + line.push('\\'); + line.push('\\'); + let ls = vec![line, "b".to_string()]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "a\\\\b"); + assert_eq!(ll.last_physical, 1); + } + + // ========================================================================= + // consume_logical — backslash-only line + // RFC §3.2: empty remainder joins directly with the next physical line. + // ========================================================================= + + #[test] + fn consume_logical_backslash_only_line_joins_with_next() { + // RFC §3.2: a line containing only '\' has empty remainder after + // removing the join marker. Direct concatenation (no separator per + // Engine Spec §7.1) produces the next line's content unchanged. + let ls = vec![bsl(""), "hello".to_string()]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "hello"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 1); + } + + #[test] + fn consume_logical_backslash_only_at_eof() { + // RFC §3.2: backslash-only line at EOF. The '\' is removed; the line + // stands alone as an empty string. + let ls = vec![bsl("")]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, ""); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 0); + } + + // ========================================================================= + // consume_logical — whitespace-only continuation line + // RFC §3.2: no trimming of continuation lines; all content is preserved. + // ========================================================================= + + #[test] + fn consume_logical_whitespace_only_continuation_preserves_spaces() { + // RFC §3.2: the continuation line is concatenated verbatim. No + // trimming occurs. Engine Spec §7.1: no separator inserted. + let ls = vec![bsl("cmd"), " ".to_string()]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "cmd "); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 1); + } + + #[test] + fn consume_logical_tab_continuation_preserved() { + // RFC §3.2 + Engine Spec §6: HTAB is valid whitespace. The joiner + // must not strip or replace it. + let ls = vec![bsl("cmd"), "\targ".to_string()]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "cmd\targ"); + } + + // ========================================================================= + // consume_logical — trailing whitespace before backslash + // Engine Spec §7.1: content before the '\' is preserved as-is. + // ========================================================================= + + #[test] + fn consume_logical_trailing_space_before_backslash_preserved() { + // Engine Spec §7.1: the joiner removes only the final '\'. Any + // trailing space before it is part of the content and is preserved. + let ls = vec![bsl("cmd "), "arg".to_string()]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "cmd arg"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 1); + } + + // ========================================================================= + // consume_logical — empty lines in join chains + // RFC §3.2: joining behavior does not depend on line content. + // ========================================================================= + + #[test] + fn consume_logical_empty_continuation_line() { + // RFC §3.2: empty continuation line produces no additional content. + // Engine Spec §7.1: no separator, so result is just the first line's + // content (without the backslash). + let ls = vec![bsl("prefix"), "".to_string()]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "prefix"); + assert_eq!(ll.last_physical, 1); + } + + // ========================================================================= + // consume_logical — chained EOF backslash after join + // RFC §3.2 step 4: repeat if result still ends with '\'. + // ========================================================================= + + #[test] + fn consume_logical_mid_chain_eof_removes_backslash() { + // RFC §3.2: two physical lines both end with '\', but there is no + // third line. The first join succeeds; the second '\' hits EOF and + // is silently removed. + let ls = vec![bsl("a"), bsl("b")]; + let mut cursor = 0; + let ll = consume_logical(&ls, &mut cursor).unwrap(); + assert_eq!(ll.text, "ab"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 1); + } + + // ========================================================================= + // LineJoiner — interleaving modes with shared cursor + // Engine Spec §5.3: idle uses next_logical, in-fence uses next_physical. + // ========================================================================= + + #[test] + fn physical_then_logical_shares_cursor() { + // Engine Spec §5.3 / §7.2: after consuming physical lines (fence body), + // switching back to next_logical resumes at the correct position. + let input = vec!["fence body".to_string(), "```".to_string(), bsl("/cmd arg"), "rest".to_string()]; + let mut j = LineJoiner::new(input); + // Simulate in-fence: consume two physical lines. + let (idx0, _) = j.next_physical().unwrap(); + assert_eq!(idx0, 0); + let (idx1, _) = j.next_physical().unwrap(); + assert_eq!(idx1, 1); + // Transition to idle: next_logical joins lines 2-3. + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "/cmd argrest"); + assert_eq!(ll.first_physical, 2); + assert_eq!(ll.last_physical, 3); + assert!(j.is_exhausted()); + } + + // ========================================================================= + // RFC Appendix B.2 — updated assertion for no-separator semantics + // Engine Spec §7.1 + RFC Appendix B.2 + // ========================================================================= + + #[test] + fn appendix_b2_with_leading_whitespace_on_continuations() { + // RFC Appendix B.2 shows continuation lines indented with spaces. + // After joining (no separator), the leading spaces on continuation + // lines provide the visual separation. + // Input: + // "/deploy production \" (trailing space before \) + // " --region us-west-2 \" (two leading spaces, trailing space before \) + // " --canary" (two leading spaces) + // Result: "production " + " --region..." + " --canary" + let input = vec![bsl("/deploy production "), bsl(" --region us-west-2 "), " --canary".to_string()]; + let mut j = LineJoiner::new(input); + let ll = j.next_logical().unwrap(); + assert_eq!(ll.text, "/deploy production --region us-west-2 --canary"); + assert_eq!(ll.first_physical, 0); + assert_eq!(ll.last_physical, 2); + } + // ========================================================================= // Property tests // ========================================================================= @@ -384,49 +604,3 @@ mod tests { } } } - -// ============================================================================= -// TEST GAPS: spec areas this file's functions touch but are not tested -// ============================================================================= -// -// | Spec Section | Gap | Severity | -// |-----------------------------------------|----------------------------------------------|----------| -// | Engine Spec §7.1 vs RFC §3.2 step 3 | SPACE INSERTION CONFLICT: The code inserts | CRITICAL | -// | | a space between joined lines (text.push(' ')) | | -// | | matching RFC §3.2 step 3. But Engine Spec | | -// | | v0.5.0 §7.1 says "No separator character is | | -// | | inserted. This is true POSIX backslash- | | -// | | newline removal." and §15.1 migration notes | | -// | | explicitly changed this from v0.3.0. | | -// | | THE RFC AND ENGINE SPEC CONTRADICT EACH | | -// | | OTHER. Must resolve which is authoritative. | | -// |-----------------------------------------------------------------------|----------| -// | RFC §3.2 | MULTI-BACKSLASH: No test for a line ending | MEDIUM | -// | | with multiple backslashes (e.g., "foo\\\\"). | | -// | | Two backslashes: the last '\' is the join | | -// | | marker, the preceding '\' is content. Only | | -// | | the final '\' should be removed. | | -// |-----------------------------------------------------------------------|----------| -// | RFC §3.2 | EMPTY LINE WITH BACKSLASH: No test for a | LOW | -// | | line that is just "\" (backslash only). Should | | -// | | join with next line, producing just the next | | -// | | line's content (with or without leading space | | -// | | depending on space-insertion resolution). | | -// |-----------------------------------------------------------------------|----------| -// | RFC §3.3 | PHYSICAL LINE TRACKING FOR TEXT BLOCKS: No | LOW | -// | | test verifying that a joined text line's | | -// | | constituent physical lines (with backslashes) | | -// | | are preserved for text block raw content | | -// | | (Engine Spec §10). This is the text_collect | | -// | | module's job, but the LogicalLine range data | | -// | | is what enables it. | | -// |-----------------------------------------------------------------------|----------| -// | RFC §3.2 | WHITESPACE-ONLY CONTINUATION LINE: No test | LOW | -// | | for joining where the continuation line is | | -// | | only whitespace (e.g., "cmd\\" + " "). | | -// | | Verifies no trimming occurs. | | -// |-----------------------------------------------------------------------|----------| -// | Engine Spec §7.1 | NO-SEPARATOR JOIN ASSERTION: If the Engine | LOW | -// | | Spec is authoritative (no space), there is no | | -// | | test asserting direct concatenation without | | -// | | space. The current tests all assert WITH space.| | diff --git a/engine/src/lib.rs b/engine/src/lib.rs index c35b783..afd318e 100644 --- a/engine/src/lib.rs +++ b/engine/src/lib.rs @@ -1,4 +1,3 @@ -// Internal modules mod classify; mod fence; mod join; @@ -11,9 +10,10 @@ mod types; // Public API pub use parse::parse_document; pub use types::{ - ArgumentMode, Command, CommandArguments, LineRange, - ParseResult, SPEC_VERSION, TextBlock, Warning, + ArgumentMode, Command, CommandArguments, LineRange, ParseResult, SPEC_VERSION, TextBlock, Warning, }; #[cfg(test)] mod integration_tests; +#[cfg(test)] +mod test_helper; diff --git a/engine/src/normalize.rs b/engine/src/normalize.rs index 565faac..a886a49 100644 --- a/engine/src/normalize.rs +++ b/engine/src/normalize.rs @@ -106,6 +106,49 @@ mod tests { assert_eq!(normalize(input), "before\\nafter"); } + // ========================================================================= + // Boundary: single-character / single-sequence inputs + // RFC §3.1 + // ========================================================================= + + #[test] + fn lone_crlf_becomes_lone_lf() { + // RFC §3.1 step 1: a single CRLF with no surrounding content. + assert_eq!(normalize("\r\n"), "\n"); + } + + #[test] + fn lone_bare_cr_becomes_lf() { + // RFC §3.1 step 2: a single bare CR with no surrounding content. + assert_eq!(normalize("\r"), "\n"); + } + + #[test] + fn lone_lf_unchanged() { + // RFC §3.1: LF is already normalized. + assert_eq!(normalize("\n"), "\n"); + } + + // ========================================================================= + // Non-ASCII / multi-byte UTF-8 preservation + // RFC §3.1: only CR and CRLF are affected; all other bytes preserved. + // ========================================================================= + + #[test] + fn multibyte_utf8_preserved() { + // RFC §3.1: normalization only affects U+000D and U+000D U+000A. + // Multi-byte content (CJK, emoji) must survive unchanged. + let input = "héllo\r\nwörld\r🌍"; + assert_eq!(normalize(input), "héllo\nwörld\n🌍"); + } + + #[test] + fn astral_plane_chars_preserved() { + // RFC §3.1: non-BMP characters are not line terminators. + let input = "𝕳𝖊𝖑𝖑𝖔\r\n𝖂𝖔𝖗𝖑𝖉"; + assert_eq!(normalize(input), "𝕳𝖊𝖑𝖑𝖔\n𝖂𝖔𝖗𝖑𝖉"); + } + // ========================================================================= // Property tests // ========================================================================= @@ -145,45 +188,27 @@ mod tests { let result_lf = result.chars().filter(|&c| c == '\n').count(); prop_assert!(result_lf >= original_lf); } + + // ========================================================================= + // Property tests (add inside existing proptest! block) + // ========================================================================= + + // RFC §3.1 steps 1-2: CRLF (2 bytes) becomes LF (1 byte), bare CR (1 byte) + // becomes LF (1 byte). Output can never be longer than input. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn output_length_lte_input(input in "[\\x00-\\x7F]{0,500}") { + let result = normalize(&input); + prop_assert!(result.len() <= input.len()); } -} -// ============================================================================= -// TEST GAPS: spec areas this file's functions touch but are not tested -// ============================================================================= -// -// | Spec Section | Gap | Severity | -// |---------------------------------|-------------------------------------------------|----------| -// | RFC §3.1 | SPLIT_LINES: The spec says "The normalized | HIGH | -// | | input is split on LF to produce a sequence of | | -// | | physical lines." Engine Spec §5.2 defines this | | -// | | as a separate stage. This module only covers | | -// | | normalization, not splitting. If split_lines | | -// | | lives elsewhere, that's fine, but if it's | | -// | | missing entirely it's a gap. | | -// |---------------------------------|-------------------------------------------------|----------| -// | RFC §3.1 | TRAILING LF: "A trailing LF at the end of | MEDIUM | -// | | input produces a trailing empty line." No test | | -// | | verifies that normalize preserves a trailing | | -// | | \n (e.g., "hello\n" -> "hello\n"). This is | | -// | | trivially true for LF input but should be | | -// | | verified for trailing \r and trailing \r\n. | | -// |---------------------------------|-------------------------------------------------|----------| -// | RFC §3.1 | UNICODE / NON-ASCII: All property tests use | LOW | -// | | ASCII [\x00-\x7F]. No test verifies that | | -// | | multi-byte UTF-8 content (e.g., emoji, CJK) | | -// | | passes through normalization unmodified. The | | -// | | implementation (.replace) handles this | | -// | | correctly, but a test would guard regressions. | | -// |---------------------------------|-------------------------------------------------|----------| -// | RFC §3.1 | LENGTH INVARIANT: normalize can only shrink | LOW | -// | | (CRLF -> LF removes one byte) or preserve | | -// | | length. No property test asserts | | -// | | result.len() <= input.len(). | | -// |---------------------------------|-------------------------------------------------|----------| -// | Engine Spec §5.1 | PURE FUNCTION: "This stage is a pure string | INFO | -// | | transformation with no state." No test verifies | | -// | | that calling normalize multiple times on | | -// | | different inputs doesn't leak state (trivially | | -// | | true for a free function, but documenting the | | -// | | intent is useful). | | + // RFC §3.1: normalize is a pure function of its input. Same input always + // produces the same output (no hidden state). + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn deterministic(input in "[\\x00-\\x7F]{0,300}") { + prop_assert_eq!(normalize(&input), normalize(&input)); + } + + } +} diff --git a/engine/src/parse.rs b/engine/src/parse.rs index 6908a08..5d03d71 100644 --- a/engine/src/parse.rs +++ b/engine/src/parse.rs @@ -6,10 +6,7 @@ use super::{ single_line::finalize_single_line, text::{PendingText, append_text, finalize_text, start_text}, }; -use crate::{ - LineRange, - ArgumentMode, Command, ParseResult, SPEC_VERSION, TextBlock, Warning, -}; +use crate::{ArgumentMode, Command, LineRange, ParseResult, SPEC_VERSION, TextBlock, Warning}; // --- Types --- @@ -134,17 +131,15 @@ fn step_in_fence(ctx: &mut ParseCtx, joiner: &mut LineJoiner, fence: PendingFenc // --- Context helpers --- fn start_new_command(ctx: &mut ParseCtx, header: CommandHeader, first_physical: usize, last_physical: usize) { + let raw = header.raw.clone(); + let range = LineRange { start_line: first_physical, end_line: last_physical }; match header.mode { ArgumentMode::SingleLine => { - let raw = header.raw.clone(); - let range = LineRange { start_line: first_physical, end_line: last_physical }; let cmd = finalize_single_line(header, raw, ctx.cmd_seq, range); ctx.commands.push(cmd); ctx.cmd_seq += 1; } ArgumentMode::Fence => { - let raw = header.raw.clone(); - let range = LineRange { start_line: first_physical, end_line: last_physical }; let fence = open_fence(header, raw, ctx.cmd_seq, range); ctx.cmd_seq += 1; ctx.state = ParserState::InFence(fence); @@ -178,367 +173,127 @@ fn fold_physical_lines(mut text: PendingText, from: usize, to: usize, phys: &[&s text } +// parse.rs is the public orchestrator: it composes normalize, join, +// classify, singleline, fence, and text into parse_document. Each +// sub-module carries its own unit tests. Orchestration logic (state +// transitions, flush timing, counter management) is covered by +// integration tests in tests/orchestration_tests.rs and +// tests/integration_tests/. + #[cfg(test)] mod tests { - use super::parse_document; - // NOTE: All imports below are from crate::domain (external to this file). - // parse_document orchestrates: normalize, split_physical_lines, LineJoiner, - // classify_line, finalize_single_line, open_fence, accept_fence_line, - // finalize_fence, start_text, append_text, finalize_text. - use crate::{ArgumentMode, SPEC_VERSION}; - - // ========================================================================= - // Empty / trivial input - // RFC §8.2 item 4 / Engine Spec §4.2 - // ========================================================================= + use super::{ParseCtx, accumulate_text, flush_text, fold_physical_lines, split_physical_lines}; + use crate::{SPEC_VERSION, text::start_text}; - #[test] - fn empty_input() { - // RFC §8.2 item 4: "An empty input produces a result with no commands, - // no text blocks, and no warnings." - // Engine Spec §4.2: total function guarantee. - let r = parse_document(""); - assert!(r.commands.is_empty()); - assert!(r.textblocks.is_empty()); - assert!(r.warnings.is_empty()); - } + // --- split_physical_lines --- + // RFC §3.1: the normalized input is split on LF to produce a + // sequence of physical lines. A trailing LF produces a trailing + // empty line, which split_physical_lines pops. #[test] - fn whitespace_only_is_text() { - // RFC §6.3: "A text line is any non-fence-body logical line that is - // not a command line." Whitespace-only lines are text. - let r = parse_document(" "); - assert!(r.commands.is_empty()); - assert_eq!(r.textblocks.len(), 1); + fn split_empty_string_produces_empty_vec() { + // "" -> [""] -> popped -> [] + assert_eq!(split_physical_lines(""), Vec::<&str>::new()); } - // ========================================================================= - // Single-line command — end-to-end threading - // RFC §5.1 / Engine Spec §8 / Engine Spec §9.3 - // ========================================================================= - #[test] - fn single_line_command_fields() { - // RFC §5.1: single-line mode, header == payload. - // RFC §6.5: id is cmd-0. - // RFC §7.1: name, raw, range, mode, payload. - let r = parse_document("/deploy production --region us-west-2"); - assert_eq!(r.commands.len(), 1); - let cmd = &r.commands[0]; - assert_eq!(cmd.id, "cmd-0"); - assert_eq!(cmd.name, "deploy"); - assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); - assert_eq!(cmd.arguments.payload, "production --region us-west-2"); - assert_eq!(cmd.arguments.header, "production --region us-west-2"); - assert_eq!(cmd.raw, "/deploy production --region us-west-2"); - assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 0); + fn split_no_trailing_newline_kept() { + // "abc" -> ["abc"], no trailing empty element to pop + assert_eq!(split_physical_lines("abc"), vec!["abc"]); } #[test] - fn single_line_no_args() { - // RFC §4.3: "The arguments portion may be empty." - // RFC §5.1: empty args -> empty header and payload. - let r = parse_document("/ping"); - let cmd = &r.commands[0]; - assert_eq!(cmd.arguments.header, ""); - assert_eq!(cmd.arguments.payload, ""); + fn split_trailing_newline_popped() { + // "abc\n" -> ["abc", ""] -> popped -> ["abc"] + assert_eq!(split_physical_lines("abc\n"), vec!["abc"]); } - // ========================================================================= - // Fenced command — end-to-end threading - // RFC §5.2 / Engine Spec §9 - // ========================================================================= - #[test] - fn fenced_command_fields() { - // RFC §5.2.2: body lines joined with LF. - // RFC §5.2.1: fence_lang from opener. - // RFC §7.1: raw includes opener, body, closer. - // Engine Spec §3.6: range is inclusive physical lines. - let r = parse_document("/cmd ```json\nline one\nline two\n```"); - assert_eq!(r.commands.len(), 1); - let cmd = &r.commands[0]; - assert_eq!(cmd.arguments.payload, "line one\nline two"); - assert_eq!(cmd.arguments.mode, ArgumentMode::Fence); - assert_eq!(cmd.arguments.fence_lang, Some("json".to_string())); - assert_eq!(cmd.raw, "/cmd ```json\nline one\nline two\n```"); - assert_eq!(cmd.range.start_line, 0); - assert_eq!(cmd.range.end_line, 3); + fn split_single_newline_produces_one_empty_line() { + // "\n" -> ["", ""] -> popped -> [""] + assert_eq!(split_physical_lines("\n"), vec![""]); } - // ========================================================================= - // Unclosed fence - // RFC §5.2.4 / Engine Spec §9.3 step 4 - // ========================================================================= - #[test] - fn unclosed_fence_warning_and_partial_command() { - // RFC §5.2.4: "A warning of type unclosed_fence MUST be produced." - // RFC §5.2.4: "The command is complete with whatever payload has been - // accumulated through EOF." - // - // NOTE: wtype is "unclosed-fence" (kebab-case). Engine Spec §11 and - // RFC §7.4 require "unclosed_fence" (snake_case). This is a known - // code bug inherited from fence.rs finalize_fence. - let r = parse_document("/cmd ```\npartial body"); - assert_eq!(r.commands.len(), 1); - assert_eq!(r.commands[0].arguments.payload, "partial body"); - assert_eq!(r.warnings.len(), 1); - assert_eq!(r.warnings[0].wtype, "unclosed-fence"); + fn split_multiple_newlines_popped_once() { + // "a\n\n" -> ["a", "", ""] -> popped -> ["a", ""] + assert_eq!(split_physical_lines("a\n\n"), vec!["a", ""]); } - // ========================================================================= - // Text block accumulation - // RFC §6.3 / RFC §6.4 / Engine Spec §10 - // ========================================================================= + // --- fold_physical_lines --- + // Folds a slice of physical lines into a PendingText, appending + // each line via text::append_text. Pure parse.rs helper. #[test] - fn text_only() { - // RFC §6.4: "Consecutive text lines form a single text block." - // RFC §7.2: content is lines joined with LF. - let r = parse_document("line one\nline two\nline three"); - assert_eq!(r.textblocks.len(), 1); - assert_eq!(r.textblocks[0].content, "line one\nline two\nline three"); - assert_eq!(r.textblocks[0].range.start_line, 0); - assert_eq!(r.textblocks[0].range.end_line, 2); + fn folds_multiple_lines_into_pending_text() { + let initial = start_text(0, "line zero"); + let phys = vec!["line zero", "line one", "line two", "line three"]; + let result = fold_physical_lines(initial, 1, 2, &phys); + assert_eq!(result.start_line, 0); + assert_eq!(result.end_line, 2); + assert_eq!(result.lines, vec!["line zero", "line one", "line two"]); } - // ========================================================================= - // Interleaving commands and text - // RFC §6 / RFC §6.4 / Engine Spec §5.3 - // ========================================================================= + // --- accumulate_text --- + // Mutates ParseCtx.current_text. Creates a new PendingText when + // None, appends to existing when Some. #[test] - fn text_before_command() { - // RFC §6.4: text lines before a command form a text block. - let r = parse_document("preamble\n/cmd arg"); - assert_eq!(r.textblocks.len(), 1); - assert_eq!(r.textblocks[0].content, "preamble"); - assert_eq!(r.commands.len(), 1); + fn accumulate_text_creates_new_block_when_none() { + let mut ctx = ParseCtx::new(); + let phys = vec!["line 0", "line 1"]; + accumulate_text(&mut ctx, 0, 1, &phys); + let text = ctx.current_text.unwrap(); + assert_eq!(text.start_line, 0); + assert_eq!(text.end_line, 1); + assert_eq!(text.lines, vec!["line 0", "line 1"]); } #[test] - fn text_after_command() { - // RFC §6.4: "A new text block begins after a command is finalized, - // if text lines follow." - let r = parse_document("/cmd arg\npostamble"); - assert_eq!(r.commands.len(), 1); - assert_eq!(r.textblocks.len(), 1); - assert_eq!(r.textblocks[0].content, "postamble"); + fn accumulate_text_appends_to_existing_block() { + let mut ctx = ParseCtx::new(); + ctx.current_text = Some(start_text(0, "line 0")); + let phys = vec!["line 0", "line 1", "line 2"]; + accumulate_text(&mut ctx, 1, 2, &phys); + let text = ctx.current_text.unwrap(); + assert_eq!(text.start_line, 0); + assert_eq!(text.end_line, 2); + assert_eq!(text.lines, vec!["line 0", "line 1", "line 2"]); } - #[test] - fn text_between_commands() { - // RFC §6.4: text between two commands forms its own block. - let r = parse_document("/cmd1 a\nmiddle text\n/cmd2 b"); - assert_eq!(r.commands.len(), 2); - assert_eq!(r.textblocks.len(), 1); - assert_eq!(r.textblocks[0].content, "middle text"); - } + // --- flush_text --- + // Drains ParseCtx.current_text into textblocks, assigns the + // next sequential id, increments textseq. #[test] - fn consecutive_commands() { - // RFC §6.5: multiple commands in document order. - let r = parse_document("/cmd1 a\n/cmd2 b"); - assert_eq!(r.commands.len(), 2); - assert_eq!(r.commands[0].name, "cmd1"); - assert_eq!(r.commands[1].name, "cmd2"); + fn flush_text_pushes_block_and_increments_seq() { + // RFC §6.5: text blocks assigned sequential zero-based IDs. + let mut ctx = ParseCtx::new(); + ctx.current_text = Some(start_text(0, "line 0")); + flush_text(&mut ctx); + assert!(ctx.current_text.is_none()); + assert_eq!(ctx.textblocks.len(), 1); + assert_eq!(ctx.textblocks[0].id, "text-0"); + assert_eq!(ctx.text_seq, 1); } #[test] - fn fence_followed_by_command() { - // RFC Appendix C: after fence closes, parser returns to idle. - let r = parse_document("/cmd1 ```\nbody\n```\n/cmd2 arg"); - assert_eq!(r.commands.len(), 2); - assert_eq!(r.commands[0].arguments.mode, ArgumentMode::Fence); - assert_eq!(r.commands[1].arguments.mode, ArgumentMode::SingleLine); + fn flush_text_noops_when_empty() { + let mut ctx = ParseCtx::new(); + flush_text(&mut ctx); + assert!(ctx.textblocks.is_empty()); + assert_eq!(ctx.text_seq, 0); } - // ========================================================================= - // ID assignment — sequential, independent counters - // RFC §6.5 / Engine Spec §3.2 / Engine Spec §3.5 - // ========================================================================= + // --- ParseCtx::into_result --- #[test] - fn command_ids_sequential() { - // RFC §6.5: "cmd-0, cmd-1, cmd-2." - let r = parse_document("/a x\n/b y\n/c z"); - assert_eq!(r.commands[0].id, "cmd-0"); - assert_eq!(r.commands[1].id, "cmd-1"); - assert_eq!(r.commands[2].id, "cmd-2"); - } - - #[test] - fn text_ids_sequential() { - // RFC §6.5: "text-0, text-1, text-2." - let r = parse_document("aaa\n/cmd x\nbbb\n/cmd y\nccc"); - assert_eq!(r.textblocks[0].id, "text-0"); - assert_eq!(r.textblocks[1].id, "text-1"); - assert_eq!(r.textblocks[2].id, "text-2"); - } - - #[test] - fn command_and_text_ids_independent() { - // RFC §6.5: command and text block ID sequences are independent. - let r = parse_document("prose\n/cmd arg\nmore prose"); - assert_eq!(r.commands[0].id, "cmd-0"); - assert_eq!(r.textblocks[0].id, "text-0"); - assert_eq!(r.textblocks[1].id, "text-1"); - } - - // ========================================================================= - // raw field — physical line preservation - // RFC §7.1 / Engine Spec §9.3 step 2 - // ========================================================================= - - #[test] - fn single_line_raw() { - // RFC §7.1: raw is "the exact source text from the normalized input." - let r = parse_document("/echo hello world"); - assert_eq!(r.commands[0].raw, "/echo hello world"); - } - - #[test] - fn joined_command_raw_preserves_backslashes() { - // RFC §7.1: "For joined commands, this includes all physical lines - // with their backslashes and LF separators." - let r = parse_document("/deploy prod \\\n --region us-west-2"); - assert_eq!(r.commands[0].raw, "/deploy prod \\\n --region us-west-2"); - } - - #[test] - fn fenced_raw_includes_all_lines() { - // RFC §7.1: "For fenced commands, this includes the opener line, - // all body lines, and the closer line if present." - let r = parse_document("/cmd ```\nbody\n```"); - assert_eq!(r.commands[0].raw, "/cmd ```\nbody\n```"); - } - - // ========================================================================= - // Trailing newline edge case - // Engine Spec §5.2 / RFC §3.1 - // ========================================================================= - - #[test] - fn trailing_newline_no_empty_text_block() { - // Engine Spec §5.2: "the current engine pops a trailing empty element - // when the input ends with LF." - let r = parse_document("/cmd arg\n"); - assert_eq!(r.commands.len(), 1); - assert!(r.textblocks.is_empty()); - } - - // ========================================================================= - // Version field - // Engine Spec §14 / Engine Spec §3.1 - // ========================================================================= - - #[test] - fn version_set() { - // Engine Spec §14: SPEC_VERSION populated into ParseResult.version. - let r = parse_document(""); - assert_eq!(r.version, SPEC_VERSION); - } - - // ========================================================================= - // Property tests - // ========================================================================= - - use proptest::prelude::*; - - proptest! { - // RFC §8.2 item 4: "always produces a valid result for any input." - // Engine Spec §4.2: total function, never panics. - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn never_panics(input in "\\PC{0,500}") { - let _ = parse_document(&input); - } - - // Engine Spec §14: version is always SPEC_VERSION. - #[test] - #[cfg_attr(feature = "tdd", ignore)] - fn version_always_spec_version(input in "\\PC{0,200}") { - let r = parse_document(&input); - prop_assert_eq!(r.version, SPEC_VERSION); - } + fn parse_ctx_into_result_injects_version() { + let ctx = ParseCtx::new(); + let result = ctx.into_result(); + assert_eq!(result.version, SPEC_VERSION); + assert!(result.commands.is_empty()); + assert!(result.textblocks.is_empty()); + assert!(result.warnings.is_empty()); } } - -// ============================================================================= -// TEST GAPS: spec areas this file's functions touch but are not tested -// ============================================================================= -// -// | Spec Section | Gap | Severity | -// |-----------------------------------|--------------------------------------------------|----------| -// | Engine Spec §11 / RFC §7.4 | WARNING TYPE STRING: code emits "unclosed-fence" | CRITICAL | -// | | (kebab-case) but Engine Spec §11 requires | | -// | | "unclosed_fence" (snake_case). This is a code bug | | -// | | in finalize_fence, observable at integration level. | | -// |-----------------------------------|--------------------------------------------------|----------| -// | Engine Spec §7.1 vs RFC §3.2 | SPACE INSERTION: join.rs inserts a space between | CRITICAL | -// | | joined lines. Engine Spec §7.1 says "No separator | | -// | | character is inserted." RFC §3.2 step 3 says | | -// | | "separated by a single SPACE." Specs contradict. | | -// | | No integration test asserts the exact join result. | | -// | | joined_command_raw_preserves_backslashes tests raw | | -// | | but not the joined logical line / payload content. | | -// |-----------------------------------|--------------------------------------------------|----------| -// | RFC §6.1 / Engine Spec §10 | TEXT BLOCK CONTENT IS PHYSICAL: Engine Spec §10 | HIGH | -// | | says "A logical line formed by backslash | | -// | | continuation contributes all of its constituent | | -// | | physical lines with backslashes intact." No test | | -// | | verifies that a text block containing a joined line | | -// | | preserves the physical lines (with backslashes) in | | -// | | its content, not the joined logical text. | | -// |-----------------------------------|--------------------------------------------------|----------| -// | RFC Appendix B.5 | MIXED SCENARIO: RFC Appendix B.5 has a full | MEDIUM | -// | | worked example with blank line in text, two | | -// | | commands, and trailing text. No integration test | | -// | | replicates this exact scenario end-to-end. | | -// |-----------------------------------|--------------------------------------------------|----------| -// | RFC Appendix B.6 | INVALID SLASH LINES: RFC Appendix B.6 shows | MEDIUM | -// | | "/123", "/ bare slash", "/Hello" as text lines. | | -// | | No integration test verifies these pass through | | -// | | parse_document as text blocks. | | -// |-----------------------------------|--------------------------------------------------|----------| -// | RFC Appendix B.4 | BACKSLASH JOIN INTO FENCE: A command line | MEDIUM | -// | | split across physical lines via backslash that | | -// | | joins into a fence opener (e.g., "/mcp \" + "```"). | | -// | | No integration test for this scenario. | | -// |-----------------------------------|--------------------------------------------------|----------| -// | RFC §5.2.3 / Appendix B.8-B.9 | FENCE CLOSER WITH TRAILING BACKSLASH: B.8 shows | MEDIUM | -// | | "```\" is NOT a closer (backslash makes it non- | | -// | | solely-backtick). B.9 shows "```" as a valid closer | | -// | | followed by content that joins. Neither scenario | | -// | | has an integration test. | | -// |-----------------------------------|--------------------------------------------------|----------| -// | RFC §6.4 | BLANK LINES IN TEXT BLOCKS: No test for blank | LOW | -// | | lines within a text block (e.g., "a\n\nb" producing | | -// | | content "a\n\nb"). Covered in text.rs unit tests | | -// | | but not at integration level. | | -// |-----------------------------------|--------------------------------------------------|----------| -// | RFC §8.3 | ROUNDTRIP FIDELITY: No test asserts that | LOW | -// | | P(F(P(I))) == P(I) for any input. This would | | -// | | require a formatter, which is an SDK concern, but | | -// | | the engine should preserve enough data to enable it.| | -// |-----------------------------------|--------------------------------------------------|----------| -// | RFC §8.4 | DETERMINISM: No property test asserts that | LOW | -// | | parse_document(x) == parse_document(x) for same | | -// | | input. Trivially true for pure functions but would | | -// | | guard against accidental HashMap usage. | | -// |-----------------------------------|--------------------------------------------------|----------| -// | Engine Spec §6 | WHITESPACE DEFINITION: step_idle calls | HIGH | -// | | classify_line which uses .trim_start() and | | -// | | .is_whitespace() internally. Engine Spec §6 | | -// | | mandates SP+HTAB only. No integration test with | | -// | | U+00A0 or other Unicode WSP verifying that such | | -// | | lines are NOT treated as having leading whitespace. | | -// | | This is a classify.rs bug observable here. | | -// |-----------------------------------|--------------------------------------------------|----------| -// | Engine Spec §5.2 | SPLIT_PHYSICAL_LINES EDGE CASES: split_physical_ | LOW | -// | | lines is a private fn in this file. No direct test | | -// | | for inputs like "\n" (single LF), "\n\n" (two LFs),| | -// | | or "no newline" (no LF). The trailing-newline test | | -// | | covers one case only. | | diff --git a/engine/src/single_line.rs b/engine/src/single_line.rs index 247fcb8..3214fb4 100644 --- a/engine/src/single_line.rs +++ b/engine/src/single_line.rs @@ -96,43 +96,64 @@ mod tests { assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); assert_eq!(cmd.arguments.fence_lang, None); } -} -// ============================================================================= -// TEST GAPS: spec areas this file's functions touch but are not tested -// ============================================================================= -// -// | Spec Section | Gap | Severity | -// |---------------------------------|-------------------------------------------------|----------| -// | Engine Spec §9.3 | NO WARNINGS VECTOR: finalize_single_line returns| LOW | -// | | a Command, not (Command, Vec). The | | -// | | fence path returns warnings but single-line | | -// | | cannot produce warnings per spec, so this is | | -// | | correct. However the asymmetric return types | | -// | | between finalize_single_line and finalize_fence | | -// | | should be documented. | | -// |---------------------------------|-------------------------------------------------|----------| -// | RFC §7.1 | RAW FOR JOINED COMMANDS: When a single-line | MEDIUM | -// | | command spans multiple physical lines via | | -// | | backslash joining (RFC Appendix B.2), the raw | | -// | | field should contain all physical lines with | | -// | | backslashes and LF separators. This function | | -// | | accepts raw as a parameter (the caller builds | | -// | | it), so it's not this function's concern, but | | -// | | no test verifies multi-physical-line raw input. | | -// |---------------------------------|-------------------------------------------------|----------| -// | RFC §7.1 | RAW WITH LEADING WHITESPACE: No test verifies | LOW | -// | | that raw containing leading whitespace (e.g., | | -// | | " /cmd arg") is passed through unmodified. | | -// |---------------------------------|-------------------------------------------------|----------| -// | Engine Spec §3.4 | ARGUMENT MODE SERIALIZATION: Engine Spec says | INFO | -// | | "String serialization is the SDK's | | -// | | responsibility." The enum value | | -// | | ArgumentMode::SingleLine is set correctly, but | | -// | | no test confirms the enum variant name maps to | | -// | | "single-line" (SDK concern, not engine). | | -// |---------------------------------|-------------------------------------------------|----------| -// | (none) | NO PROPERTY TESTS: This is a simple mapping | LOW | -// | | function with no branching logic. Property tests | | -// | | would add minimal value, but one could assert | | -// | | payload == header for all inputs. | | + // ========================================================================= + // Raw passthrough: leading whitespace and joined lines + // RFC §7.1 + // ========================================================================= + + #[test] + fn raw_with_leading_whitespace_preserved() { + // RFC §7.1: raw is the exact source text including leading whitespace. + let h = make_header("cmd", "arg"); + let cmd = finalize_single_line(h, " /cmd arg".into(), 0, LineRange { start_line: 0, end_line: 0 }); + assert_eq!(cmd.raw, " /cmd arg"); + } + + #[test] + fn raw_with_multi_physical_lines_preserved() { + // RFC §7.1 / RFC Appendix B.2: caller builds raw from joined physical + // lines. This function must pass it through unmodified. + let h = make_header("deploy", "prod --region us-west-2"); + let raw = "/deploy prod \\\n --region us-west-2".to_string(); + let cmd = finalize_single_line(h, raw.clone(), 0, LineRange { start_line: 0, end_line: 1 }); + assert_eq!(cmd.raw, raw); + assert_eq!(cmd.range.start_line, 0); + assert_eq!(cmd.range.end_line, 1); + } + + // ========================================================================= + // Property tests + // ========================================================================= + + use proptest::prelude::*; + + proptest! { + // RFC §4.4: in single-line mode header and payload are always identical. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn payload_always_equals_header( + name in "[a-z][a-z0-9]{0,10}", + args in "[a-zA-Z0-9 ]{0,50}" + ) { + let h = make_header(&name, &args); + let raw = format!("/{name} {args}"); + let cmd = finalize_single_line(h, raw, 0, LineRange { start_line: 0, end_line: 0 }); + prop_assert_eq!(&cmd.arguments.payload, &cmd.arguments.header); + } + + // Engine Spec §3.3: fence_lang is always None for single-line commands. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fence_lang_always_none( + name in "[a-z][a-z0-9]{0,10}", + args in "[a-zA-Z0-9 ]{0,50}" + ) { + let h = make_header(&name, &args); + let raw = format!("/{name} {args}"); + let cmd = finalize_single_line(h, raw, 0, LineRange { start_line: 0, end_line: 0 }); + prop_assert_eq!(cmd.arguments.fence_lang, None); + prop_assert_eq!(cmd.arguments.mode, ArgumentMode::SingleLine); + } + } +} diff --git a/engine/src/test_helper.rs b/engine/src/test_helper.rs new file mode 100644 index 0000000..57be879 --- /dev/null +++ b/engine/src/test_helper.rs @@ -0,0 +1,16 @@ +#![allow(dead_code)] + +use proptest::prelude::*; + +use crate::fence::{PendingFence, accept_fence_line}; + +pub(crate) fn valid_command_name() -> impl Strategy { + "[a-z][a-z0-9\\-]{0,15}".prop_filter("no trailing hyphen", |s| !s.ends_with('-')) +} + +pub(crate) fn feed_body(fence: PendingFence, lines: &[String]) -> PendingFence { + lines.iter().enumerate().fold(fence, |f, (i, line)| { + let (next, _) = accept_fence_line(f, i + 1, line); + next + }) +} diff --git a/engine/src/text.rs b/engine/src/text.rs index ced7673..4c97da7 100644 --- a/engine/src/text.rs +++ b/engine/src/text.rs @@ -147,6 +147,54 @@ mod tests { assert_eq!(block.content, "before\n\nafter"); } + // ========================================================================= + // Zero-append finalize (text block boundary at EOF) + // RFC §6.1 / Engine Spec §10.3 + // ========================================================================= + + #[test] + fn finalize_immediately_after_start() { + // RFC §6.1: text block may end on the same line it started (single + // logical line before EOF or command trigger). + let block = finalize_text(start_text(5, "only line"), 2); + assert_eq!(block.id, "text-2"); + assert_eq!(block.content, "only line"); + assert_eq!(block.range.start_line, 5); + assert_eq!(block.range.end_line, 5); + } + + // ========================================================================= + // All-blank-lines text block + // RFC §6.1 + // ========================================================================= + + #[test] + fn all_blank_lines_produce_newline_only_content() { + // RFC §6.1: blank lines are included verbatim. + // Three empty strings joined with "\n" produce "\n\n". + let pt = start_text(0, ""); + let pt = append_text(pt, 1, ""); + let pt = append_text(pt, 2, ""); + let block = finalize_text(pt, 0); + assert_eq!(block.content, "\n\n"); + } + + // ========================================================================= + // Content preservation: verbatim pass-through + // RFC §6.1 + // ========================================================================= + + #[test] + fn content_with_special_characters_preserved() { + // RFC §6.1: text block content preserves lines verbatim. + // Backslashes, backticks, and unicode are not interpreted. + let pt = start_text(0, "trailing backslash\\"); + let pt = append_text(pt, 1, "```not a fence```"); + let pt = append_text(pt, 2, "emoji: 🎉"); + let block = finalize_text(pt, 0); + assert_eq!(block.content, "trailing backslash\\\n```not a fence```\nemoji: 🎉"); + } + // ========================================================================= // Property tests // ========================================================================= diff --git a/engine/src/types.rs b/engine/src/types.rs index 67617c3..2843e20 100644 --- a/engine/src/types.rs +++ b/engine/src/types.rs @@ -1,4 +1,4 @@ -pub const SPEC_VERSION: &str = "0.3.0"; +pub const SPEC_VERSION: &str = "0.5.0"; /// Inclusive line range (zero-based). #[derive(Debug, Clone, PartialEq, Eq)] @@ -50,7 +50,6 @@ pub struct ParseResult { pub warnings: Vec, } - /// Non-fatal conditions detected during parsing. /// /// Warnings are collected in `ParseResult.warnings` rather than diff --git a/engine/tests/orchestration_tests.rs b/engine/tests/orchestration_tests.rs deleted file mode 100644 index 33c1513..0000000 --- a/engine/tests/orchestration_tests.rs +++ /dev/null @@ -1,322 +0,0 @@ -//! Integration tests for parse_document orchestration. -//! -//! These tests verify that the sub-modules are wired together correctly -//! by the state machine in parse.rs. Each test targets a specific -//! orchestration concern that cannot be tested at the unit level. -//! -//! Sub-module behavior (normalization, classification, joining, fence -//! accumulation, text accumulation) is covered by unit tests in each -//! module's #[cfg(test)] block. These tests do NOT re-verify sub-module -//! logic; they verify the decisions the orchestrator makes. - -use slasher_engine::{parse_document, ArgumentMode, SPEC_VERSION}; - -// ============================================================================= -// State: Idle -> single-line command -> Idle -// Orchestration: classify_line returns Command(SingleLine), orchestrator -// calls finalize_single_line, increments cmd_seq, returns to Idle. -// ============================================================================= - -#[test] -fn single_line_returns_to_idle() { - // Two consecutive single-line commands prove the state machine returns - // to Idle after each. If it didn't, the second would be lost. - let r = parse_document("/cmd1 a\n/cmd2 b"); - assert_eq!(r.commands.len(), 2); - assert_eq!(r.commands[0].name, "cmd1"); - assert_eq!(r.commands[1].name, "cmd2"); -} - -// ============================================================================= -// State: Idle -> fence command -> InFence -> Completed -> Idle -// Orchestration: classify_line returns Command(Fence), orchestrator calls -// open_fence, transitions to InFence, feeds physical lines via -// next_physical, detects Completed, finalizes, returns to Idle. -// ============================================================================= - -#[test] -fn fence_completes_and_returns_to_idle() { - // A fenced command followed by a single-line command proves the full - // Idle -> InFence -> Idle cycle. - let r = parse_document("/fenced ```\nbody line\n```\n/after arg"); - assert_eq!(r.commands.len(), 2); - assert_eq!(r.commands[0].arguments.mode, ArgumentMode::Fence); - assert_eq!(r.commands[0].arguments.payload, "body line"); - assert_eq!(r.commands[1].arguments.mode, ArgumentMode::SingleLine); - assert_eq!(r.commands[1].name, "after"); -} - -// ============================================================================= -// State: InFence -> EOF (unclosed) -// Orchestration: next_physical returns None, orchestrator calls -// finalize_fence(fence, true), pushes warning. -// ============================================================================= - -#[test] -fn unclosed_fence_at_eof() { - let r = parse_document("/cmd ```\nline one\nline two"); - assert_eq!(r.commands.len(), 1); - assert_eq!(r.commands[0].arguments.payload, "line one\nline two"); - assert_eq!(r.warnings.len(), 1); - assert_eq!(r.warnings[0].start_line, Some(0)); -} - -// ============================================================================= -// Flush: text flushed before command -// Orchestration: when classify_line returns Command, flush_text is called -// first, finalizing any pending text block. -// ============================================================================= - -#[test] -fn text_flushed_before_command() { - let r = parse_document("prose line\n/cmd arg"); - assert_eq!(r.textblocks.len(), 1); - assert_eq!(r.textblocks[0].content, "prose line"); - assert_eq!(r.textblocks[0].id, "text-0"); - // Text block appears before the command in document order. - assert_eq!(r.textblocks[0].range.end_line, 0); - assert_eq!(r.commands[0].range.start_line, 1); -} - -// ============================================================================= -// Flush: text flushed at EOF -// Orchestration: after the main loop breaks, flush_text is called to -// finalize any trailing text. -// ============================================================================= - -#[test] -fn text_flushed_at_eof() { - let r = parse_document("/cmd arg\ntrailing prose"); - assert_eq!(r.commands.len(), 1); - assert_eq!(r.textblocks.len(), 1); - assert_eq!(r.textblocks[0].content, "trailing prose"); -} - -// ============================================================================= -// Flush: no spurious text blocks -// Orchestration: consecutive commands with no text between them must NOT -// produce an empty text block. -// ============================================================================= - -#[test] -fn no_text_block_between_consecutive_commands() { - let r = parse_document("/cmd1 a\n/cmd2 b\n/cmd3 c"); - assert_eq!(r.commands.len(), 3); - assert!(r.textblocks.is_empty()); -} - -#[test] -fn no_text_block_after_fence_closer_before_command() { - let r = parse_document("/f ```\nbody\n```\n/cmd arg"); - assert_eq!(r.commands.len(), 2); - assert!(r.textblocks.is_empty()); -} - -// ============================================================================= -// Raw wiring: joined single-line command -// Orchestration: step_idle rebuilds raw from physical line slice -// phys[first..=last].join("\n"), preserving backslashes. -// ============================================================================= - -#[test] -fn joined_command_raw_has_physical_lines() { - let r = parse_document("/deploy prod \\\n --region us-west-2"); - assert_eq!(r.commands.len(), 1); - // raw must contain both physical lines with the backslash intact. - assert_eq!(r.commands[0].raw, "/deploy prod \\\n --region us-west-2"); - // But the payload is the joined logical content. - assert!(r.commands[0].arguments.payload.contains("--region")); -} - -// ============================================================================= -// Raw wiring: fenced command -// Orchestration: raw is seeded by open_fence (opener line from phys slice), -// then accept_fence_line appends body and closer. -// ============================================================================= - -#[test] -fn fenced_raw_includes_opener_body_closer() { - let r = parse_document("/cmd ```\nfirst\nsecond\n```"); - assert_eq!(r.commands[0].raw, "/cmd ```\nfirst\nsecond\n```"); -} - -// ============================================================================= -// Physical text: text block uses physical lines, not logical lines -// Orchestration: accumulate_text calls fold_physical_lines over the phys -// slice, so backslash-continued text lines keep their backslashes. -// Engine Spec §10: "A logical line formed by backslash continuation -// contributes all of its constituent physical lines with backslashes -// intact to the text block content." -// ============================================================================= - -#[test] -fn text_block_preserves_physical_lines_with_backslashes() { - // "hello \" and " world" are two physical lines that join into one - // logical line. The text block content must have BOTH physical lines. - let r = parse_document("hello \\\n world"); - assert_eq!(r.textblocks.len(), 1); - assert_eq!(r.textblocks[0].content, "hello \\\n world"); - assert_eq!(r.textblocks[0].range.start_line, 0); - assert_eq!(r.textblocks[0].range.end_line, 1); -} - -// ============================================================================= -// ID counters: independent cmd_seq and text_seq -// Orchestration: cmd_seq and text_seq are separate fields in ParseCtx, -// incremented independently. -// ============================================================================= - -#[test] -fn id_counters_independent_across_interleaving() { - let r = parse_document("t0\n/c0 a\nt1\n/c1 b\nt2"); - assert_eq!(r.textblocks[0].id, "text-0"); - assert_eq!(r.commands[0].id, "cmd-0"); - assert_eq!(r.textblocks[1].id, "text-1"); - assert_eq!(r.commands[1].id, "cmd-1"); - assert_eq!(r.textblocks[2].id, "text-2"); -} - -#[test] -fn fenced_command_shares_cmd_seq_with_single_line() { - // A fence command consumes cmd_seq=0, the next single-line gets cmd_seq=1. - let r = parse_document("/fenced ```\nbody\n```\n/single arg"); - assert_eq!(r.commands[0].id, "cmd-0"); - assert_eq!(r.commands[1].id, "cmd-1"); -} - -// ============================================================================= -// split_physical_lines: trailing LF handling -// Orchestration: split_physical_lines pops trailing empty element so -// "input\n" does not create a phantom empty text block. -// ============================================================================= - -#[test] -fn trailing_lf_no_phantom_text_block() { - let r = parse_document("/cmd arg\n"); - assert_eq!(r.commands.len(), 1); - assert!(r.textblocks.is_empty()); -} - -#[test] -fn no_trailing_lf_still_processes_last_line() { - let r = parse_document("/cmd1 a\n/cmd2 b"); - assert_eq!(r.commands.len(), 2); - assert_eq!(r.commands[1].name, "cmd2"); -} - -// ============================================================================= -// Fence boundary: next_logical resumes after fence closes -// Orchestration: after FenceResult::Completed, state returns to Idle, -// and the next iteration calls next_logical (with joining active). -// This means a backslash-continued line after a fence closer is joined. -// ============================================================================= - -#[test] -fn joining_resumes_after_fence_closes() { - // After the fence closes, "hello \" + " world" should join into one - // logical text line. - let r = parse_document("/cmd ```\nbody\n```\nhello \\\n world"); - assert_eq!(r.commands.len(), 1); - assert_eq!(r.textblocks.len(), 1); - // The text block preserves physical lines (backslash intact). - assert_eq!(r.textblocks[0].content, "hello \\\n world"); - // But it's a single logical line, so only one text block. - assert_eq!(r.textblocks[0].range.start_line, 3); - assert_eq!(r.textblocks[0].range.end_line, 4); -} - -// ============================================================================= -// Fence body: next_physical bypasses joining -// Orchestration: in InFence state, the orchestrator calls next_physical, -// so backslashes inside the fence body are NOT consumed as join markers. -// ============================================================================= - -#[test] -fn backslash_in_fence_body_is_literal() { - let r = parse_document("/cmd ```\nline one \\\nline two\n```"); - assert_eq!(r.commands.len(), 1); - // Both lines are separate payload lines; the backslash did not join them. - assert_eq!(r.commands[0].arguments.payload, "line one \\\nline two"); -} - -// ============================================================================= -// Version wiring -// Orchestration: into_result sets version from SPEC_VERSION. -// ============================================================================= - -#[test] -fn version_from_spec_constant() { - let r = parse_document(""); - assert_eq!(r.version, SPEC_VERSION); -} - -// ============================================================================= -// Full scenario: RFC Appendix B.5 equivalent -// Orchestration: exercises the complete interleaving of text blocks, -// commands, blank lines, and ID assignment in a single document. -// ============================================================================= - -#[test] -fn full_scenario_text_commands_interleaved() { - let input = "Welcome to the system.\n\n/deploy staging\n/notify team --channel ops\nDone."; - let r = parse_document(input); - - // Text block 0: lines 0-1 (prose + blank line) - assert_eq!(r.textblocks[0].id, "text-0"); - assert_eq!(r.textblocks[0].content, "Welcome to the system.\n"); - assert_eq!(r.textblocks[0].range.start_line, 0); - assert_eq!(r.textblocks[0].range.end_line, 1); - - // Commands - assert_eq!(r.commands[0].id, "cmd-0"); - assert_eq!(r.commands[0].name, "deploy"); - assert_eq!(r.commands[0].arguments.payload, "staging"); - - assert_eq!(r.commands[1].id, "cmd-1"); - assert_eq!(r.commands[1].name, "notify"); - assert_eq!(r.commands[1].arguments.payload, "team --channel ops"); - - // Text block 1: trailing prose - assert_eq!(r.textblocks[1].id, "text-1"); - assert_eq!(r.textblocks[1].content, "Done."); -} - -// ============================================================================= -// Empty and minimal inputs -// Orchestration: total function guarantee, no panics, correct empty result. -// ============================================================================= - -#[test] -fn empty_input() { - let r = parse_document(""); - assert!(r.commands.is_empty()); - assert!(r.textblocks.is_empty()); - assert!(r.warnings.is_empty()); -} - -#[test] -fn single_newline_only() { - // "\n" normalizes to one LF. split_physical_lines produces [""] then - // pops it. Result should be empty. - let r = parse_document("\n"); - assert!(r.commands.is_empty()); - assert!(r.textblocks.is_empty()); -} - -// ============================================================================= -// Invalid slash lines flow through as text -// RFC §4.5 / RFC Appendix B.6 -// ============================================================================= - -#[test] -fn invalid_slash_lines_are_text() { - let input = "/123\n/ bare\n/Hello\n/cmd- trailing\n/deploy staging"; - let r = parse_document(input); - // Only "/deploy" is a valid command. - assert_eq!(r.commands.len(), 1); - assert_eq!(r.commands[0].name, "deploy"); - // The four invalid lines form one text block before the command. - assert_eq!(r.textblocks.len(), 1); - assert_eq!(r.textblocks[0].range.start_line, 0); - assert_eq!(r.textblocks[0].range.end_line, 3); -} diff --git a/engine/tests/parse_document.rs b/engine/tests/parse_document.rs new file mode 100644 index 0000000..73c6a9a --- /dev/null +++ b/engine/tests/parse_document.rs @@ -0,0 +1,520 @@ +//! Orchestration tests for the parse.rs state machine. +//! +//! These test the DECISIONS and WIRING that parse.rs performs when +//! composing sub-modules. Sub-module behavior (classify, join, fence, +//! text, normalize, single_line) is covered by their own #[cfg(test)]. +//! Cross-module composition (normalize→join, classify→fence) is covered +//! by integration_tests/. +//! +//! What lives here: +//! - State transitions (Idle ↔ InFence) +//! - flush_text timing +//! - accumulate_text physical-line folding +//! - raw field reconstruction (phys[first..last].join) +//! - split_physical_lines edge cases +//! - cmd_seq / text_seq counter management +//! - ParseResult envelope (version, empty input) + +use slasher_engine::{ArgumentMode, LineRange, SPEC_VERSION, parse_document}; + +// ========================================================================= +// Envelope: ParseResult shape regardless of content +// Engine Spec §14, §3.1, §8.2 +// ========================================================================= + +#[test] +fn empty_input_produces_empty_envelope() { + // Engine Spec §14: version always set. §8.2: total function. + let r = parse_document(""); + assert_eq!(r.version, SPEC_VERSION); + assert!(r.commands.is_empty()); + assert!(r.textblocks.is_empty()); + assert!(r.warnings.is_empty()); +} + +#[test] +fn version_populated_on_any_input() { + // Engine Spec §14: version is SPEC_VERSION regardless of content. + let r = parse_document("/cmd arg"); + assert_eq!(r.version, SPEC_VERSION); +} + +// ========================================================================= +// split_physical_lines: trailing newline handling +// Engine Spec §5.2 (private to parse.rs) +// ========================================================================= + +#[test] +fn trailing_lf_does_not_create_empty_text_block() { + // Engine Spec §5.2: trailing empty element popped after split. + let r = parse_document("/cmd arg\n"); + assert_eq!(r.commands.len(), 1); + assert!(r.textblocks.is_empty()); +} + +#[test] +fn single_lf_produces_one_empty_text_line() { + // Engine Spec §5.2: "\n" → ["", ""], pop trailing → [""]. + let r = parse_document("\n"); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, ""); +} + +// ========================================================================= +// State: Idle → single-line → Idle +// Orchestration: classify returns Command(SingleLine), finalize_single_line +// called, state remains Idle for next iteration. +// ========================================================================= + +#[test] +fn single_line_command_returns_to_idle() { + // Two consecutive single-line commands prove Idle→Idle loop works. + let r = parse_document("/cmd1 a\n/cmd2 b"); + assert_eq!(r.commands.len(), 2); + assert_eq!(r.commands[0].arguments.mode, ArgumentMode::SingleLine); + assert_eq!(r.commands[1].arguments.mode, ArgumentMode::SingleLine); +} + +// ========================================================================= +// State: Idle → InFence → Idle +// Orchestration: classify returns Command(Fence), open_fence called, +// state becomes InFence. Closer detected → finalize_fence, back to Idle. +// ========================================================================= + +#[test] +fn fence_completes_and_returns_to_idle() { + // Fence followed by another command proves Idle restored after fence. + let r = parse_document("/fenced ```\nbody\n```\n/after arg"); + assert_eq!(r.commands.len(), 2); + assert_eq!(r.commands[0].arguments.mode, ArgumentMode::Fence); + assert_eq!(r.commands[1].arguments.mode, ArgumentMode::SingleLine); +} + +// ========================================================================= +// State: InFence → EOF (unclosed) +// Orchestration: loop breaks during InFence, finalize_fence(true) called. +// ========================================================================= + +#[test] +fn unclosed_fence_at_eof_produces_warning_and_partial() { + // RFC §5.2.4: unclosed_fence warning + accumulated payload. + let r = parse_document("/cmd ```\npartial body"); + assert_eq!(r.commands.len(), 1); + assert_eq!(r.commands[0].arguments.payload, "partial body"); + assert_eq!(r.warnings.len(), 1); + assert_eq!(r.warnings[0].wtype, "unclosed_fence"); +} + +#[test] +fn unclosed_fence_with_no_body_at_eof() { + // RFC §5.2.4: fence opened then immediate EOF → empty payload + warning. + let r = parse_document("/cmd ```"); + assert_eq!(r.commands[0].arguments.payload, ""); + assert_eq!(r.warnings.len(), 1); +} + +// ========================================================================= +// flush_text timing +// Orchestration: flush_text called BEFORE start_new_command (on command +// trigger) and AFTER main loop (at EOF). +// ========================================================================= + +#[test] +fn text_flushed_before_command_trigger() { + // Orchestration: text must be finalized before the command is created. + let r = parse_document("prose\n/cmd arg"); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "prose"); + assert_eq!(r.textblocks[0].id, "text-0"); + assert_eq!(r.commands[0].id, "cmd-0"); +} + +#[test] +fn text_flushed_at_eof() { + // Orchestration: trailing text finalized after loop breaks. + let r = parse_document("/cmd arg\ntrailing"); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "trailing"); +} + +#[test] +fn no_spurious_text_between_consecutive_commands() { + // Orchestration: flush_text no-ops when current_text is None. + let r = parse_document("/a x\n/b y\n/c z"); + assert_eq!(r.commands.len(), 3); + assert!(r.textblocks.is_empty()); +} + +#[test] +fn no_spurious_text_after_fence_closer() { + // Fence closer → Idle, immediate command → no empty text block. + let r = parse_document("/f ```\n```\n/cmd arg"); + assert_eq!(r.commands.len(), 2); + assert!(r.textblocks.is_empty()); +} + +#[test] +fn text_after_fence_forms_separate_block() { + // RFC §6.4: text after command gets its own block. + let r = parse_document("/f ```\nbody\n```\ntrailing text"); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "trailing text"); +} + +// ========================================================================= +// accumulate_text: physical-line folding +// Orchestration: text blocks use physical lines (pre-join), not logical. +// This is parse.rs-specific logic via fold_physical_lines. +// ========================================================================= + +#[test] +fn text_block_preserves_backslash_as_physical_lines() { + // Engine Spec §10: text content stores physical lines verbatim. + // The backslash is NOT a join marker for text — it's literal content. + let r = parse_document("hello \\\nworld"); + assert_eq!(r.textblocks[0].content, "hello \\\nworld"); +} + +#[test] +fn text_block_range_covers_physical_lines() { + // Engine Spec §3.6: range spans all physical lines of the text block. + let r = parse_document("line one\nline two\nline three"); + assert_eq!(r.textblocks[0].range.start_line, 0); + assert_eq!(r.textblocks[0].range.end_line, 2); +} + +#[test] +fn multi_line_text_joined_with_lf() { + // RFC §6.1 + Engine Spec §10.3: content = lines.join("\n"). + let r = parse_document("a\nb\nc"); + assert_eq!(r.textblocks[0].content, "a\nb\nc"); +} + +// ========================================================================= +// raw field reconstruction +// Orchestration: step_idle builds raw via phys[first..last].join("\n"). +// This is glue code in parse.rs, not tested by sub-modules. +// ========================================================================= + +#[test] +fn single_line_raw_is_source_text() { + // RFC §7.1: raw = exact source text. + let r = parse_document("/echo hello world"); + assert_eq!(r.commands[0].raw, "/echo hello world"); +} + +#[test] +fn joined_command_raw_preserves_backslashes_and_newlines() { + // RFC §7.1: raw includes physical lines with backslashes and LF. + // Orchestration: header.raw = phys[first..last].join("\n"). + let r = parse_document("/deploy \\\n--region us-west-2"); + assert_eq!(r.commands[0].raw, "/deploy \\\n--region us-west-2"); +} + +#[test] +fn fenced_raw_includes_opener_body_closer() { + // RFC §7.1: raw = opener + body + closer, all joined with LF. + let r = parse_document("/cmd ```\nbody\n```"); + assert_eq!(r.commands[0].raw, "/cmd ```\nbody\n```"); +} + +// ========================================================================= +// Counter management: cmd_seq and text_seq +// Orchestration: independent zero-based counters incremented in +// start_new_command and flush_text respectively. +// ========================================================================= + +#[test] +fn command_ids_sequential_across_modes() { + // RFC §6.5: cmd-0, cmd-1, cmd-2 regardless of single-line vs fence. + let r = parse_document("/a x\n/b ```\nbody\n```\n/c z"); + assert_eq!(r.commands[0].id, "cmd-0"); + assert_eq!(r.commands[1].id, "cmd-1"); + assert_eq!(r.commands[2].id, "cmd-2"); +} + +#[test] +fn text_ids_sequential() { + // RFC §6.5: text-0, text-1. + let r = parse_document("first\n/cmd x\nsecond"); + assert_eq!(r.textblocks[0].id, "text-0"); + assert_eq!(r.textblocks[1].id, "text-1"); +} + +#[test] +fn command_and_text_counters_independent() { + // Engine Spec §3.2 + §3.5: cmd and text sequences are independent. + let r = parse_document("prose\n/cmd arg\nmore prose"); + assert_eq!(r.commands[0].id, "cmd-0"); + assert_eq!(r.textblocks[0].id, "text-0"); + assert_eq!(r.textblocks[1].id, "text-1"); +} + +// ========================================================================= +// Range wiring: orchestrator passes correct physical line indices +// Engine Spec §3.6 +// ========================================================================= + +#[test] +fn single_line_command_range() { + // Orchestration: first_physical == last_physical for non-joined command. + let r = parse_document("text\n/cmd arg"); + assert_eq!(r.commands[0].range, LineRange { start_line: 1, end_line: 1 }); +} + +#[test] +fn joined_command_range_spans_physical_lines() { + // Orchestration: range from logical line's first/last physical. + let r = parse_document("/deploy \\\n--region \\\nus-west-2"); + assert_eq!(r.commands[0].range, LineRange { start_line: 0, end_line: 2 }); +} + +#[test] +fn fenced_command_range_includes_closer() { + // Engine Spec §3.6: range covers opener through closer. + let r = parse_document("/cmd ```\nbody\n```"); + assert_eq!(r.commands[0].range, LineRange { start_line: 0, end_line: 2 }); +} + +// ========================================================================= +// CRLF normalization threading +// Orchestration: normalize() called before split_physical_lines. +// Proves the orchestrator's first pipeline step works. +// ========================================================================= + +#[test] +fn crlf_normalized_before_processing() { + // RFC §3.1: CRLF → LF before any other processing. + let r = parse_document("/cmd arg\r\ntext line"); + assert_eq!(r.commands.len(), 1); + assert_eq!(r.textblocks[0].content, "text line"); +} + +// ========================================================================= +// Full interleaving: mixed document +// Orchestration: all state transitions, flush timing, and counters +// exercised in a single realistic scenario. +// ========================================================================= + +#[test] +fn full_mixed_document() { + // Preamble text → single-line cmd → fenced cmd → trailing text. + let input = "Preamble.\n/cmd1 hello\n/cmd2 ```json\n{\"a\":1}\n```\nEpilogue."; + let r = parse_document(input); + + assert_eq!(r.textblocks.len(), 2); + assert_eq!(r.textblocks[0].content, "Preamble."); + assert_eq!(r.textblocks[0].id, "text-0"); + assert_eq!(r.textblocks[1].content, "Epilogue."); + assert_eq!(r.textblocks[1].id, "text-1"); + + assert_eq!(r.commands.len(), 2); + assert_eq!(r.commands[0].id, "cmd-0"); + assert_eq!(r.commands[0].arguments.mode, ArgumentMode::SingleLine); + assert_eq!(r.commands[1].id, "cmd-1"); + assert_eq!(r.commands[1].arguments.mode, ArgumentMode::Fence); + assert_eq!(r.commands[1].arguments.payload, "{\"a\":1}"); + + assert!(r.warnings.is_empty()); +} + +// ========================================================================= +// Trailing backslash at EOF +// Orchestration: joiner strips trailing backslash when no next line +// exists. State machine must not emit a warning for this case. +// Engine Spec §7.1, RFC §2.2.2 +// ========================================================================= + +#[test] +fn trailing_backslash_at_eof_stripped_silently() { + // §2.2.2: trailing backslash at EOF is removed, no warning emitted. + // Orchestration: joiner consumes the backslash, step_idle receives + // the stripped logical line, classifies it as single-line command. + let r = parse_document("/echo hello\\"); + assert_eq!(r.commands.len(), 1); + assert_eq!(r.commands[0].arguments.payload, "hello"); + assert!(r.warnings.is_empty()); +} + +// --- Text accumulation edge cases --- +// Orchestration: accumulatetext and flushtext handle text-only, whitespace-only, +// blank-line-containing, and between-commands scenarios correctly. + +#[test] +fn whitespace_only_input_is_text() { + // Orchestration: classify returns Text for whitespace-only lines, + // accumulatetext collects them, flushtext emits one text block at EOF. + // RFC §6.3: whitespace-only lines are text lines. + let r = parse_document(" "); + assert!(r.commands.is_empty()); + assert_eq!(r.textblocks.len(), 1); +} + +#[test] +fn text_only_input_produces_single_block() { + // Orchestration: when every logical line classifies as Text, the state + // machine never leaves Idle and all lines accumulate into one block. + // RFC §6.4: consecutive text lines form a single text block. + // RFC §7.2: content is lines joined with LF. + let r = parse_document("line one\ntwo\nthree"); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "line one\ntwo\nthree"); + assert_eq!(r.textblocks[0].range.start_line, 0); + assert_eq!(r.textblocks[0].range.end_line, 2); +} + +#[test] +fn text_between_commands_forms_own_block() { + // Orchestration: flushtext is called before each command trigger, + // so text sandwiched between two single-line commands becomes its + // own block with the correct id and content. + // RFC §6.4: text between two commands forms its own block. + let r = parse_document("/cmd1 a\nmiddle text\n/cmd2 b"); + assert_eq!(r.commands.len(), 2); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "middle text"); +} + +#[test] +fn blank_lines_do_not_split_text_block() { + // Orchestration: blank lines classify as Text, so accumulatetext + // keeps appending them to the current pending block. No flush occurs. + // RFC §6.4: blank lines within a text region are included in the text block. + let r = parse_document("a\n\n\na"); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "a\n\n\na"); +} + +// --- splitphysicallines edge cases --- +// Orchestration: splitphysicallines (private to parse.rs) determines how +// normalized input becomes physical lines. These verify boundary behaviors +// that propagate through the full pipeline. + +#[test] +fn double_lf_produces_two_empty_text_lines() { + // Orchestration: "\n\n" splits to ["", "", ""], trailing pop → ["", ""]. + // Both empty strings classify as Text, accumulate into one block. + // Engine Spec §5.2. + let r = parse_document("\n\n"); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "\n"); +} + +#[test] +fn no_newline_input_is_single_physical_line() { + // Orchestration: "abc" splits to ["abc"], no trailing empty to pop. + // Single text line produces one text block. + // Engine Spec §5.2. + let r = parse_document("abc"); + assert_eq!(r.textblocks.len(), 1); + assert_eq!(r.textblocks[0].content, "abc"); +} + +// ========================================================================= +// Property tests +// ========================================================================= + +use proptest::prelude::*; + +proptest! { + // Engine Spec §8.2: total function, never panics on any input. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn never_panics_on_arbitrary_input(input in "\\PC{0,500}") { + let _ = parse_document(&input); + } + + // Engine Spec §14: version is always SPEC_VERSION. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn version_always_spec_version(input in "\\PC{0,200}") { + let r = parse_document(&input); + prop_assert_eq!(r.version, SPEC_VERSION); + } + + // Orchestration invariant: commands + textblocks partition all + // non-empty physical lines. No line is lost or double-counted. + // Orchestration invariant: commands + textblocks partition all + // non-empty physical lines. No line is lost or double-counted. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn output_items_cover_all_lines(input in "[a-zA-Z0-9/ `\n]{1,300}") { + let r = parse_document(&input); + let line_count = input.lines().count().max(1); + let mut covered = vec![false; line_count]; + + let lines: Vec = r.commands.iter().map(|c| &c.range) + .chain(r.textblocks.iter().map(|t| &t.range)) + .flat_map(|range| range.start_line..=range.end_line) + .filter(|&i| i < line_count) + .collect(); + + for i in lines { + prop_assert!(!covered[i], "line {} double-covered", i); + covered[i] = true; + } + } + + + // Orchestration invariant: command IDs are always sequential. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn command_ids_always_sequential(input in "[a-zA-Z0-9/ `\n]{0,300}") { + let r = parse_document(&input); + for (i, cmd) in r.commands.iter().enumerate() { + prop_assert_eq!(&cmd.id, &format!("cmd-{i}")); + } + } + + // Orchestration invariant: text IDs are always sequential. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_ids_always_sequential(input in "[a-zA-Z0-9/ `\n]{0,300}") { + let r = parse_document(&input); + for (i, tb) in r.textblocks.iter().enumerate() { + prop_assert_eq!(&tb.id, &format!("text-{i}")); + } + } + + // Orchestration invariant: input with no slash-prefixed lines + // produces zero commands and at least one text block. + // §6 + §7: non-command lines are always collected as text. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn text_only_input_produces_zero_commands( + lines in prop::collection::vec("[^/\\n\\r][^\\n\\r]{0,40}", 1..10) + ) { + let input = lines.join("\n"); + let r = parse_document(&input); + prop_assert_eq!(r.commands.len(), 0); + prop_assert!(!r.textblocks.is_empty()); + } + + // Orchestration invariant: fence body passes through the state + // machine verbatim. No joining, no escaping, no transformation. + // §5.2.2: fence body content is opaque payload. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn fenced_content_preserved_verbatim(body in "[^`\\n\\r]{1,80}") { + let input = format!("/cmd ```\n{}\n```", body); + let r = parse_document(&input); + prop_assert_eq!(r.commands.len(), 1); + prop_assert_eq!(&r.commands.first().unwrap().arguments.payload, &body); + } + + // Orchestration invariant: any fence reaching EOF without a closer + // always produces exactly one warning. The state machine must + // call finalize_fence(true) when the loop breaks during InFence. + // §5.2.4: unclosed_fence warning is mandatory. + #[test] + #[cfg_attr(feature = "tdd", ignore)] + fn unclosed_fence_always_produces_warning(body in "[^`]{0,80}") { + let input = format!("/cmd ```\n{}", body); + let r = parse_document(&input); + prop_assert_eq!(r.commands.len(), 1); + prop_assert_eq!(r.warnings.len(), 1); + prop_assert_eq!(&r.warnings.first().unwrap().wtype, "unclosed_fence"); + } + + +}