Improve accuracy of supports_string_literal_concatenation_with_newline

src/ast/comments.rs

@@ -71,7 +71,7 @@ impl Comments {
     /// // all comments appearing before line seven, i.e. before the first statement itself
     /// assert_eq!(
     ///    &comments.find(..Location::new(7, 1)).map(|c| c.as_str()).collect::<Vec<_>>(),
-    ///    &["\n header comment ...\n ... spanning multiple lines\n", " first statement\n"]);
+    ///    &["\n header comment ...\n ... spanning multiple lines\n", " first statement"]);
     ///
     /// // all comments appearing within the first statement
     /// assert_eq!(
@@ -81,7 +81,7 @@ impl Comments {
     /// // all comments appearing within or after the first statement
     /// assert_eq!(
     ///    &comments.find(Location::new(7, 1)..).map(|c| c.as_str()).collect::<Vec<_>>(),
-    ///    &[" world ", " second statement\n", " trailing comment\n"]);
+    ///    &[" world ", " second statement", " trailing comment"]);
     /// ```
     ///
     /// The [Spanned](crate::ast::Spanned) trait allows you to access location

src/ast/mod.rs

@@ -12085,7 +12085,8 @@ impl fmt::Display for OptimizerHint {
                 f.write_str(prefix)?;
                 f.write_str(&self.prefix)?;
                 f.write_str("+")?;
-                f.write_str(&self.text)
+                f.write_str(&self.text)?;
+                f.write_str("\n")
             }
             OptimizerHintStyle::MultiLine => {
                 f.write_str("/*")?;

src/tokenizer.rs

@@ -521,7 +521,7 @@ impl fmt::Display for Whitespace {
             Whitespace::Space => f.write_str(" "),
             Whitespace::Newline => f.write_str("\n"),
             Whitespace::Tab => f.write_str("\t"),
-            Whitespace::SingleLineComment { prefix, comment } => write!(f, "{prefix}{comment}"),
+            Whitespace::SingleLineComment { prefix, comment } => writeln!(f, "{prefix}{comment}"),
             Whitespace::MultiLineComment(s) => write!(f, "/*{s}*/"),
         }
     }
@@ -2037,18 +2037,11 @@ impl<'a> Tokenizer<'a> {
 
     // Consume characters until newline
     fn tokenize_single_line_comment(&self, chars: &mut State) -> String {
-        let mut comment = peeking_take_while(chars, |ch| match ch {
+        peeking_take_while(chars, |ch| match ch {
             '\n' => false,                                           // Always stop at \n
             '\r' if dialect_of!(self is PostgreSqlDialect) => false, // Stop at \r for Postgres
             _ => true, // Keep consuming for other characters
-        });
-
-        if let Some(ch) = chars.next() {
-            assert!(ch == '\n' || ch == '\r');
-            comment.push(ch);
-        }
-
-        comment
+        })
     }
 
     /// Tokenize an identifier or keyword, after the first char is already consumed.
@@ -3346,8 +3339,9 @@ mod tests {
                     Token::Number("0".to_string(), false),
                     Token::Whitespace(Whitespace::SingleLineComment {
                         prefix: "--".to_string(),
-                        comment: "this is a comment\n".to_string(),
+                        comment: "this is a comment".to_string(),
                     }),
+                    Token::Whitespace(Whitespace::Newline),
                     Token::Number("1".to_string(), false),
                 ],
             ),
@@ -3367,8 +3361,9 @@ mod tests {
                     Token::Number("0".to_string(), false),
                     Token::Whitespace(Whitespace::SingleLineComment {
                         prefix: "--".to_string(),
-                        comment: "this is a comment\r\n".to_string(),
+                        comment: "this is a comment\r".to_string(),
                     }),
+                    Token::Whitespace(Whitespace::Newline),
                     Token::Number("1".to_string(), false),
                 ],
             ),
@@ -3392,8 +3387,9 @@ mod tests {
             Token::Number("1".to_string(), false),
             Token::Whitespace(Whitespace::SingleLineComment {
                 prefix: "--".to_string(),
-                comment: "\r".to_string(),
+                comment: "".to_string(),
             }),
+            Token::Whitespace(Whitespace::Newline), // Postgres treats \r as newline in single-line comments
             Token::Number("0".to_string(), false),
         ];
         compare(expected, tokens);
@@ -4220,16 +4216,19 @@ mod tests {
             vec![
                 Token::Whitespace(Whitespace::SingleLineComment {
                     prefix: "--".to_string(),
-                    comment: "\n".to_string(),
+                    comment: "".to_string(),
                 }),
+                Token::Whitespace(Whitespace::Newline),
                 Token::Whitespace(Whitespace::SingleLineComment {
                     prefix: "--".to_string(),
-                    comment: " Table structure for table...\n".to_string(),
+                    comment: " Table structure for table...".to_string(),
                 }),
+                Token::Whitespace(Whitespace::Newline),
                 Token::Whitespace(Whitespace::SingleLineComment {
                     prefix: "--".to_string(),
-                    comment: "\n".to_string(),
+                    comment: "".to_string(),
                 }),
+                Token::Whitespace(Whitespace::Newline),
             ],
         );
     }

tests/sqlparser_comments.rs

@@ -50,10 +50,10 @@ more*/
         vec![
             CommentWithSpan {
                 comment: Comment::SingleLine {
-                    content: " second line comment\n".into(),
+                    content: " second line comment".into(),
                     prefix: "--".into()
                 },
-                span: Span::new((2, 1).into(), (3, 1).into()),
+                span: Span::new((2, 1).into(), (2, 23).into()),
             },
             CommentWithSpan {
                 comment: Comment::MultiLine(" inline comment after `from` ".into()),

tests/sqlparser_common.rs

@@ -18357,6 +18357,14 @@ fn parse_adjacent_string_literal_concatenation() {
         'd'
     )"#;
     dialects.one_statement_parses_to(sql, "SELECT 'abc' IN ('abc', 'd')");
+
+    let sql = r#"
+    SELECT 'abc' in ('a'
+        'b' -- COMMENT
+        'c',
+        'd'
+    )"#;
+    dialects.one_statement_parses_to(sql, "SELECT 'abc' IN ('abc', 'd')");
 }
 
 #[test]

tests/sqlparser_oracle.rs

@@ -364,10 +364,7 @@ fn test_optimizer_hints() {
         "SELECT --+ one two three /* asdf */\n 1 FROM dual",
     );
     assert_eq!(select.optimizer_hints.len(), 1);
-    assert_eq!(
-        select.optimizer_hints[0].text,
-        " one two three /* asdf */\n"
-    );
+    assert_eq!(select.optimizer_hints[0].text, " one two three /* asdf */");
     assert_eq!(select.optimizer_hints[0].prefix, "");
 
     // inserts
@@ -396,7 +393,7 @@ fn test_optimizer_hints() {
     );
     assert_eq!(select.optimizer_hints.len(), 1);
     assert_eq!(select.optimizer_hints[0].prefix, "abc");
-    assert_eq!(select.optimizer_hints[0].text, " text\n");
+    assert_eq!(select.optimizer_hints[0].text, " text");
 }
 
 #[test]

tests/sqlparser_snowflake.rs

@@ -596,8 +596,9 @@ fn test_snowflake_single_line_tokenize() {
         Token::make_keyword("TABLE"),
         Token::Whitespace(Whitespace::SingleLineComment {
             prefix: "#".to_string(),
-            comment: " this is a comment \n".to_string(),
+            comment: " this is a comment ".to_string(),
         }),
+        Token::Whitespace(Whitespace::Newline),
         Token::make_word("table_1", None),
     ];
 
@@ -613,8 +614,9 @@ fn test_snowflake_single_line_tokenize() {
         Token::Whitespace(Whitespace::Space),
         Token::Whitespace(Whitespace::SingleLineComment {
             prefix: "//".to_string(),
-            comment: " this is a comment \n".to_string(),
+            comment: " this is a comment ".to_string(),
         }),
+        Token::Whitespace(Whitespace::Newline),
         Token::make_word("table_1", None),
     ];