diff --git a/formatters/basic/formatter_test.go b/formatters/basic/formatter_test.go index 4e32ce4c..9ec0c880 100644 --- a/formatters/basic/formatter_test.go +++ b/formatters/basic/formatter_test.go @@ -15,6 +15,8 @@ package basic_test import ( + "fmt" + "strings" "testing" "github.com/google/yamlfmt/formatters/basic" @@ -371,6 +373,28 @@ c: 3 } } +// TestRetainLineBreaksLongLine is a regression test for a line longer than +// bufio.Scanner's default 64KiB token limit. The retain-line-breaks feature +// scans the document line by line; before the fix this failed with +// "bufio.Scanner: token too long" on real YAML carrying a large single-line +// value (e.g. SwiftPM .build manifests). See internal/linescan. +func TestRetainLineBreaksLongLine(t *testing.T) { + longValue := strings.Repeat("x", 200*1024) // 200KiB, well past the 64KiB limit + input := "a: 1\n\nbig: " + longValue + "\nb: 2\n" + for _, single := range []bool{false, true} { + t.Run(fmt.Sprintf("single=%t", single), func(t *testing.T) { + f, err := factory.NewFormatter(map[string]any{ + "retain_line_breaks": true, + "retain_line_breaks_single": single, + }) + require.NoError(t, err) + got, err := f.Format([]byte(input)) + require.NoError(t, err) + require.Contains(t, string(got), longValue, "the long single-line value must survive formatting") + }) + } +} + func stripTrailingNewline(s string) string { // strip trailing \n or \r\n characters if len(s) > 0 { diff --git a/internal/features/trim_whitespace.go b/internal/features/trim_whitespace.go index 7b5bd636..5f094a31 100644 --- a/internal/features/trim_whitespace.go +++ b/internal/features/trim_whitespace.go @@ -15,12 +15,11 @@ package features import ( - "bufio" - "bytes" "context" "strings" "github.com/google/yamlfmt" + "github.com/google/yamlfmt/internal/linescan" ) func MakeFeatureTrimTrailingWhitespace(linebreakStr string) yamlfmt.Feature { @@ -32,8 +31,7 @@ func MakeFeatureTrimTrailingWhitespace(linebreakStr string) yamlfmt.Feature { func trimTrailingWhitespaceFeature(linebreakStr string) yamlfmt.FeatureFunc { return func(_ context.Context, content []byte) (context.Context, []byte, error) { - buf := bytes.NewBuffer(content) - s := bufio.NewScanner(buf) + s := linescan.NewScanner(content) newLines := []string{} for s.Scan() { newLines = append(newLines, strings.TrimRight(s.Text(), " ")) diff --git a/internal/hotfix/retain_line_break.go b/internal/hotfix/retain_line_break.go index a7a139e6..b11bdd77 100644 --- a/internal/hotfix/retain_line_break.go +++ b/internal/hotfix/retain_line_break.go @@ -18,12 +18,12 @@ package hotfix import ( - "bufio" "bytes" "context" "strings" "github.com/google/yamlfmt" + "github.com/google/yamlfmt/internal/linescan" ) const lineBreakPlaceholder = "#magic___^_^___line" @@ -54,8 +54,7 @@ func MakeFeatureRetainLineBreak(linebreakStr string, chomp bool) yamlfmt.Feature func replaceLineBreakFeature(newlineStr string, chomp bool) yamlfmt.FeatureFunc { return func(_ context.Context, content []byte) (context.Context, []byte, error) { var buf bytes.Buffer - reader := bytes.NewReader(content) - scanner := bufio.NewScanner(reader) + scanner := linescan.NewScanner(content) var inLineBreaks bool var padding paddinger for scanner.Scan() { @@ -82,8 +81,7 @@ func replaceLineBreakFeature(newlineStr string, chomp bool) yamlfmt.FeatureFunc func restoreLineBreakFeature(newlineStr string) yamlfmt.FeatureFunc { return func(_ context.Context, content []byte) (context.Context, []byte, error) { var buf bytes.Buffer - reader := bytes.NewReader(content) - scanner := bufio.NewScanner(reader) + scanner := linescan.NewScanner(content) for scanner.Scan() { txt := scanner.Text() if strings.TrimSpace(txt) == "" { diff --git a/internal/hotfix/strip_directives.go b/internal/hotfix/strip_directives.go index 63e1c6e6..5766fdce 100644 --- a/internal/hotfix/strip_directives.go +++ b/internal/hotfix/strip_directives.go @@ -15,12 +15,11 @@ package hotfix import ( - "bufio" - "bytes" "context" "strings" "github.com/google/yamlfmt" + "github.com/google/yamlfmt/internal/linescan" ) type directiveKey string @@ -51,8 +50,7 @@ func MakeFeatureStripDirectives(lineSepChar string) yamlfmt.Feature { func stripDirectivesFeature(lineSepChar string) yamlfmt.FeatureFunc { return func(ctx context.Context, content []byte) (context.Context, []byte, error) { directives := []Directive{} - reader := bytes.NewReader(content) - scanner := bufio.NewScanner(reader) + scanner := linescan.NewScanner(content) result := "" currLine := 1 for scanner.Scan() { @@ -76,8 +74,7 @@ func restoreDirectivesFeature(lineSepChar string) yamlfmt.FeatureFunc { directives := DirectivesFromContext(ctx) directiveIdx := 0 doneDirectives := directiveIdx == len(directives) - reader := bytes.NewReader(content) - scanner := bufio.NewScanner(reader) + scanner := linescan.NewScanner(content) result := "" currLine := 1 for scanner.Scan() { diff --git a/internal/linescan/linescan.go b/internal/linescan/linescan.go new file mode 100644 index 00000000..0f9d7a9e --- /dev/null +++ b/internal/linescan/linescan.go @@ -0,0 +1,29 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package linescan provides a line scanner that is not subject to bufio's +// default token size limit. +package linescan + +import ( + "bufio" + "bytes" +) + +// NewScanner returns a bufio.Scanner over content that can read a line of any length. +func NewScanner(content []byte) *bufio.Scanner { + scanner := bufio.NewScanner(bytes.NewReader(content)) + scanner.Buffer(nil, len(content)+1) + return scanner +} diff --git a/internal/linescan/linescan_test.go b/internal/linescan/linescan_test.go new file mode 100644 index 00000000..7799d7d7 --- /dev/null +++ b/internal/linescan/linescan_test.go @@ -0,0 +1,60 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linescan_test + +import ( + "bufio" + "strings" + "testing" + + "github.com/google/yamlfmt/internal/linescan" +) + +// A line longer than bufio's default token limit must scan without error, +// which a plain bufio.NewScanner cannot do. +func TestNewScannerLongLine(t *testing.T) { + long := strings.Repeat("x", 10*bufio.MaxScanTokenSize) // 640KiB, one line + content := []byte("a\n" + long + "\nb\n") + + var got []string + scanner := linescan.NewScanner(content) + for scanner.Scan() { + got = append(got, scanner.Text()) + } + if err := scanner.Err(); err != nil { + t.Fatalf("unexpected scan error: %v", err) + } + + want := []string{"a", long, "b"} + if len(got) != len(want) { + t.Fatalf("got %d lines, want %d", len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("line %d mismatch: len(got)=%d len(want)=%d", i, len(got[i]), len(want[i])) + } + } +} + +// Empty content is a valid (zero-line) scan, not an error. +func TestNewScannerEmpty(t *testing.T) { + scanner := linescan.NewScanner(nil) + for scanner.Scan() { + t.Fatalf("empty content should yield no lines, got %q", scanner.Text()) + } + if err := scanner.Err(); err != nil { + t.Fatalf("unexpected scan error: %v", err) + } +}