Fix bufio.Scanner limit

formatters/basic/formatter_test.go

@@ -15,6 +15,8 @@
 package basic_test
 
 import (
+	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/google/yamlfmt/formatters/basic"
@@ -371,6 +373,28 @@ c: 3
 	}
 }
 
+// TestRetainLineBreaksLongLine is a regression test for a line longer than
+// bufio.Scanner's default 64KiB token limit. The retain-line-breaks feature
+// scans the document line by line; before the fix this failed with
+// "bufio.Scanner: token too long" on real YAML carrying a large single-line
+// value (e.g. SwiftPM .build manifests). See internal/linescan.
+func TestRetainLineBreaksLongLine(t *testing.T) {
+	longValue := strings.Repeat("x", 200*1024) // 200KiB, well past the 64KiB limit
+	input := "a: 1\n\nbig: " + longValue + "\nb: 2\n"
+	for _, single := range []bool{false, true} {
+		t.Run(fmt.Sprintf("single=%t", single), func(t *testing.T) {
+			f, err := factory.NewFormatter(map[string]any{
+				"retain_line_breaks":        true,
+				"retain_line_breaks_single": single,
+			})
+			require.NoError(t, err)
+			got, err := f.Format([]byte(input))
+			require.NoError(t, err)
+			require.Contains(t, string(got), longValue, "the long single-line value must survive formatting")
+		})
+	}
+}
+
 func stripTrailingNewline(s string) string {
 	// strip trailing \n or \r\n characters
 	if len(s) > 0 {

internal/features/trim_whitespace.go

@@ -15,12 +15,11 @@
 package features
 
 import (
-	"bufio"
-	"bytes"
 	"context"
 	"strings"
 
 	"github.com/google/yamlfmt"
+	"github.com/google/yamlfmt/internal/linescan"
 )
 
 func MakeFeatureTrimTrailingWhitespace(linebreakStr string) yamlfmt.Feature {
@@ -32,8 +31,7 @@ func MakeFeatureTrimTrailingWhitespace(linebreakStr string) yamlfmt.Feature {
 
 func trimTrailingWhitespaceFeature(linebreakStr string) yamlfmt.FeatureFunc {
 	return func(_ context.Context, content []byte) (context.Context, []byte, error) {
-		buf := bytes.NewBuffer(content)
-		s := bufio.NewScanner(buf)
+		s := linescan.NewScanner(content)
 		newLines := []string{}
 		for s.Scan() {
 			newLines = append(newLines, strings.TrimRight(s.Text(), " "))

internal/hotfix/retain_line_break.go

@@ -18,12 +18,12 @@
 package hotfix
 
 import (
-	"bufio"
 	"bytes"
 	"context"
 	"strings"
 
 	"github.com/google/yamlfmt"
+	"github.com/google/yamlfmt/internal/linescan"
 )
 
 const lineBreakPlaceholder = "#magic___^_^___line"
@@ -54,8 +54,7 @@ func MakeFeatureRetainLineBreak(linebreakStr string, chomp bool) yamlfmt.Feature
 func replaceLineBreakFeature(newlineStr string, chomp bool) yamlfmt.FeatureFunc {
 	return func(_ context.Context, content []byte) (context.Context, []byte, error) {
 		var buf bytes.Buffer
-		reader := bytes.NewReader(content)
-		scanner := bufio.NewScanner(reader)
+		scanner := linescan.NewScanner(content)
 		var inLineBreaks bool
 		var padding paddinger
 		for scanner.Scan() {
@@ -82,8 +81,7 @@ func replaceLineBreakFeature(newlineStr string, chomp bool) yamlfmt.FeatureFunc
 func restoreLineBreakFeature(newlineStr string) yamlfmt.FeatureFunc {
 	return func(_ context.Context, content []byte) (context.Context, []byte, error) {
 		var buf bytes.Buffer
-		reader := bytes.NewReader(content)
-		scanner := bufio.NewScanner(reader)
+		scanner := linescan.NewScanner(content)
 		for scanner.Scan() {
 			txt := scanner.Text()
 			if strings.TrimSpace(txt) == "" {

internal/hotfix/strip_directives.go

@@ -15,12 +15,11 @@
 package hotfix
 
 import (
-	"bufio"
-	"bytes"
 	"context"
 	"strings"
 
 	"github.com/google/yamlfmt"
+	"github.com/google/yamlfmt/internal/linescan"
 )
 
 type directiveKey string
@@ -51,8 +50,7 @@ func MakeFeatureStripDirectives(lineSepChar string) yamlfmt.Feature {
 func stripDirectivesFeature(lineSepChar string) yamlfmt.FeatureFunc {
 	return func(ctx context.Context, content []byte) (context.Context, []byte, error) {
 		directives := []Directive{}
-		reader := bytes.NewReader(content)
-		scanner := bufio.NewScanner(reader)
+		scanner := linescan.NewScanner(content)
 		result := ""
 		currLine := 1
 		for scanner.Scan() {
@@ -76,8 +74,7 @@ func restoreDirectivesFeature(lineSepChar string) yamlfmt.FeatureFunc {
 		directives := DirectivesFromContext(ctx)
 		directiveIdx := 0
 		doneDirectives := directiveIdx == len(directives)
-		reader := bytes.NewReader(content)
-		scanner := bufio.NewScanner(reader)
+		scanner := linescan.NewScanner(content)
 		result := ""
 		currLine := 1
 		for scanner.Scan() {

internal/linescan/linescan.go

@@ -0,0 +1,29 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package linescan provides a line scanner that is not subject to bufio's
+// default token size limit.
+package linescan
+
+import (
+	"bufio"
+	"bytes"
+)
+
+// NewScanner returns a bufio.Scanner over content that can read a line of any length.
+func NewScanner(content []byte) *bufio.Scanner {
+	scanner := bufio.NewScanner(bytes.NewReader(content))
+	scanner.Buffer(nil, len(content)+1)
+	return scanner
+}

internal/linescan/linescan_test.go

@@ -0,0 +1,60 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package linescan_test
+
+import (
+	"bufio"
+	"strings"
+	"testing"
+
+	"github.com/google/yamlfmt/internal/linescan"
+)
+
+// A line longer than bufio's default token limit must scan without error,
+// which a plain bufio.NewScanner cannot do.
+func TestNewScannerLongLine(t *testing.T) {
+	long := strings.Repeat("x", 10*bufio.MaxScanTokenSize) // 640KiB, one line
+	content := []byte("a\n" + long + "\nb\n")
+
+	var got []string
+	scanner := linescan.NewScanner(content)
+	for scanner.Scan() {
+		got = append(got, scanner.Text())
+	}
+	if err := scanner.Err(); err != nil {
+		t.Fatalf("unexpected scan error: %v", err)
+	}
+
+	want := []string{"a", long, "b"}
+	if len(got) != len(want) {
+		t.Fatalf("got %d lines, want %d", len(got), len(want))
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Fatalf("line %d mismatch: len(got)=%d len(want)=%d", i, len(got[i]), len(want[i]))
+		}
+	}
+}
+
+// Empty content is a valid (zero-line) scan, not an error.
+func TestNewScannerEmpty(t *testing.T) {
+	scanner := linescan.NewScanner(nil)
+	for scanner.Scan() {
+		t.Fatalf("empty content should yield no lines, got %q", scanner.Text())
+	}
+	if err := scanner.Err(); err != nil {
+		t.Fatalf("unexpected scan error: %v", err)
+	}
+}