From 40ed75de31cc66cb9ca2f24af1abe4e2bd752df7 Mon Sep 17 00:00:00 2001 From: Sam Guymer Date: Sat, 11 Jul 2026 09:46:29 +1000 Subject: [PATCH 1/3] Parsing optimizations TODO --- benchmark/README.md | 20 +-- .../src/main/scala/ceesvee/CsvParser.scala | 132 ++++++------------ .../main/scala/ceesvee/CsvParserVector.scala | 95 ++++--------- 3 files changed, 76 insertions(+), 171 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index ef6f626..2d793b0 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -5,21 +5,21 @@ AMD Ryzen 9 9950X ``` # JMH version: 1.37 # VM version: JDK 25.0.3, OpenJDK 64-Bit Server VM, 25.0.3 -Benchmark Mode Cnt Score Error Units -ParserBenchmark.ceesvee avgt 10 272.163 ± 1.733 us/op -ParserBenchmark.ceesveeVector avgt 10 131.243 ± 0.577 us/op -ParserBenchmark.scalaCsv avgt 10 740.304 ± 3.177 us/op -ParserBenchmark.univocity avgt 10 199.074 ± 0.384 us/op +Benchmark Mode Cnt Score Error Units +ParserBenchmark.ceesvee avgt 10 214.007 ± 2.136 us/op +ParserBenchmark.ceesveeVector avgt 10 96.536 ± 0.203 us/op +ParserBenchmark.scalaCsv avgt 10 750.618 ± 1.876 us/op +ParserBenchmark.univocity avgt 10 196.028 ± 1.183 us/op ``` ``` # JMH version: 1.37 # VM version: JDK 25.0.3, OpenJDK 64-Bit Server VM, 25.0.3+9-jvmci-25.1-b19 -Benchmark Mode Cnt Score Error Units -ParserBenchmark.ceesvee avgt 10 194.363 ± 1.911 us/op -ParserBenchmark.ceesveeVector avgt 10 1454.794 ± 11.985 us/op -ParserBenchmark.scalaCsv avgt 10 791.310 ± 1.224 us/op -ParserBenchmark.univocity avgt 10 207.725 ± 0.946 us/op +Benchmark Mode Cnt Score Error Units +ParserBenchmark.ceesvee avgt 10 117.435 ± 0.841 us/op +ParserBenchmark.ceesveeVector avgt 10 969.052 ± 3.997 us/op +ParserBenchmark.scalaCsv avgt 10 782.370 ± 2.014 us/op +ParserBenchmark.univocity avgt 10 202.184 ± 1.849 us/op ``` `benchmark/Jmh/run -i 10 -wi 5 -f 1 -t 2 ceesvee.benchmark.DecoderBenchmark` diff --git a/modules/core/src/main/scala/ceesvee/CsvParser.scala b/modules/core/src/main/scala/ceesvee/CsvParser.scala index 47a9d7f..555fdec 100644 --- a/modules/core/src/main/scala/ceesvee/CsvParser.scala +++ b/modules/core/src/main/scala/ceesvee/CsvParser.scala @@ -86,9 +86,13 @@ object CsvParser { in: Iterator[String], options: Options, )(implicit f: Factory[String, C[String]]): Iterator[C[String]] = { - splitLines(in, options) - .filter(str => !ignoreLine(str, options)) - .map(parseLine(_, options)) + val lines = splitLines(in, options) + val withoutIgnoredLines = if (options.skipBlankRows || options.commentPrefix.exists(_.nonEmpty)) { + lines.filter(str => !ignoreLine(str, options)) + } else { + lines + } + withoutIgnoredLines.map(parseLine(_, options)) } def ignoreLine(line: String, options: Options): Boolean = { @@ -265,29 +269,6 @@ object CsvParser { (State(leftover, insideQuoteIndex = insideQuoteIndex, previousCarriageReturn = previousCarriageReturn), builder.result()) } - private case class Slice(start: Int, end: Int) - private object Slice { - @SuppressWarnings(Array( - "org.wartremover.warts.MutableDataStructures", - "org.wartremover.warts.NonUnitStatements", - "org.wartremover.warts.SeqApply", - "org.wartremover.warts.Var", - "org.wartremover.warts.While", - )) - def slice(slices: mutable.ArrayBuffer[Slice], line: String) = { - val sb = new mutable.StringBuilder - val n = slices.length - var i = 0 - while (i < n) { - val slice = slices(i) - sb append line.substring(slice.start, slice.end) - i = i + 1 - } - slices.clear() - sb.result() - } - } - /** * Parse a line into a collection of CSV fields. */ @@ -302,79 +283,50 @@ object CsvParser { options: Options, )(implicit f: Factory[String, C[String]]): C[String] = { val fields = f.newBuilder - - val slices = mutable.ArrayBuffer.empty[Slice] + val escapedField = new mutable.StringBuilder + val delimiter = options.delimiter match { + case Options.Delimiter.Comma => ',' + case Options.Delimiter.Tab => '\t' + } var sliceStart = 0 - var i = 0 var insideQuote = false while (i < line.length) { - options.delimiter match { - case Options.Delimiter.Comma => - (line(i): @switch) match { - - case ',' => - if (!insideQuote) { - { - slices.addOne(Slice(sliceStart, i)) - fields addOne trimString(options, Slice.slice(slices, line)) - } - i += 1 - sliceStart = i - } else { - i += 1 - } - - case '"' => - if (insideQuote && (i + 1) < line.length && line(i + 1) == '"') { // escaped quote - slices.addOne(Slice(sliceStart, i)) - sliceStart = i + 1 - i += 2 - } else { - i += 1 - insideQuote = !insideQuote - } - - case _ => - i += 1 - } - - case Options.Delimiter.Tab => - (line(i): @switch) match { - - case '\t' => - if (!insideQuote) { - { - slices.addOne(Slice(sliceStart, i)) - fields addOne trimString(options, Slice.slice(slices, line)) - } - i += 1 - sliceStart = i - } else { - i += 1 - } - - case '"' => - if (insideQuote && (i + 1) < line.length && line(i + 1) == '"') { // escaped quote - slices.addOne(Slice(sliceStart, i)) - sliceStart = i + 1 - i += 2 - } else { - i += 1 - insideQuote = !insideQuote - } - - case _ => - i += 1 - } + val char = line.charAt(i) + if (char == delimiter && !insideQuote) { + val field = if (escapedField.isEmpty) { + line.substring(sliceStart, i) + } else { + escapedField.append(line.substring(sliceStart, i)) + val result = escapedField.result() + escapedField.clear() + result + } + fields.addOne(trimString(options, field)) + i += 1 + sliceStart = i + } else if (char == '"') { + if (insideQuote && (i + 1) < line.length && line.charAt(i + 1) == '"') { + escapedField.append(line.substring(sliceStart, i)) + sliceStart = i + 1 + i += 2 + } else { + i += 1 + insideQuote = !insideQuote + } + } else { + i += 1 } } - { - slices.addOne(Slice(sliceStart, i)) - fields addOne trimString(options, Slice.slice(slices, line)) + val field = if (escapedField.isEmpty) { + line.substring(sliceStart, i) + } else { + escapedField.append(line.substring(sliceStart, i)) + escapedField.result() } + fields.addOne(trimString(options, field)) fields.result() } diff --git a/modules/core/src/main/scala/ceesvee/CsvParserVector.scala b/modules/core/src/main/scala/ceesvee/CsvParserVector.scala index 8d66673..44ca7f9 100644 --- a/modules/core/src/main/scala/ceesvee/CsvParserVector.scala +++ b/modules/core/src/main/scala/ceesvee/CsvParserVector.scala @@ -26,9 +26,13 @@ object CsvParserVector { in: Iterator[Array[Byte]], options: Options, )(implicit f: Factory[String, C[String]]): Iterator[C[String]] = { - splitLines(in, options) - .filter(bytes => !CsvParser.ignoreLine(new String(bytes, Utf8), options)) - .map(parseLine(_, options)) + val lines = splitLines(in, options) + val withoutIgnoredLines = if (options.skipBlankRows || options.commentPrefix.exists(_.nonEmpty)) { + lines.filter(bytes => !CsvParser.ignoreLine(new String(bytes, Utf8), options)) + } else { + lines + } + withoutIgnoredLines.map(parseLine(_, options)) } /** @@ -117,35 +121,7 @@ object CsvParserVector { } val quotes = vector.eq(Quote) - var mask = quotes - var betweenQuotes = 0L - - // set all bits between quotes - var quoteStart = if (insideQuote) 0 else -1 - while (mask.anyTrue()) { - val r = mask.firstTrue() - mask = mask.xor(VectorMask.fromLong(vector.species(), 1L << r)) - - if (quoteStart >= 0) { - var j = r - 1 - while (j >= quoteStart) { - betweenQuotes = betweenQuotes | (1L << j) - j = j - 1 - } - quoteStart = -1 - } else { - quoteStart = r - } - } - if (quoteStart >= 0) { - var j = ByteVectorSpecies.length - 1 - while (j >= quoteStart) { - betweenQuotes = betweenQuotes | (1L << j) - j = j - 1 - } - } - - val quoteMask = quotes.or(VectorMask.fromLong(vector.species(), betweenQuotes)) + val quoteMask = VectorMask.fromLong(vector.species(), setAllBitsBetweenQuotes(quotes.toLong, insideQuote)) insideQuote = (quotes.trueCount() + (if (insideQuote) 1 else 0)) % 2 == 1 val crChars = vector.eq(CarriageReturn) @@ -229,35 +205,7 @@ object CsvParserVector { } val quotes = vector.eq(Quote) - var mask = quotes - var betweenQuotes = 0L - - // set all bits between quotes - var quoteStart = if (insideQuote) 0 else -1 - while (mask.anyTrue()) { - val r = mask.firstTrue() - mask = mask.xor(VectorMask.fromLong(vector.species(), 1L << r)) - - if (quoteStart >= 0) { - var j = r - 1 - while (j >= quoteStart) { - betweenQuotes = betweenQuotes | (1L << j) - j = j - 1 - } - quoteStart = -1 - } else { - quoteStart = r - } - } - if (quoteStart >= 0) { - var j = ByteVectorSpecies.length - 1 - while (j >= quoteStart) { - betweenQuotes = betweenQuotes | (1L << j) - j = j - 1 - } - } - - val quoteMask = quotes.or(VectorMask.fromLong(vector.species(), betweenQuotes)) + val quoteMask = VectorMask.fromLong(vector.species(), setAllBitsBetweenQuotes(quotes.toLong, insideQuote)) insideQuote = (quotes.trueCount() + (if (insideQuote) 1 else 0)) % 2 == 1 val delimiterChars = vector.eq(delimiter) @@ -277,7 +225,7 @@ object CsvParserVector { delimiterIgnoringWithinQuotesBitSet ^ java.lang.Long.lowestOneBit(delimiterIgnoringWithinQuotesBitSet) val sliceTo = i + r - val str = handleField(arraySlice(bytes, sliceStart, sliceTo, i, 0), options) + val str = handleField(bytes, sliceStart, sliceTo, options) val _ = builder += str sliceStart = sliceTo + 1 } @@ -285,20 +233,14 @@ object CsvParserVector { i = i + ByteVectorSpecies.length } - val remaining = if (sliceStart == 0) { - bytes - } else { - bytes.slice(sliceStart, bytes.length) - } - - val str = handleField(remaining, options) + val str = handleField(bytes, sliceStart, bytes.length, options) val _ = builder += str builder.result() } - private def handleField(bytes: Array[Byte], options: Options) = { - val str = new String(bytes, Utf8) + private def handleField(bytes: Array[Byte], from: Int, until: Int, options: Options) = { + val str = new String(bytes, from, until - from, Utf8) val trimmed = Options.Trim.True.strip(str) if (trimmed.length >= 2 && trimmed.charAt(0) == '"' && trimmed.charAt(trimmed.length - 1) == '"') { @@ -308,6 +250,17 @@ object CsvParserVector { } } + private def setAllBitsBetweenQuotes(quotes: Long, startsInsideQuote: Boolean) = { + var parity = quotes + parity = parity ^ (parity << 1) + parity = parity ^ (parity << 2) + parity = parity ^ (parity << 4) + parity = parity ^ (parity << 8) + parity = parity ^ (parity << 16) + parity = parity ^ (parity << 32) + (if (startsInsideQuote) ~parity else parity) | quotes + } + private def arraySlice(src: Array[Byte], from: Int, to: Int, offset: Int, ignore: Long) = { var from_ = from var to_ = to From f7edf7ca8238cf08a92793b4634452f387638308 Mon Sep 17 00:00:00 2001 From: Sam Guymer Date: Sat, 11 Jul 2026 10:52:38 +1000 Subject: [PATCH 2/3] avoid filter --- modules/core/src/main/scala/ceesvee/CsvParser.scala | 5 ++++- .../core/src/main/scala/ceesvee/CsvParserVector.scala | 2 +- .../fs2/src/main/scala/ceesvee/fs2/Fs2CsvParser.scala | 10 +++++++++- .../zio/src/main/scala/ceesvee/zio/ZioCsvParser.scala | 9 ++++++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/scala/ceesvee/CsvParser.scala b/modules/core/src/main/scala/ceesvee/CsvParser.scala index 555fdec..2bc1547 100644 --- a/modules/core/src/main/scala/ceesvee/CsvParser.scala +++ b/modules/core/src/main/scala/ceesvee/CsvParser.scala @@ -87,7 +87,7 @@ object CsvParser { options: Options, )(implicit f: Factory[String, C[String]]): Iterator[C[String]] = { val lines = splitLines(in, options) - val withoutIgnoredLines = if (options.skipBlankRows || options.commentPrefix.exists(_.nonEmpty)) { + val withoutIgnoredLines = if (canIgnoreLines(options)) { lines.filter(str => !ignoreLine(str, options)) } else { lines @@ -100,6 +100,9 @@ object CsvParser { ignoreTrimmedLine(l, options) } + private[ceesvee] def canIgnoreLines(options: Options): Boolean = + options.skipBlankRows || options.commentPrefix.exists(_.nonEmpty) + private[ceesvee] def ignoreTrimmedLine(line: String, options: Options): Boolean = { isBlank(line, options) || isComment(line, options) } diff --git a/modules/core/src/main/scala/ceesvee/CsvParserVector.scala b/modules/core/src/main/scala/ceesvee/CsvParserVector.scala index 44ca7f9..6ba6efc 100644 --- a/modules/core/src/main/scala/ceesvee/CsvParserVector.scala +++ b/modules/core/src/main/scala/ceesvee/CsvParserVector.scala @@ -27,7 +27,7 @@ object CsvParserVector { options: Options, )(implicit f: Factory[String, C[String]]): Iterator[C[String]] = { val lines = splitLines(in, options) - val withoutIgnoredLines = if (options.skipBlankRows || options.commentPrefix.exists(_.nonEmpty)) { + val withoutIgnoredLines = if (CsvParser.canIgnoreLines(options)) { lines.filter(bytes => !CsvParser.ignoreLine(new String(bytes, Utf8), options)) } else { lines diff --git a/modules/fs2/src/main/scala/ceesvee/fs2/Fs2CsvParser.scala b/modules/fs2/src/main/scala/ceesvee/fs2/Fs2CsvParser.scala index 4c066bd..eb5c281 100644 --- a/modules/fs2/src/main/scala/ceesvee/fs2/Fs2CsvParser.scala +++ b/modules/fs2/src/main/scala/ceesvee/fs2/Fs2CsvParser.scala @@ -12,6 +12,7 @@ import scala.collection.immutable.ArraySeq object Fs2CsvParser { import CsvParser.Error import CsvParser.State + import CsvParser.canIgnoreLines import CsvParser.ignoreLine import CsvParser.parseLine import CsvParser.splitStrings @@ -25,8 +26,15 @@ object Fs2CsvParser { def parse[F[_]: RaiseThrowable]( options: CsvParser.Options, ): fs2Pipe[F, String, ArraySeq[String]] = { + val withoutIgnoredLines: fs2Pipe[F, String, String] = + if (canIgnoreLines(options)) { + _.filter(str => !ignoreLine(str, options)) + } else { + identity + } + _.through(splitLines(options)) - .filter(str => !ignoreLine(str, options)) + .through(withoutIgnoredLines) .map(parseLine[ArraySeq](_, options)) } diff --git a/modules/zio/src/main/scala/ceesvee/zio/ZioCsvParser.scala b/modules/zio/src/main/scala/ceesvee/zio/ZioCsvParser.scala index 66d2a97..f22634b 100644 --- a/modules/zio/src/main/scala/ceesvee/zio/ZioCsvParser.scala +++ b/modules/zio/src/main/scala/ceesvee/zio/ZioCsvParser.scala @@ -10,6 +10,7 @@ import ceesvee.CsvParser object ZioCsvParser { import CsvParser.Error import CsvParser.State + import CsvParser.canIgnoreLines import CsvParser.ignoreLine import CsvParser.parseLine import CsvParser.splitStrings @@ -20,8 +21,14 @@ object ZioCsvParser { def parse( options: CsvParser.Options, )(implicit trace: ZIOTrace): ZPipeline[Any, Error, String, Chunk[String]] = { + val withoutIgnoredLines = if (canIgnoreLines(options)) { + ZPipeline.filter[String](str => !ignoreLine(str, options)) + } else { + ZPipeline.identity[String] + } + splitLines(options) >>> - ZPipeline.filter[String](str => !ignoreLine(str, options)) >>> + withoutIgnoredLines >>> ZPipeline.map(parseLine[Chunk](_, options)) } From bb2cac0e7fd014464f76035332b497417b514046 Mon Sep 17 00:00:00 2001 From: Sam Guymer Date: Sat, 11 Jul 2026 10:55:42 +1000 Subject: [PATCH 3/3] 3 --- modules/core/src/main/scala/ceesvee/CsvParser.scala | 7 ++++--- .../src/main/scala/ceesvee/zio/ZioCsvParserVector.scala | 8 +++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/scala/ceesvee/CsvParser.scala b/modules/core/src/main/scala/ceesvee/CsvParser.scala index 2bc1547..c5bb250 100644 --- a/modules/core/src/main/scala/ceesvee/CsvParser.scala +++ b/modules/core/src/main/scala/ceesvee/CsvParser.scala @@ -95,14 +95,15 @@ object CsvParser { withoutIgnoredLines.map(parseLine(_, options)) } + private[ceesvee] def canIgnoreLines(options: Options): Boolean = { + options.skipBlankRows || options.commentPrefix.exists(_.nonEmpty) + } + def ignoreLine(line: String, options: Options): Boolean = { val l = options.trim.strip(line) ignoreTrimmedLine(l, options) } - private[ceesvee] def canIgnoreLines(options: Options): Boolean = - options.skipBlankRows || options.commentPrefix.exists(_.nonEmpty) - private[ceesvee] def ignoreTrimmedLine(line: String, options: Options): Boolean = { isBlank(line, options) || isComment(line, options) } diff --git a/modules/zio/src/main/scala/ceesvee/zio/ZioCsvParserVector.scala b/modules/zio/src/main/scala/ceesvee/zio/ZioCsvParserVector.scala index f61515c..ac321bb 100644 --- a/modules/zio/src/main/scala/ceesvee/zio/ZioCsvParserVector.scala +++ b/modules/zio/src/main/scala/ceesvee/zio/ZioCsvParserVector.scala @@ -22,8 +22,14 @@ object ZioCsvParserVector { def parse( options: CsvParser.Options, )(implicit trace: ZIOTrace): ZPipeline[Any, Error, Byte, Chunk[String]] = { + val withoutIgnoredLines = if (CsvParser.canIgnoreLines(options)) { + ZPipeline.filter[Array[Byte]](bytes => !CsvParser.ignoreLine(new String(bytes, StandardCharsets.UTF_8), options)) + } else { + ZPipeline.identity[Array[Byte]] + } + _splitLines(options) >>> - ZPipeline.filter[Array[Byte]](bytes => !CsvParser.ignoreLine(new String(bytes, StandardCharsets.UTF_8), options)) >>> + withoutIgnoredLines >>> ZPipeline.map(parseLine[Chunk](_, options)) }