Skip to content

Parsing optimizations #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
guymers merged 3 commits into from
Jul 11, 2026
Merged

Parsing optimizations #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
40ed75d
Parsing optimizations
guymers Jul 10, 2026
f7edf7c
avoid filter
guymers Jul 11, 2026
bb2cac0
3
guymers Jul 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions benchmark/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,21 @@ AMD Ryzen 9 9950X
```
# JMH version: 1.37
# VM version: JDK 25.0.3, OpenJDK 64-Bit Server VM, 25.0.3
Benchmark Mode Cnt Score Error Units
ParserBenchmark.ceesvee avgt 10 272.163 ± 1.733 us/op
ParserBenchmark.ceesveeVector avgt 10 131.243 ± 0.577 us/op
ParserBenchmark.scalaCsv avgt 10 740.304 ± 3.177 us/op
ParserBenchmark.univocity avgt 10 199.074 ± 0.384 us/op
Benchmark Mode Cnt Score Error Units
ParserBenchmark.ceesvee avgt 10 214.007 ± 2.136 us/op
ParserBenchmark.ceesveeVector avgt 10 96.536 ± 0.203 us/op
ParserBenchmark.scalaCsv avgt 10 750.618 ± 1.876 us/op
ParserBenchmark.univocity avgt 10 196.028 ± 1.183 us/op
```

```
# JMH version: 1.37
# VM version: JDK 25.0.3, OpenJDK 64-Bit Server VM, 25.0.3+9-jvmci-25.1-b19
Benchmark Mode Cnt Score Error Units
ParserBenchmark.ceesvee avgt 10 194.363 ± 1.911 us/op
ParserBenchmark.ceesveeVector avgt 10 1454.794 ± 11.985 us/op
ParserBenchmark.scalaCsv avgt 10 791.310 ± 1.224 us/op
ParserBenchmark.univocity avgt 10 207.725 ± 0.946 us/op
Benchmark Mode Cnt Score Error Units
ParserBenchmark.ceesvee avgt 10 117.435 ± 0.841 us/op
ParserBenchmark.ceesveeVector avgt 10 969.052 ± 3.997 us/op
ParserBenchmark.scalaCsv avgt 10 782.370 ± 2.014 us/op
ParserBenchmark.univocity avgt 10 202.184 ± 1.849 us/op
```

`benchmark/Jmh/run -i 10 -wi 5 -f 1 -t 2 ceesvee.benchmark.DecoderBenchmark`
Expand Down
136 changes: 46 additions & 90 deletions modules/core/src/main/scala/ceesvee/CsvParser.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,17 @@ object CsvParser {
in: Iterator[String],
options: Options,
)(implicit f: Factory[String, C[String]]): Iterator[C[String]] = {
splitLines(in, options)
.filter(str => !ignoreLine(str, options))
.map(parseLine(_, options))
val lines = splitLines(in, options)
val withoutIgnoredLines = if (canIgnoreLines(options)) {
lines.filter(str => !ignoreLine(str, options))
} else {
lines
}
withoutIgnoredLines.map(parseLine(_, options))
}

private[ceesvee] def canIgnoreLines(options: Options): Boolean = {
options.skipBlankRows || options.commentPrefix.exists(_.nonEmpty)
}

def ignoreLine(line: String, options: Options): Boolean = {
Expand Down Expand Up @@ -265,29 +273,6 @@ object CsvParser {
(State(leftover, insideQuoteIndex = insideQuoteIndex, previousCarriageReturn = previousCarriageReturn), builder.result())
}

private case class Slice(start: Int, end: Int)
private object Slice {
@SuppressWarnings(Array(
"org.wartremover.warts.MutableDataStructures",
"org.wartremover.warts.NonUnitStatements",
"org.wartremover.warts.SeqApply",
"org.wartremover.warts.Var",
"org.wartremover.warts.While",
))
def slice(slices: mutable.ArrayBuffer[Slice], line: String) = {
val sb = new mutable.StringBuilder
val n = slices.length
var i = 0
while (i < n) {
val slice = slices(i)
sb append line.substring(slice.start, slice.end)
i = i + 1
}
slices.clear()
sb.result()
}
}

/**
* Parse a line into a collection of CSV fields.
*/
Expand All @@ -302,79 +287,50 @@ object CsvParser {
options: Options,
)(implicit f: Factory[String, C[String]]): C[String] = {
val fields = f.newBuilder

val slices = mutable.ArrayBuffer.empty[Slice]
val escapedField = new mutable.StringBuilder
val delimiter = options.delimiter match {
case Options.Delimiter.Comma => ','
case Options.Delimiter.Tab => '\t'
}
var sliceStart = 0

var i = 0
var insideQuote = false

while (i < line.length) {
options.delimiter match {
case Options.Delimiter.Comma =>
(line(i): @switch) match {

case ',' =>
if (!insideQuote) {
{
slices.addOne(Slice(sliceStart, i))
fields addOne trimString(options, Slice.slice(slices, line))
}
i += 1
sliceStart = i
} else {
i += 1
}

case '"' =>
if (insideQuote && (i + 1) < line.length && line(i + 1) == '"') { // escaped quote
slices.addOne(Slice(sliceStart, i))
sliceStart = i + 1
i += 2
} else {
i += 1
insideQuote = !insideQuote
}

case _ =>
i += 1
}

case Options.Delimiter.Tab =>
(line(i): @switch) match {

case '\t' =>
if (!insideQuote) {
{
slices.addOne(Slice(sliceStart, i))
fields addOne trimString(options, Slice.slice(slices, line))
}
i += 1
sliceStart = i
} else {
i += 1
}

case '"' =>
if (insideQuote && (i + 1) < line.length && line(i + 1) == '"') { // escaped quote
slices.addOne(Slice(sliceStart, i))
sliceStart = i + 1
i += 2
} else {
i += 1
insideQuote = !insideQuote
}

case _ =>
i += 1
}
val char = line.charAt(i)
if (char == delimiter && !insideQuote) {
val field = if (escapedField.isEmpty) {
line.substring(sliceStart, i)
} else {
escapedField.append(line.substring(sliceStart, i))
val result = escapedField.result()
escapedField.clear()
result
}
fields.addOne(trimString(options, field))
i += 1
sliceStart = i
} else if (char == '"') {
if (insideQuote && (i + 1) < line.length && line.charAt(i + 1) == '"') {
escapedField.append(line.substring(sliceStart, i))
sliceStart = i + 1
i += 2
} else {
i += 1
insideQuote = !insideQuote
}
} else {
i += 1
}
}

{
slices.addOne(Slice(sliceStart, i))
fields addOne trimString(options, Slice.slice(slices, line))
val field = if (escapedField.isEmpty) {
line.substring(sliceStart, i)
} else {
escapedField.append(line.substring(sliceStart, i))
escapedField.result()
}
fields.addOne(trimString(options, field))

fields.result()
}
Expand Down
95 changes: 24 additions & 71 deletions modules/core/src/main/scala/ceesvee/CsvParserVector.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,13 @@ object CsvParserVector {
in: Iterator[Array[Byte]],
options: Options,
)(implicit f: Factory[String, C[String]]): Iterator[C[String]] = {
splitLines(in, options)
.filter(bytes => !CsvParser.ignoreLine(new String(bytes, Utf8), options))
.map(parseLine(_, options))
val lines = splitLines(in, options)
val withoutIgnoredLines = if (CsvParser.canIgnoreLines(options)) {
lines.filter(bytes => !CsvParser.ignoreLine(new String(bytes, Utf8), options))
} else {
lines
}
withoutIgnoredLines.map(parseLine(_, options))
}

/**
Expand Down Expand Up @@ -117,35 +121,7 @@ object CsvParserVector {
}

val quotes = vector.eq(Quote)
var mask = quotes
var betweenQuotes = 0L

// set all bits between quotes
var quoteStart = if (insideQuote) 0 else -1
while (mask.anyTrue()) {
val r = mask.firstTrue()
mask = mask.xor(VectorMask.fromLong(vector.species(), 1L << r))

if (quoteStart >= 0) {
var j = r - 1
while (j >= quoteStart) {
betweenQuotes = betweenQuotes | (1L << j)
j = j - 1
}
quoteStart = -1
} else {
quoteStart = r
}
}
if (quoteStart >= 0) {
var j = ByteVectorSpecies.length - 1
while (j >= quoteStart) {
betweenQuotes = betweenQuotes | (1L << j)
j = j - 1
}
}

val quoteMask = quotes.or(VectorMask.fromLong(vector.species(), betweenQuotes))
val quoteMask = VectorMask.fromLong(vector.species(), setAllBitsBetweenQuotes(quotes.toLong, insideQuote))
insideQuote = (quotes.trueCount() + (if (insideQuote) 1 else 0)) % 2 == 1

val crChars = vector.eq(CarriageReturn)
Expand Down Expand Up @@ -229,35 +205,7 @@ object CsvParserVector {
}

val quotes = vector.eq(Quote)
var mask = quotes
var betweenQuotes = 0L

// set all bits between quotes
var quoteStart = if (insideQuote) 0 else -1
while (mask.anyTrue()) {
val r = mask.firstTrue()
mask = mask.xor(VectorMask.fromLong(vector.species(), 1L << r))

if (quoteStart >= 0) {
var j = r - 1
while (j >= quoteStart) {
betweenQuotes = betweenQuotes | (1L << j)
j = j - 1
}
quoteStart = -1
} else {
quoteStart = r
}
}
if (quoteStart >= 0) {
var j = ByteVectorSpecies.length - 1
while (j >= quoteStart) {
betweenQuotes = betweenQuotes | (1L << j)
j = j - 1
}
}

val quoteMask = quotes.or(VectorMask.fromLong(vector.species(), betweenQuotes))
val quoteMask = VectorMask.fromLong(vector.species(), setAllBitsBetweenQuotes(quotes.toLong, insideQuote))
insideQuote = (quotes.trueCount() + (if (insideQuote) 1 else 0)) % 2 == 1

val delimiterChars = vector.eq(delimiter)
Expand All @@ -277,28 +225,22 @@ object CsvParserVector {
delimiterIgnoringWithinQuotesBitSet ^ java.lang.Long.lowestOneBit(delimiterIgnoringWithinQuotesBitSet)

val sliceTo = i + r
val str = handleField(arraySlice(bytes, sliceStart, sliceTo, i, 0), options)
val str = handleField(bytes, sliceStart, sliceTo, options)
val _ = builder += str
sliceStart = sliceTo + 1
}

i = i + ByteVectorSpecies.length
}

val remaining = if (sliceStart == 0) {
bytes
} else {
bytes.slice(sliceStart, bytes.length)
}

val str = handleField(remaining, options)
val str = handleField(bytes, sliceStart, bytes.length, options)
val _ = builder += str

builder.result()
}

private def handleField(bytes: Array[Byte], options: Options) = {
val str = new String(bytes, Utf8)
private def handleField(bytes: Array[Byte], from: Int, until: Int, options: Options) = {
val str = new String(bytes, from, until - from, Utf8)
val trimmed = Options.Trim.True.strip(str)

if (trimmed.length >= 2 && trimmed.charAt(0) == '"' && trimmed.charAt(trimmed.length - 1) == '"') {
Expand All @@ -308,6 +250,17 @@ object CsvParserVector {
}
}

private def setAllBitsBetweenQuotes(quotes: Long, startsInsideQuote: Boolean) = {
var parity = quotes
parity = parity ^ (parity << 1)
parity = parity ^ (parity << 2)
parity = parity ^ (parity << 4)
parity = parity ^ (parity << 8)
parity = parity ^ (parity << 16)
parity = parity ^ (parity << 32)
(if (startsInsideQuote) ~parity else parity) | quotes
}

private def arraySlice(src: Array[Byte], from: Int, to: Int, offset: Int, ignore: Long) = {
var from_ = from
var to_ = to
Expand Down
10 changes: 9 additions & 1 deletion modules/fs2/src/main/scala/ceesvee/fs2/Fs2CsvParser.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import scala.collection.immutable.ArraySeq
object Fs2CsvParser {
import CsvParser.Error
import CsvParser.State
import CsvParser.canIgnoreLines
import CsvParser.ignoreLine
import CsvParser.parseLine
import CsvParser.splitStrings
Expand All @@ -25,8 +26,15 @@ object Fs2CsvParser {
def parse[F[_]: RaiseThrowable](
options: CsvParser.Options,
): fs2Pipe[F, String, ArraySeq[String]] = {
val withoutIgnoredLines: fs2Pipe[F, String, String] =
if (canIgnoreLines(options)) {
_.filter(str => !ignoreLine(str, options))
} else {
identity
}

_.through(splitLines(options))
.filter(str => !ignoreLine(str, options))
.through(withoutIgnoredLines)
.map(parseLine[ArraySeq](_, options))
}

Expand Down
9 changes: 8 additions & 1 deletion modules/zio/src/main/scala/ceesvee/zio/ZioCsvParser.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import ceesvee.CsvParser
object ZioCsvParser {
import CsvParser.Error
import CsvParser.State
import CsvParser.canIgnoreLines
import CsvParser.ignoreLine
import CsvParser.parseLine
import CsvParser.splitStrings
Expand All @@ -20,8 +21,14 @@ object ZioCsvParser {
def parse(
options: CsvParser.Options,
)(implicit trace: ZIOTrace): ZPipeline[Any, Error, String, Chunk[String]] = {
val withoutIgnoredLines = if (canIgnoreLines(options)) {
ZPipeline.filter[String](str => !ignoreLine(str, options))
} else {
ZPipeline.identity[String]
}

splitLines(options) >>>
ZPipeline.filter[String](str => !ignoreLine(str, options)) >>>
withoutIgnoredLines >>>
ZPipeline.map(parseLine[Chunk](_, options))
}

Expand Down
Loading