From 037d1b5d826a74ee08c15ff787cb0e1d8fb6ddce Mon Sep 17 00:00:00 2001 From: Kacy Fortner Date: Tue, 11 Aug 2026 00:30:08 +0000 Subject: [PATCH] perf: build std string encoders and escapers in linear time these functions accumulated their output with `s = s + piece` in a loop, so each append recopied the whole string built so far and the work grew quadratically with the input. rebuild each one through a byte buffer or a list join and materialize the string once at the end, the same shape #686 gave base64. the network-facing hot paths are url percent encode/decode (every query param, form body, and csrf token), json string unescaping inside parse, grpc-message percent coding, and template `<% for %>` rendering. the rest are std primitives a lot of higher-level code sits on: hex, base32, and base58 encoding; the log, metrics, and html escapers; strings swap_case, reverse, and repeat; text fold, sanitize, from_chars, from_code_points; regex replace_all; term strip; collections join_with; yaml quoted-scalar decoding; and config json string reading. output is byte-identical: the encoders and escapers are pinned against python's urllib, json, and binascii and the rfc base32/base58 vectors at sizes from 0 to 65537, and a new golden test locks the user-visible ones. --- std/collections.pith | 11 ++-- std/config.pith | 15 +++-- std/encoding.pith | 50 ++++++++++++----- std/html.pith | 24 +++++--- std/json.pith | 39 +++++++------ std/log.pith | 30 ++++++---- std/metrics.pith | 36 +++++++----- std/net/grpc.pith | 34 +++++++---- std/net/url.pith | 38 +++++++++---- std/regex.pith | 11 +++- std/strings.pith | 31 ++++++---- std/template.pith | 35 +++++++++--- std/term.pith | 9 ++- std/text.pith | 42 +++++++++----- std/yaml.pith | 51 ++++++++++------- .../cases/test_string_builder_linearity.pith | 56 +++++++++++++++++++ .../test_string_builder_linearity.txt | 13 +++++ 17 files changed, 368 insertions(+), 157 deletions(-) create mode 100644 tests/cases/test_string_builder_linearity.pith create mode 100644 tests/expected/test_string_builder_linearity.txt diff --git a/std/collections.pith b/std/collections.pith index 2a7967aa..13165ce6 100644 --- a/std/collections.pith +++ b/std/collections.pith @@ -570,12 +570,15 @@ pub fn join_with(list: List[String], sep: String) -> String: n := list.len() if n == 0: return "" - mut result := list[0] - mut i := 1 + # copy into a local list and join once: appending each element to the + # result string recopied everything accumulated so far, which made the + # join quadratic in the total length. + mut parts: List[String] := [] + mut i := 0 while i < n: - result = result + sep + list[i] + parts.push(list[i]) i = i + 1 - return result + return parts.join(sep) test "collection constructors and aggregations work": nums := range(1, 5) diff --git a/std/config.pith b/std/config.pith index 60bc481d..fa8a5f8a 100644 --- a/std/config.pith +++ b/std/config.pith @@ -116,21 +116,24 @@ fn config_json_skip_ws(input: String, position: Int) -> Int: fn config_json_read_string(input: String, position: Int) -> ConfigStringScan: mut pos := position - mut out := "" if pos >= input.len() or input[pos] != chr(34): - return ConfigStringScan(out, pos) + return ConfigStringScan("", pos) + # collect the pieces and join once at the end: appending each character + # to the result string recopied everything accumulated so far, which + # made reading a long string value quadratic in its length. + mut parts: List[String] := [] pos = pos + 1 while pos < input.len(): if input[pos] == chr(92): if pos + 1 < input.len(): - out = out + input[pos + 1] + parts.push(input[pos + 1]) pos = pos + 2 continue if input[pos] == chr(34): - return ConfigStringScan(out, pos + 1) - out = out + input[pos] + return ConfigStringScan(parts.join(""), pos + 1) + parts.push(input[pos]) pos = pos + 1 - return ConfigStringScan(out, pos) + return ConfigStringScan(parts.join(""), pos) fn config_json_starts_at(input: String, position: Int, text: String) -> Bool: if position + text.len() > input.len(): diff --git a/std/encoding.pith b/std/encoding.pith index f4ad9770..8d9793d3 100644 --- a/std/encoding.pith +++ b/std/encoding.pith @@ -235,7 +235,12 @@ fn encoding_b64_decode_utf8(input: String) -> String!: return b64_decode_utf8(input) fn b32_encode_with(input: Bytes, table: String, padded: Bool) -> String: - mut result := "" + # appending one alphabet character at a time to the result string + # recopied the whole accumulated string, which made base32 encoding + # quadratic in the input; a byte buffer keeps each append constant. + alphabet := bytes.from_string_utf8(table) + out := bytes.buffer_with_capacity((input.len() + 4) / 5 * 8) + defer out.free() mut buffer := 0 mut bits_left := 0 mut i := 0 @@ -244,17 +249,20 @@ fn b32_encode_with(input: Bytes, table: String, padded: Bool) -> String: bits_left = bits_left + 8 while bits_left >= 5: index := buffer / pow2(bits_left - 5) - result = result + table[index % 32] + out.write_byte(alphabet[index % 32]) bits_left = bits_left - 5 buffer = buffer % pow2(bits_left) i = i + 1 if bits_left > 0: index := buffer * pow2(5 - bits_left) - result = result + table[index % 32] + out.write_byte(alphabet[index % 32]) if padded: - while result.len() % 8 != 0: - result = result + "=" - return result + while out.len() % 8 != 0: + out.write_word(PAD_BYTE, 1) + # every byte written above came from an ascii alphabet or is "=", so the + # decode cannot fail; an empty result would be a visible total failure + # rather than a silently wrong encoding. + return out.bytes().to_string_utf8() catch "" fn b32_decode_with(input: String, table: String, padded: Bool, ignore_ws: Bool) -> Bytes!: cleaned := clean_encoded(input, ignore_ws)! @@ -345,19 +353,27 @@ pub fn base58_encode(input: Bytes) -> String: carry = carry / 58 i = i + 1 - mut result := "" + # appending one alphabet character at a time to the result string + # recopied the whole accumulated string, which made base58 encoding + # quadratic in the output; a byte buffer keeps each append constant. + table := bytes.from_string_utf8(alphabet) + out := bytes.buffer_with_capacity(zeroes + digits.len()) + defer out.free() i = 0 while i < zeroes: - result = result + alphabet[0] + out.write_byte(table[0]) i = i + 1 mut j := digits.len() - 1 while j >= 0 and digits.get(j) == 0: j = j - 1 while j >= 0: - result = result + alphabet[digits.get(j)] + out.write_byte(table[digits.get(j)]) j = j - 1 - return result + # every byte written above came from an ascii alphabet, so the decode + # cannot fail; an empty result would be a visible total failure rather + # than a silently wrong encoding. + return out.bytes().to_string_utf8() catch "" # decode bitcoin base58 into bytes. pub fn base58_decode(input: String) -> Bytes!: @@ -413,13 +429,21 @@ fn hex_nibble(c: String) -> Int: return - 1 fn hex_with(input: Bytes, digits: String) -> String: - mut result := "" + # appending each digit pair to the result string recopied the whole + # accumulated string, which made hex encoding quadratic in the input. + # both digits of a byte pack into one two-byte buffer append. + table := bytes.from_string_utf8(digits) + out := bytes.buffer_with_capacity(input.len() * 2) + defer out.free() mut position := 0 while position < input.len(): byte_value := input[position] - result = result + digits[byte_value / 16] + digits[byte_value % 16] + out.write_word(table[byte_value / 16] + table[byte_value % 16] * 256, 2) position = position + 1 - return result + # every byte written above came from an ascii digit table, so the decode + # cannot fail; an empty result would be a visible total failure rather + # than a silently wrong encoding. + return out.bytes().to_string_utf8() catch "" # encode bytes as lowercase hexadecimal. pub fn to_hex(input: Bytes) -> String: diff --git a/std/html.pith b/std/html.pith index 3d665caa..2b9bcc22 100644 --- a/std/html.pith +++ b/std/html.pith @@ -107,23 +107,28 @@ fn entity_for(byte: Int) -> String: # >= 0x80, so no character is ever split by an entity and non-ascii text is # copied through whole. pub fn escape(input: String) -> String: + # collect the runs and entities in a list and join once at the end: + # appending each piece to the result string recopied everything + # accumulated so far, which made escaping quadratic in the number of + # entities — input that alternates a plain character with an escaped one + # hit the worst case. length := input.len() - mut out := "" + mut parts: List[String] := [] mut copied := 0 mut position := 0 while position < length: entity := entity_for(ord(input[position])) if entity != "": if position > copied: - out = out + input.substring(copied, position) - out = out + entity + parts.push(input.substring(copied, position)) + parts.push(entity) copied = position + 1 position = position + 1 if copied == 0: return input if copied < length: - out = out + input.substring(copied, length) - return out + parts.push(input.substring(copied, length)) + return parts.join("") # Whether `input` contains a character `escape` would rewrite. # @@ -161,14 +166,17 @@ fn scheme_allowed(scheme: String) -> Bool: # happily run. stripping first means the allowlist sees the same scheme the # browser will. fn strip_url_noise(input: String) -> String: - mut out := "" + # collect the kept characters and join once at the end: appending each + # one to the result string recopied everything accumulated so far, which + # made stripping quadratic in the url. + mut parts: List[String] := [] mut position := 0 while position < input.len(): byte := ord(input[position]) if byte > 32 and byte != 127: - out = out + input[position] + parts.push(input[position]) position = position + 1 - return out + return parts.join("") # true when `input` begins with a sequence a browser may read as the start of # a protocol-relative url, which navigates off-site under the current scheme. diff --git a/std/json.pith b/std/json.pith index 2177074c..1e77947b 100644 --- a/std/json.pith +++ b/std/json.pith @@ -412,38 +412,41 @@ fn json_utf8_encode(code: Int) -> String: # quotes). returns the decoded text and whether every escape was valid. # rfc 8259 escapes only; a \u surrogate pair combines into one code point. fn json_unescape_text(raw: String) -> (String, Bool): - mut out := "" + # collect the pieces and join once at the end: appending to the result + # string one character at a time recopied everything accumulated so far + # on every append, so a string full of escapes decoded in quadratic time. + mut parts: List[String] := [] mut i := 0 while i < raw.len(): c := raw[i] if c != chr(92): - out = out + c + parts.push(c) i = i + 1 continue if i + 1 >= raw.len(): - return (out, false) + return (parts.join(""), false) e := raw[i + 1] if e == chr(34) or e == chr(92) or e == "/": - out = out + e + parts.push(e) elif e == "b": - out = out + chr(8) + parts.push(chr(8)) elif e == "f": - out = out + chr(12) + parts.push(chr(12)) elif e == "n": - out = out + chr(10) + parts.push(chr(10)) elif e == "r": - out = out + chr(13) + parts.push(chr(13)) elif e == "t": - out = out + chr(9) + parts.push(chr(9)) elif e == "u": if i + 5 >= raw.len(): - return (out, false) + return (parts.join(""), false) mut code := 0 mut d := 0 while d < 4: digit := json_hex_digit_value(raw[i + 2 + d]) if digit < 0: - return (out, false) + return (parts.join(""), false) code = code * 16 + digit d = d + 1 i = i + 6 @@ -451,27 +454,27 @@ fn json_unescape_text(raw: String) -> (String, Bool): # the pair combines into one supplementary code point. if code >= 0xD800 and code <= 0xDBFF: if i + 5 >= raw.len() or raw[i] != chr(92) or raw[i + 1] != "u": - return (out, false) + return (parts.join(""), false) mut low := 0 d = 0 while d < 4: digit := json_hex_digit_value(raw[i + 2 + d]) if digit < 0: - return (out, false) + return (parts.join(""), false) low = low * 16 + digit d = d + 1 if low < 0xDC00 or low > 0xDFFF: - return (out, false) + return (parts.join(""), false) code = 0x10000 + (code - 0xD800) * 1024 + (low - 0xDC00) i = i + 6 elif code >= 0xDC00 and code <= 0xDFFF: - return (out, false) - out = out + json_utf8_encode(code) + return (parts.join(""), false) + parts.push(json_utf8_encode(code)) continue else: - return (out, false) + return (parts.join(""), false) i = i + 2 - return (out, true) + return (parts.join(""), true) fn json_byte_read_string(input: Bytes, pos: Int) -> JsonByteString: if pos < 0 or pos >= input.len() or input[pos] != 34: diff --git a/std/log.pith b/std/log.pith index ced96379..4106cad0 100644 --- a/std/log.pith +++ b/std/log.pith @@ -354,20 +354,23 @@ fn needs_console_quotes(value: String) -> Bool: return value.contains(" ") or value.contains(chr(9)) or value.contains(chr(10)) or value.contains(chr(34)) fn console_escape(value: String) -> String: - mut out := "" + # collect the pieces and join once at the end: appending each character + # to the result string recopied everything accumulated so far, which + # made escaping quadratic in the value. + mut parts: List[String] := [] mut i := 0 while i < value.len(): c := value[i] if c == chr(34): - out = out + chr(92) + chr(34) + parts.push(chr(92) + chr(34)) elif c == chr(92): - out = out + chr(92) + chr(92) + parts.push(chr(92) + chr(92)) elif c == chr(10): - out = out + chr(92) + "n" + parts.push(chr(92) + "n") else: - out = out + c + parts.push(c) i = i + 1 - return out + return parts.join("") fn render_console_field(field: Field) -> String: if field.kind == "raw": @@ -390,20 +393,23 @@ fn render_console_fields(fields: List[Field]) -> String: return " " + parts.join(" ") fn json_escape(text: String) -> String: - mut out := "" + # collect the pieces and join once at the end: appending each character + # to the result string recopied everything accumulated so far, which + # made escaping quadratic in the text. + mut parts: List[String] := [] mut i := 0 while i < text.len(): c := text[i] if c == chr(34): - out = out + chr(92) + chr(34) + parts.push(chr(92) + chr(34)) elif c == chr(92): - out = out + chr(92) + chr(92) + parts.push(chr(92) + chr(92)) elif c == chr(10): - out = out + chr(92) + "n" + parts.push(chr(92) + "n") else: - out = out + c + parts.push(c) i = i + 1 - return out + return parts.join("") fn json_string_field(name: String, value: String) -> String: return chr(34) + name + chr(34) + ":" + chr(34) + json_escape(value) + chr(34) diff --git a/std/metrics.pith b/std/metrics.pith index c3611df3..94f45743 100644 --- a/std/metrics.pith +++ b/std/metrics.pith @@ -80,18 +80,21 @@ fn unit_of(bare: String) -> String: # escape help text for a prometheus `# HELP` line: backslash and newline only # (unlike a label value, a double-quote needs no escaping here). fn escape_help(text: String) -> String: - mut out := "" + # collect the pieces and join once at the end: appending each character + # to the result string recopied everything accumulated so far, which + # made escaping quadratic in the text. + mut parts: List[String] := [] mut i := 0 while i < text.len(): c := text[i] if c == chr(92): - out = out + chr(92) + chr(92) + parts.push(chr(92) + chr(92)) elif c == chr(10): - out = out + chr(92) + "n" + parts.push(chr(92) + "n") else: - out = out + c + parts.push(c) i = i + 1 - return out + return parts.join("") # label names follow the same rule as metric names. fn normalize_label_name(name: String) -> String: @@ -99,20 +102,23 @@ fn normalize_label_name(name: String) -> String: # escape a label value for prometheus text: backslash, double-quote, newline. fn escape_label_value(value: String) -> String: - mut out := "" + # collect the pieces and join once at the end: appending each character + # to the result string recopied everything accumulated so far, which + # made escaping quadratic in the value. + mut parts: List[String] := [] mut i := 0 while i < value.len(): c := value[i] if c == chr(92): - out = out + chr(92) + chr(92) + parts.push(chr(92) + chr(92)) elif c == chr(34): - out = out + chr(92) + chr(34) + parts.push(chr(92) + chr(34)) elif c == chr(10): - out = out + chr(92) + "n" + parts.push(chr(92) + "n") else: - out = out + c + parts.push(c) i = i + 1 - return out + return parts.join("") # build the {k="v",...} suffix from [k1,v1,k2,v2,...] with keys sorted so a label # set has one canonical series key regardless of argument order. empty -> "". @@ -191,11 +197,15 @@ fn metric_name_char(c: String) -> String: fn normalize_metric_name(name: String) -> String: if name == "": return "metric" - mut out := "" + # collect the pieces and join once at the end: appending each character + # to the result string recopied everything accumulated so far, which + # made normalizing quadratic in the name. + mut parts: List[String] := [] mut i := 0 while i < name.len(): - out = out + metric_name_char(name[i]) + parts.push(metric_name_char(name[i])) i = i + 1 + out := parts.join("") if out == "": return "metric" if out[0] >= "0" and out[0] <= "9": diff --git a/std/net/grpc.pith b/std/net/grpc.pith index 1d01b9ad..1a79df2a 100644 --- a/std/net/grpc.pith +++ b/std/net/grpc.pith @@ -492,8 +492,12 @@ fn message_from(headers: List[hpack.HeaderField], trailers: List[hpack.HeaderFie # character in an error message writes a field value a compliant client must # reject, which turns a readable error into a broken response. fn percent_encode(text: String) -> String: - hex_digits := "0123456789ABCDEF" - mut out := "" + hex_digits := bytes.from_string_utf8("0123456789ABCDEF") + # appending to the result string one character at a time recopied the + # whole accumulated string on every append, which made encoding quadratic + # in the message; a byte buffer keeps each append constant. + out := bytes.buffer_with_capacity(text.len()) + defer out.free() mut i := 0 while i < text.len(): c := text[i] @@ -501,31 +505,41 @@ fn percent_encode(text: String) -> String: # it to the 0-255 the escape needs. code := bits.band(ord(c), 255) if code >= 32 and code <= 126 and c != "%": - out = out + c + out.write_byte(code) else: - out = out + "%" + hex_digits[code / 16] + hex_digits[code % 16] + out.write_byte(37) + out.write_byte(hex_digits[code / 16]) + out.write_byte(hex_digits[code % 16]) i = i + 1 - return out + # every byte written above is printable ascii or an ascii escape, so the + # decode cannot fail; an empty result would be a visible total failure + # rather than a silently wrong encoding. + return out.bytes().to_string_utf8() catch "" # the inverse. a malformed escape is left as written rather than dropped, so a # server that never encoded its message still reads back as itself. fn percent_decode(text: String) -> String: - mut out := "" + # collect the pieces and join once at the end: appending each character + # to the result string recopied everything accumulated so far, which made + # decoding quadratic in the message. a list survives a decoded byte that + # is not valid utf-8, which a byte-buffer round-trip through utf-8 + # validation would not. + mut parts: List[String] := [] mut i := 0 while i < text.len(): if text[i] == "%" and i + 2 < text.len(): high := percent_nibble(text[i + 1]) low := percent_nibble(text[i + 2]) if high >= 0 and low >= 0: - out = out + chr(high * 16 + low) + parts.push(chr(high * 16 + low)) i = i + 3 else: - out = out + text[i] + parts.push(text[i]) i = i + 1 else: - out = out + text[i] + parts.push(text[i]) i = i + 1 - return out + return parts.join("") fn percent_nibble(c: String) -> Int: if c >= "0" and c <= "9": diff --git a/std/net/url.pith b/std/net/url.pith index c31e6f59..d407a6b2 100644 --- a/std/net/url.pith +++ b/std/net/url.pith @@ -12,6 +12,8 @@ # path(u) # "/api" # encode("hello world") # "hello%20world" +import std.bytes as bytes + # =============================================================== # URL # =============================================================== @@ -217,21 +219,30 @@ fn is_unreserved(c: String) -> Bool: # Percent-encodes a string per RFC 3986. # Unreserved characters are left as-is, all others become %XX. pub fn encode(input: String) -> String: - hex_digits := "0123456789abcdef" - mut result := "" + hex_digits := bytes.from_string_utf8("0123456789abcdef") + # appending to the result string one character at a time recopied the whole + # accumulated string on every append, which made encoding quadratic in the + # input; a byte buffer keeps each append constant. + out := bytes.buffer_with_capacity(input.len()) + defer out.free() mut position := 0 while position < input.len(): character := input[position] if is_unreserved(character): - result = result + character + out.write_byte(ord(character)) if not is_unreserved(character): # a bare ord() sign-extends, so every byte above 0x7F came back # negative and indexed off the front of the digit table. percent # encoding is defined over bytes, so mask it back to 0-255. char_value := bit_and(ord(character), 255) - result = result + "%" + hex_digits[char_value / 16] + hex_digits[char_value % 16] + out.write_byte(37) + out.write_byte(hex_digits[char_value / 16]) + out.write_byte(hex_digits[char_value % 16]) position = position + 1 - return result + # every byte written above is unreserved ascii or an ascii escape, so the + # decode cannot fail; an empty result would be a visible total failure + # rather than a silently wrong encoding. + return out.bytes().to_string_utf8() catch "" # Converts a hex digit character to its numeric value. fn hex_digit(c: String) -> Int: @@ -246,7 +257,12 @@ fn hex_digit(c: String) -> Int: # Percent-decodes a string, converting %XX sequences to characters. # Also converts "+" to space (application/x-www-form-urlencoded). pub fn decode(input: String) -> String: - mut result := "" + # collect the pieces and join once at the end: appending to the result + # string directly recopied everything accumulated so far on every append, + # which made decoding quadratic in the input. a list survives a decoded + # byte that is not valid utf-8 (say "%ff"), which a byte-buffer round-trip + # through utf-8 validation would not. + mut parts: List[String] := [] mut position := 0 while position < input.len(): # these were three sequential ifs. after the escape branch consumed a @@ -257,18 +273,18 @@ pub fn decode(input: String) -> String: high_nibble := hex_digit(input[position + 1]) low_nibble := hex_digit(input[position + 2]) if high_nibble >= 0 and low_nibble >= 0: - result = result + chr(high_nibble * 16 + low_nibble) + parts.push(chr(high_nibble * 16 + low_nibble)) position = position + 3 else: - result = result + input[position] + parts.push(input[position]) position = position + 1 elif input[position] == "+": - result = result + " " + parts.push(" ") position = position + 1 else: - result = result + input[position] + parts.push(input[position]) position = position + 1 - return result + return parts.join("") test "percent coding survives non-ascii and a trailing escape": # a bare ord() sign-extends, so every byte above 0x7F indexed off the diff --git a/std/regex.pith b/std/regex.pith index c8b30f1f..406af4d4 100644 --- a/std/regex.pith +++ b/std/regex.pith @@ -503,12 +503,17 @@ impl Regex: ## replace every match with the replacement text. pub fn replace_all(input: String, replacement: String) -> String: - mut out := "" + # collect the pieces and join once at the end: appending each segment + # to the result string recopied everything accumulated so far, which + # made replacement quadratic in the number of matches. + mut parts: List[String] := [] mut last := 0 for m in self.find_all(input): - out = out + input.substring(last, m.start) + replacement + parts.push(input.substring(last, m.start)) + parts.push(replacement) last = m.stop - return out + input.substring(last, input.len()) + parts.push(input.substring(last, input.len())) + return parts.join("") test "zero-width patterns match the empty string": assert(compile("a*")!.is_match("")) diff --git a/std/strings.pith b/std/strings.pith index 59cd46b7..10d4034c 100644 --- a/std/strings.pith +++ b/std/strings.pith @@ -196,18 +196,21 @@ pub fn title(s: String) -> String: # swaps the case of each ascii letter. pub fn swap_case(s: String) -> String: - mut result := "" + # collect the pieces and join once at the end: appending each character + # to the result string recopied everything accumulated so far, which + # made swapping quadratic in the input. + mut parts: List[String] := [] mut i := 0 while i < s.len(): c := s[i] if is_upper(c): - result = result + c.to_lower() + parts.push(c.to_lower()) elif is_lower(c): - result = result + c.to_upper() + parts.push(c.to_upper()) else: - result = result + c + parts.push(c) i = i + 1 - return result + return parts.join("") # compares two strings for equality, ignoring ascii case. # @@ -467,12 +470,15 @@ pub fn trim_suffix(s: String, suffix: String) -> String: # --------------------------------------------------------------- fn repeat_text(text: String, times: Int) -> String: - mut result := "" + # collect the copies and join once at the end: appending one copy at a + # time recopied the whole accumulated string, which made a wide pad + # quadratic in its width. + mut parts: List[String] := [] mut i := 0 while i < times: - result = result + text + parts.push(text) i = i + 1 - return result + return parts.join("") # centers the string in a field of the given width. pub fn center(s: String, width: Int) -> String: @@ -495,12 +501,15 @@ pub fn zfill(s: String, width: Int) -> String: # instead would turn any multi-byte character into an invalid sequence. pub fn reverse(s: String) -> String: parts := text.chars(s) - mut result := "" + # collect the characters in reverse order and join once at the end: + # appending each one to the result string recopied everything accumulated + # so far, which made reversal quadratic in the input. + mut reversed: List[String] := [] mut i := parts.len() - 1 while i >= 0: - result = result + parts[i] + reversed.push(parts[i]) i = i - 1 - return result + return reversed.join("") # --------------------------------------------------------------- # Safe indexing and extraction diff --git a/std/template.pith b/std/template.pith index 855ae946..a08384ce 100644 --- a/std/template.pith +++ b/std/template.pith @@ -475,18 +475,29 @@ fn extend_values(values: List[Int], value: Int) -> List[Int]: return out fn render_range(t: Template, start: Int, stop: Int, root: Int, names: List[String], values: List[Int]) -> String!TemplateError: - mut out := "" + # collect the rendered pieces and join once at the end: appending each + # piece to the result string recopied everything rendered so far, which + # made a `<% for %>` loop quadratic in the number of rows. the running + # byte count keeps the output cap check without measuring the joined + # string every iteration. + mut parts: List[String] := [] + mut out_len := 0 mut index := start while index < stop: node := t.nodes[index] if node.kind == NODE_TEXT: - out = out + node.text + parts.push(node.text) + out_len = out_len + node.text.len() index = index + 1 elif node.kind == NODE_ESCAPED: - out = out + html.escape(stringify(resolve(root, names, values, node.path))) + piece := html.escape(stringify(resolve(root, names, values, node.path))) + parts.push(piece) + out_len = out_len + piece.len() index = index + 1 elif node.kind == NODE_RAW: - out = out + stringify(resolve(root, names, values, node.path)) + piece := stringify(resolve(root, names, values, node.path)) + parts.push(piece) + out_len = out_len + piece.len() index = index + 1 elif node.kind == NODE_IF: mut taken := truthy(resolve(root, names, values, node.path)) @@ -496,9 +507,13 @@ fn render_range(t: Template, start: Int, stop: Int, root: Int, names: List[Strin mut body_end := node.close if node.alt >= 0: body_end = node.alt - out = out + render_range(t, index + 1, body_end, root, names, values)! + piece := render_range(t, index + 1, body_end, root, names, values)! + parts.push(piece) + out_len = out_len + piece.len() elif node.alt >= 0: - out = out + render_range(t, node.alt + 1, node.close, root, names, values)! + piece := render_range(t, node.alt + 1, node.close, root, names, values)! + parts.push(piece) + out_len = out_len + piece.len() index = node.close + 1 elif node.kind == NODE_FOR: items := resolve(root, names, values, node.path) @@ -507,14 +522,16 @@ fn render_range(t: Template, start: Int, stop: Int, root: Int, names: List[Strin while item_index < count: inner_names := extend_names(names, node.name) inner_values := extend_values(values, json.array_get(items, item_index)) - out = out + render_range(t, index + 1, node.close, root, inner_names, inner_values)! + piece := render_range(t, index + 1, node.close, root, inner_names, inner_values)! + parts.push(piece) + out_len = out_len + piece.len() item_index = item_index + 1 index = node.close + 1 else: index = index + 1 - if out.len() > MAX_OUTPUT_BYTES: + if out_len > MAX_OUTPUT_BYTES: fail TemplateError("rendered output is larger than {MAX_OUTPUT_BYTES} bytes", node.offset) - return out + return parts.join("") # Renders `t` with `data`, escaping every `<%= %>` value. # diff --git a/std/term.pith b/std/term.pith index d960eabf..f06678de 100644 --- a/std/term.pith +++ b/std/term.pith @@ -71,7 +71,10 @@ pub fn italic(s: String) -> String: # Strip all ANSI escape codes from a string pub fn strip(s: String) -> String: esc := chr(27) - mut result := "" + # collect the kept characters and join once at the end: appending each + # one to the result string recopied everything accumulated so far, which + # made stripping quadratic in the input. + mut parts: List[String] := [] mut in_esc := false for c in s: if c == esc: @@ -79,8 +82,8 @@ pub fn strip(s: String) -> String: elif in_esc and c == "m": in_esc = false elif not in_esc: - result = result + c - return result + parts.push(c) + return parts.join("") test "term colors and styles wrap ansi codes": esc := chr(27) diff --git a/std/text.pith b/std/text.pith index c682780e..cfe12dd5 100644 --- a/std/text.pith +++ b/std/text.pith @@ -300,17 +300,23 @@ pub fn char_offsets(s: String) -> List[Int]: # join characters back into one string. pub fn from_chars(parts: List[String]) -> String: - mut out := "" + # join in one pass: appending each character to the result string + # recopied everything accumulated so far, which made rebuilding a string + # from its characters quadratic in the input. + mut collected: List[String] := [] for part in parts: - out = out + part - return out + collected.push(part) + return collected.join("") # build a string from code points. fails on any value `encode` rejects. pub fn from_code_points(points: List[Int]) -> String!: - mut out := "" + # collect the encoded characters and join once at the end: appending each + # one to the result string recopied everything accumulated so far, which + # made building from code points quadratic in the input. + mut parts: List[String] := [] for point in points: - out = out + encode(point)! - return out + parts.push(encode(point)!) + return parts.join("") # --------------------------------------------------------------------------- # character-indexed access @@ -395,16 +401,19 @@ pub fn truncate_with(s: String, max_chars: Int, suffix: String) -> String: pub fn sanitize(s: String) -> String: if is_valid(s): return s - mut out := "" + # collect the pieces and join once at the end: appending each character + # to the result string recopied everything accumulated so far, which + # made repair quadratic in the input. + mut parts: List[String] := [] mut position := 0 while position < s.len(): step := decode_step(s, position) if step.code_point < 0: - out = out + replacement_char() + parts.push(replacement_char()) else: - out = out + s.substring(position, position + step.size) + parts.push(s.substring(position, position + step.size)) position = position + step.size - return out + return parts.join("") # --------------------------------------------------------------------------- # case folding @@ -450,20 +459,23 @@ pub fn fold_code_point(code_point: Int) -> Int: # folding would close that gap, at the cost of a mapping that changes length; # see the module header. pub fn fold(s: String) -> String: - mut out := "" + # collect the folded characters and join once at the end: appending each + # one to the result string recopied everything accumulated so far, which + # made folding quadratic in the input. + mut parts: List[String] := [] mut position := 0 while position < s.len(): step := decode_step(s, position) if step.code_point < 0: - out = out + replacement_char() + parts.push(replacement_char()) else: folded := fold_code_point(step.code_point) if folded == step.code_point: - out = out + s.substring(position, position + step.size) + parts.push(s.substring(position, position + step.size)) else: - out = out + encode(folded).unwrap_or(replacement_char()) + parts.push(encode(folded).unwrap_or(replacement_char())) position = position + step.size - return out + return parts.join("") # compare two strings ignoring case, the whole way up unicode rather than just # through ascii. diff --git a/std/yaml.pith b/std/yaml.pith index 8eb0c9f6..0cbf2e53 100644 --- a/std/yaml.pith +++ b/std/yaml.pith @@ -420,12 +420,15 @@ fn utf8_encode(code: Int) -> String: # decodes the body of a double-quoted scalar (the text between the quotes). fn decode_double_quoted(body: String) -> String: - mut out := "" + # collect the pieces and join once at the end: appending each character + # to the result string recopied everything accumulated so far, so a + # scalar full of escapes decoded in quadratic time. + mut parts: List[String] := [] mut i := 0 while i < body.len(): c := body[i] if c != chr(92): - out = out + c + parts.push(c) i = i + 1 continue if i + 1 >= body.len(): @@ -433,25 +436,25 @@ fn decode_double_quoted(body: String) -> String: return "" e := body[i + 1] if e == "n": - out = out + chr(10) + parts.push(chr(10)) elif e == "t": - out = out + chr(9) + parts.push(chr(9)) elif e == "r": - out = out + chr(13) + parts.push(chr(13)) elif e == "0": - out = out + chr(0) + parts.push(chr(0)) elif e == "a": - out = out + chr(7) + parts.push(chr(7)) elif e == "b": - out = out + chr(8) + parts.push(chr(8)) elif e == "f": - out = out + chr(12) + parts.push(chr(12)) elif e == "v": - out = out + chr(11) + parts.push(chr(11)) elif e == "e": - out = out + chr(27) + parts.push(chr(27)) elif e == chr(34) or e == chr(92) or e == "/" or e == " ": - out = out + e + parts.push(e) elif e == "u": if i + 5 >= body.len(): yaml_fail("truncated \\u escape in a double-quoted scalar") @@ -465,29 +468,32 @@ fn decode_double_quoted(body: String) -> String: return "" code = code * 16 + digit d = d + 1 - out = out + utf8_encode(code) + parts.push(utf8_encode(code)) i = i + 6 continue else: yaml_fail("unknown escape \\" + e + " in a double-quoted scalar") return "" i = i + 2 - return out + return parts.join("") # decodes the body of a single-quoted scalar. the only escape is a doubled # quote, and backslashes are literal. fn decode_single_quoted(body: String) -> String: - mut out := "" + # collect the pieces and join once at the end: appending each character + # to the result string recopied everything accumulated so far, which + # made decoding quadratic in the scalar. + mut parts: List[String] := [] mut i := 0 while i < body.len(): c := body[i] if c == "'" and i + 1 < body.len() and body[i + 1] == "'": - out = out + "'" + parts.push("'") i = i + 2 continue - out = out + c + parts.push(c) i = i + 1 - return out + return parts.join("") # the text of a quoted scalar, with its quotes removed and escapes applied. fn decode_quoted(text: String) -> String: @@ -874,12 +880,15 @@ fn parse_inline_value(text: String) -> Int: # =============================================================== fn repeat_newlines(count: Int) -> String: - mut out := "" + # collect the pieces and join once at the end: appending one newline at a + # time recopied the whole accumulated string, so a long run of blank + # lines in a block scalar cost quadratic time. + mut parts: List[String] := [] mut i := 0 while i < count: - out = out + chr(10) + parts.push(chr(10)) i = i + 1 - return out + return parts.join("") # reads a `|` or `>` block whose header is `header` and whose owning key sits # at `parent_indent`. diff --git a/tests/cases/test_string_builder_linearity.pith b/tests/cases/test_string_builder_linearity.pith new file mode 100644 index 00000000..d57f094c --- /dev/null +++ b/tests/cases/test_string_builder_linearity.pith @@ -0,0 +1,56 @@ +# the string builders that used to accumulate with `s = s + piece` in a loop: +# url percent coding, json string unescaping, template rendering, and html +# escaping. each value here is pinned against an external oracle (python's +# urllib/json/binascii and the rfc test vectors), so a rewrite of the +# accumulation strategy that changes a single output byte fails this test. + +import std.net.url as url +import std.json as json +import std.encoding as encoding +import std.bytes as bytes +import std.template as template +import std.html as html + +fn main() -> Int!: + # rfc 3986 percent coding, non-ascii included; matches urllib.parse + print("urlenc " + url.encode("hello world/Ärger?a=b&c=+~. 日本")) + print("urldec " + url.decode("hello%20world%2f%C3%84rger+%2B%zz%")) + print("urlround " + url.decode(url.encode("Ärger 日本 a+b%c"))) + + # json string escapes through parse, surrogate pair included; matches + # python json.loads + parsed := json.parse("[\"a\\n\\t\\u00e9\\ud83d\\ude00\\\"\\\\\\/ z\\u0041\"]") + print("jsonun " + json.get_string(json.array_get(parsed, 0))) + + # hex over every byte value; matches binascii.hexlify + all_bytes := bytes.buffer_with_capacity(256) + defer all_bytes.free() + mut i := 0 + while i < 256: + all_bytes.write_byte(i)! + i = i + 1 + hex_all := encoding.to_hex(all_bytes.bytes()) + print("hexlen {hex_all.len()}") + print("hexhead " + hex_all.substring(0, 32)) + print("hextail " + hex_all.substring(hex_all.len() - 16, hex_all.len())) + + # rfc 4648 base32 test vectors + print("b32 " + encoding.base32_encode(bytes.from_string_utf8("foobar")) + " " + encoding.base32_encode(bytes.from_string_utf8("f"))) + print("b32hex " + encoding.base32_hex_encode(bytes.from_string_utf8("foobar"))) + + # base58 bitcoin alphabet + print("b58 " + encoding.base58_encode(bytes.from_string_utf8("hello"))) + + # html escape rewrites all five characters and passes non-ascii through + print("html " + html.escape("&'日本'")) + + # a template for loop escapes each row + items := template.list() + items.push_text("") + items.push_text("a&b") + items.push_text("ü") + data := template.context() + data.set_child("items", items) + page := template.render_string("", data) catch "TEMPLATE_ERR" + print("template " + page) + return 0 diff --git a/tests/expected/test_string_builder_linearity.txt b/tests/expected/test_string_builder_linearity.txt new file mode 100644 index 00000000..15f62352 --- /dev/null +++ b/tests/expected/test_string_builder_linearity.txt @@ -0,0 +1,13 @@ +urlenc hello%20world%2f%c3%84rger%3fa%3db%26c%3d%2b~.%20%e6%97%a5%e6%9c%ac +urldec hello world/Ärger +%zz% +urlround Ärger 日本 a+b%c +jsonun a + é😀"\/ zA +hexlen 512 +hexhead 000102030405060708090a0b0c0d0e0f +hextail f8f9fafbfcfdfeff +b32 MZXW6YTBOI====== MY====== +b32hex CPNMUOJ1E8====== +b58 Cn8eVZg +html <a href="x">&'日本'</a> +template