From 08bcf57a4ab7f2708c84034a9fab1d8ced268f0e Mon Sep 17 00:00:00 2001 From: Nick Nassiri Date: Sun, 21 Jun 2026 23:06:19 -0700 Subject: [PATCH] bench(#856): add 6 demonstrative cross-runtime workloads Broadens the cross-runtime suite beyond the original arithmetic kernels with workloads where SharpTS-compiled, Node (V8), and Bun (JSC) diverge most: json, regex, sort, typed-arrays, binary-trees, brainfuck - json/typed-arrays/binary-trees are number->number, added to the shared benchmarks/scripts/lib/algorithms.ts (also embedded by SharpTS.Benchmarks), with matching Idiomatic/Equivalent C# baselines and a new BenchmarkDotNet StarterWorkloadBenchmarks.cs (SharpTS-compiled vs native vs boxed C#). - regex/sort/brainfuck are self-contained scripts (seeded Park-Miller LCG for deterministic, cross-runtime-identical data). All six produce byte-identical results across interpreter, compiled, Node, and Bun (equal work), so any gap is pure codegen. They immediately surfaced two compiled-output gaps now tracked as #877 (Array.sort is O(n^2) insertion sort) and #878 (Float64Array element access is boxed + virtually dispatched). --- .../Baselines/EquivalentCSharp.cs | 80 ++++++++++++++++++ .../Baselines/IdiomaticCSharp.cs | 61 ++++++++++++++ .../Benchmarks/StarterWorkloadBenchmarks.cs | 84 +++++++++++++++++++ benchmarks/scripts/binary-trees.ts | 9 ++ benchmarks/scripts/brainfuck.ts | 78 +++++++++++++++++ benchmarks/scripts/json.ts | 8 ++ benchmarks/scripts/lib/algorithms.ts | 60 +++++++++++++ benchmarks/scripts/regex.ts | 29 +++++++ benchmarks/scripts/sort.ts | 49 +++++++++++ benchmarks/scripts/typed-arrays.ts | 8 ++ 10 files changed, 466 insertions(+) create mode 100644 SharpTS.Benchmarks/Benchmarks/StarterWorkloadBenchmarks.cs create mode 100644 benchmarks/scripts/binary-trees.ts create mode 100644 benchmarks/scripts/brainfuck.ts create mode 100644 benchmarks/scripts/json.ts create mode 100644 benchmarks/scripts/regex.ts create mode 100644 benchmarks/scripts/sort.ts create mode 100644 benchmarks/scripts/typed-arrays.ts diff --git a/SharpTS.Benchmarks/Baselines/EquivalentCSharp.cs b/SharpTS.Benchmarks/Baselines/EquivalentCSharp.cs index 5b906d8a..c5c36e0d 100644 --- a/SharpTS.Benchmarks/Baselines/EquivalentCSharp.cs +++ b/SharpTS.Benchmarks/Baselines/EquivalentCSharp.cs @@ -72,4 +72,84 @@ public static class EquivalentCSharp } return count; } + + /// + /// JSON round-trip using Dictionary<string, object?> nodes - mirrors the + /// boxed object/array representation the SharpTS runtime produces. + /// + public static object? JsonRoundTrip(object? n) + { + double nVal = Convert.ToDouble(n); + int nInt = (int)nVal; + var items = new List(nInt); + for (int i = 0; i < nInt; i++) + { + items.Add(new Dictionary + { + { "id", (double)i }, + { "name", "item-" + i }, + { "value", (double)(i * 3 - 1) }, + }); + } + var root = new Dictionary { { "items", items } }; + string json = System.Text.Json.JsonSerializer.Serialize(root); + using var doc = System.Text.Json.JsonDocument.Parse(json); + double sum = 0; + foreach (var el in doc.RootElement.GetProperty("items").EnumerateArray()) + { + sum += el.GetProperty("value").GetDouble(); + } + return sum; + } + + /// + /// Typed-array kernel using List<object?> with boxed doubles - the dynamic + /// representation tax SharpTS AVOIDS by compiling Float64Array to a real buffer. + /// + public static object? TypedArrayKernel(object? n) + { + double nVal = Convert.ToDouble(n); + int nInt = (int)nVal; + var a = new List(nInt); + for (int i = 0; i < nInt; i++) + { + a.Add(i * 1.5 + (i % 7)); + } + double sum = 0; + for (int i = 1; i < nInt - 1; i++) + { + sum += Convert.ToDouble(a[i - 1]) - 2 * Convert.ToDouble(a[i]) + Convert.ToDouble(a[i + 1]); + } + return sum; + } + + /// + /// binary-trees using Dictionary<string, object?> nodes with dynamic + /// property lookups - mirrors a boxed SharpTS object graph. + /// + public static object? BinaryTrees(object? depth) + { + double d = Convert.ToDouble(depth); + return (double)ItemCheckBoxed(BuildTreeBoxed((int)d)); + } + + private static Dictionary BuildTreeBoxed(int depth) + { + if (depth <= 0) + { + return new Dictionary { { "left", null }, { "right", null } }; + } + return new Dictionary + { + { "left", BuildTreeBoxed(depth - 1) }, + { "right", BuildTreeBoxed(depth - 1) }, + }; + } + + private static int ItemCheckBoxed(Dictionary? node) + { + if (node is null) return 1; + return 1 + ItemCheckBoxed(node["left"] as Dictionary) + + ItemCheckBoxed(node["right"] as Dictionary); + } } diff --git a/SharpTS.Benchmarks/Baselines/IdiomaticCSharp.cs b/SharpTS.Benchmarks/Baselines/IdiomaticCSharp.cs index b90e0b9a..e9e25990 100644 --- a/SharpTS.Benchmarks/Baselines/IdiomaticCSharp.cs +++ b/SharpTS.Benchmarks/Baselines/IdiomaticCSharp.cs @@ -59,4 +59,65 @@ public static int CountPrimes(int n) } return count; } + + /// + /// JSON round-trip - typed serialize, parse via JsonDocument, sum a field. + /// + public static int JsonRoundTrip(int n) + { + var items = new List(n); + for (int i = 0; i < n; i++) + { + items.Add(new { id = i, name = "item-" + i, value = i * 3 - 1 }); + } + string json = System.Text.Json.JsonSerializer.Serialize(new { items }); + using var doc = System.Text.Json.JsonDocument.Parse(json); + int sum = 0; + foreach (var el in doc.RootElement.GetProperty("items").EnumerateArray()) + { + sum += el.GetProperty("value").GetInt32(); + } + return sum; + } + + /// + /// Typed-array numeric kernel - native double[] fill + 3-point stencil sweep. + /// + public static double TypedArrayKernel(int n) + { + var a = new double[n]; + for (int i = 0; i < n; i++) + { + a[i] = i * 1.5 + (i % 7); + } + double sum = 0; + for (int i = 1; i < n - 1; i++) + { + sum += a[i - 1] - 2 * a[i] + a[i + 1]; + } + return sum; + } + + /// + /// binary-trees (CLBG) - build a node tree to `depth`, then checksum it. + /// + public static int BinaryTrees(int depth) => ItemCheck(BuildTree(depth)); + + private sealed class TreeNode + { + public TreeNode? Left; + public TreeNode? Right; + } + + private static TreeNode BuildTree(int depth) + { + if (depth <= 0) return new TreeNode(); + return new TreeNode { Left = BuildTree(depth - 1), Right = BuildTree(depth - 1) }; + } + + private static int ItemCheck(TreeNode? node) + { + if (node is null) return 1; + return 1 + ItemCheck(node.Left) + ItemCheck(node.Right); + } } diff --git a/SharpTS.Benchmarks/Benchmarks/StarterWorkloadBenchmarks.cs b/SharpTS.Benchmarks/Benchmarks/StarterWorkloadBenchmarks.cs new file mode 100644 index 00000000..55c07dba --- /dev/null +++ b/SharpTS.Benchmarks/Benchmarks/StarterWorkloadBenchmarks.cs @@ -0,0 +1,84 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Order; +using SharpTS.Benchmarks.Baselines; + +namespace SharpTS.Benchmarks.Benchmarks; + +// Starter-set workloads that broaden coverage beyond the original arithmetic +// kernels: a builtin-heavy JSON round-trip, a data-parallel typed-array kernel, +// and allocation/GC-heavy binary-trees. The TypeScript bodies live in +// benchmarks/scripts/lib/algorithms.ts (shared byte-identical with the +// cross-runtime shell harness) and are reached through a cached +// Func delegate via ComputationalBenchmarkBase.LoadCompiled. +// +// As elsewhere: SharpTS-compiled vs idiomatic C# (native types — the ceiling) +// vs "equivalent" C# (object?/boxing — the dynamic-typing tax). + +[MemoryDiagnoser] +[RankColumn] +[Orderer(SummaryOrderPolicy.FastestToSlowest)] +public class JsonRoundTripBenchmarks : ComputationalBenchmarkBase +{ + private Func _jsonRoundTrip = null!; + + [Params(100, 1000, 10000)] + public int N { get; set; } + + [GlobalSetup] + public void Setup() => _jsonRoundTrip = LoadCompiled("jsonRoundTrip"); + + [Benchmark] + public double SharpTS() => _jsonRoundTrip(N); + + [Benchmark] + public int Idiomatic() => IdiomaticCSharp.JsonRoundTrip(N); + + [Benchmark] + public object? Equivalent() => EquivalentCSharp.JsonRoundTrip((double)N); +} + +[MemoryDiagnoser] +[RankColumn] +[Orderer(SummaryOrderPolicy.FastestToSlowest)] +public class TypedArrayBenchmarks : ComputationalBenchmarkBase +{ + private Func _typedArrayKernel = null!; + + [Params(1000, 100000, 1000000)] + public int N { get; set; } + + [GlobalSetup] + public void Setup() => _typedArrayKernel = LoadCompiled("typedArrayKernel"); + + [Benchmark] + public double SharpTS() => _typedArrayKernel(N); + + [Benchmark] + public double Idiomatic() => IdiomaticCSharp.TypedArrayKernel(N); + + [Benchmark] + public object? Equivalent() => EquivalentCSharp.TypedArrayKernel((double)N); +} + +[MemoryDiagnoser] +[RankColumn] +[Orderer(SummaryOrderPolicy.FastestToSlowest)] +public class BinaryTreesBenchmarks : ComputationalBenchmarkBase +{ + private Func _binaryTrees = null!; + + [Params(8, 12, 16)] + public int N { get; set; } + + [GlobalSetup] + public void Setup() => _binaryTrees = LoadCompiled("binaryTrees"); + + [Benchmark] + public double SharpTS() => _binaryTrees(N); + + [Benchmark] + public int Idiomatic() => IdiomaticCSharp.BinaryTrees(N); + + [Benchmark] + public object? Equivalent() => EquivalentCSharp.BinaryTrees((double)N); +} diff --git a/benchmarks/scripts/binary-trees.ts b/benchmarks/scripts/binary-trees.ts new file mode 100644 index 00000000..e11a21d3 --- /dev/null +++ b/benchmarks/scripts/binary-trees.ts @@ -0,0 +1,9 @@ +import { binaryTrees } from "./lib/algorithms.ts"; +import { bench } from "./lib/bench.ts"; + +// Param is tree depth: node count is 2^(depth+1) - 1, so cost grows fast. +const params: number[] = [8, 12, 16]; +for (let p: number = 0; p < params.length; p++) { + const n: number = params[p]; + bench("binary-trees", n, () => binaryTrees(n)); +} diff --git a/benchmarks/scripts/brainfuck.ts b/benchmarks/scripts/brainfuck.ts new file mode 100644 index 00000000..6525d473 --- /dev/null +++ b/benchmarks/scripts/brainfuck.ts @@ -0,0 +1,78 @@ +import { bench } from "./lib/bench.ts"; + +// A small Brainfuck interpreter — a "macro" workload dominated by interpreter +// dispatch: a tight loop over the program string (charCodeAt), precomputed +// bracket jumps, and tape reads/writes on a Uint8Array. The program is a +// pointer-neutral snippet concatenated `n` times, so larger n = more dispatch +// (linear) with a deterministic final tape across SharpTS, Node, and Bun. + +// "+++++[>+<-]>[<+>-]<": adds 5 to cell0 (mod 256), pointer returns to cell0. +const SNIPPET: string = "+++++[>+<-]>[<+>-]<"; + +function buildProgram(reps: number): string { + let s: string = ""; + for (let i: number = 0; i < reps; i++) { + s = s + SNIPPET; + } + return s; +} + +// Precompute matching-bracket targets so '[' / ']' are O(1) in the hot loop. +function buildJumps(program: string): number[] { + const len: number = program.length; + const jumps: number[] = []; + for (let i: number = 0; i < len; i++) { + jumps.push(0); + } + const stack: number[] = []; + for (let i: number = 0; i < len; i++) { + const c: number = program.charCodeAt(i); + if (c === 91) { // '[' + stack.push(i); + } else if (c === 93) { // ']' + const open: number = stack[stack.length - 1]; + stack.pop(); + jumps[open] = i; + jumps[i] = open; + } + } + return jumps; +} + +function runBF(program: string, jumps: number[]): number { + const TAPE: number = 4096; + const tape: Uint8Array = new Uint8Array(TAPE); + let ptr: number = 0; + let ip: number = 0; + const len: number = program.length; + while (ip < len) { + const c: number = program.charCodeAt(ip); + if (c === 43) { // '+' + tape[ptr] = (tape[ptr] + 1) & 255; + } else if (c === 45) { // '-' + tape[ptr] = (tape[ptr] - 1) & 255; + } else if (c === 62) { // '>' + ptr = ptr + 1; + } else if (c === 60) { // '<' + ptr = ptr - 1; + } else if (c === 91) { // '[' + if (tape[ptr] === 0) ip = jumps[ip]; + } else if (c === 93) { // ']' + if (tape[ptr] !== 0) ip = jumps[ip]; + } + ip = ip + 1; + } + let sum: number = 0; + for (let i: number = 0; i < TAPE; i++) { + sum = sum + tape[i]; + } + return sum; +} + +const params: number[] = [50, 500, 5000]; +for (let p: number = 0; p < params.length; p++) { + const n: number = params[p]; + const program: string = buildProgram(n); + const jumps: number[] = buildJumps(program); + bench("brainfuck", n, () => runBF(program, jumps)); +} diff --git a/benchmarks/scripts/json.ts b/benchmarks/scripts/json.ts new file mode 100644 index 00000000..361fb6c0 --- /dev/null +++ b/benchmarks/scripts/json.ts @@ -0,0 +1,8 @@ +import { jsonRoundTrip } from "./lib/algorithms.ts"; +import { bench } from "./lib/bench.ts"; + +const params: number[] = [100, 1000, 10000]; +for (let p: number = 0; p < params.length; p++) { + const n: number = params[p]; + bench("json", n, () => jsonRoundTrip(n)); +} diff --git a/benchmarks/scripts/lib/algorithms.ts b/benchmarks/scripts/lib/algorithms.ts index 451f0c8e..4210f43e 100644 --- a/benchmarks/scripts/lib/algorithms.ts +++ b/benchmarks/scripts/lib/algorithms.ts @@ -98,3 +98,63 @@ export function arrayMethodWork(n: number): number { const evens = doubled.filter((x: number): boolean => x % 4 === 0); return evens.reduce((acc: number, x: number): number => acc + x, 0); } + +// ── Builtin-heavy / allocation / data-parallel workloads ────────────────── +// These diverge most across SharpTS-compiled, V8 (Node), and JSC (Bun): JSON +// and typed-array kernels lean on hand-tuned engine builtins, binary-trees on +// the GC. Each still funnels to a `number` so the BDN harness can reflect it. + +// JSON round-trip: build a record array, stringify, parse it back, sum a field. +// The single most common server hot path; V8/JSC JSON are hand-tuned C++. +export function jsonRoundTrip(n: number): number { + const items: { id: number; name: string; value: number }[] = []; + for (let i: number = 0; i < n; i++) { + items.push({ id: i, name: "item-" + i, value: i * 3 - 1 }); + } + const json: string = JSON.stringify({ items: items }); + const parsed = JSON.parse(json); + const back: { id: number; name: string; value: number }[] = parsed.items; + let sum: number = 0; + for (let i: number = 0; i < back.length; i++) { + sum = sum + back[i].value; + } + return sum; +} + +// Typed-array numeric kernel: fill a Float64Array, then a 3-point stencil sweep. +// Data-parallel arithmetic over a real typed buffer — where compiled IL should +// approach native and the dynamic/boxed representation pays the most. +export function typedArrayKernel(n: number): number { + const a: Float64Array = new Float64Array(n); + for (let i: number = 0; i < n; i++) { + a[i] = i * 1.5 + (i % 7); + } + let sum: number = 0; + for (let i: number = 1; i < n - 1; i++) { + sum = sum + (a[i - 1] - 2 * a[i] + a[i + 1]); + } + return sum; +} + +// binary-trees (Computer Language Benchmarks Game): build a `{ left, right }` +// object tree to `depth`, then checksum it. Allocates and discards heavily — +// exercises the GC and recursion rather than arithmetic. +type TreeNode = { left: TreeNode | null; right: TreeNode | null }; + +function buildTree(depth: number): TreeNode { + if (depth <= 0) { + return { left: null, right: null }; + } + return { left: buildTree(depth - 1), right: buildTree(depth - 1) }; +} + +function itemCheck(node: TreeNode | null): number { + if (node === null) { + return 1; + } + return 1 + itemCheck(node.left) + itemCheck(node.right); +} + +export function binaryTrees(depth: number): number { + return itemCheck(buildTree(depth)); +} diff --git a/benchmarks/scripts/regex.ts b/benchmarks/scripts/regex.ts new file mode 100644 index 00000000..7560fd5c --- /dev/null +++ b/benchmarks/scripts/regex.ts @@ -0,0 +1,29 @@ +import { bench } from "./lib/bench.ts"; + +// Regex throughput: V8 (Irregexp), JSC, and .NET's Regex diverge sharply here. +// The corpus is deterministic and built once per param (outside the timed fn) +// so we measure matching/replacing, not string construction. The timed fn +// funnels to a numeric checksum for the anti-DCE guard. + +function buildCorpus(n: number): string { + let s: string = ""; + for (let i: number = 0; i < n; i++) { + s = s + "user" + i + "@host" + (i % 97) + ".com paid 1" + i + " on day " + (i % 365) + "; "; + } + return s; +} + +function regexWork(corpus: string): number { + const emails = corpus.match(/\w+@\w+\.\w+/g); + const emailCount: number = emails === null ? 0 : emails.length; + const masked: string = corpus.replace(/\d+/g, "#"); + const hasDay: boolean = /day \d+/.test(corpus); + return emailCount + masked.length + (hasDay ? 1 : 0); +} + +const params: number[] = [100, 1000, 10000]; +for (let p: number = 0; p < params.length; p++) { + const n: number = params[p]; + const corpus: string = buildCorpus(n); + bench("regex", n, () => regexWork(corpus)); +} diff --git a/benchmarks/scripts/sort.ts b/benchmarks/scripts/sort.ts new file mode 100644 index 00000000..10df1760 --- /dev/null +++ b/benchmarks/scripts/sort.ts @@ -0,0 +1,49 @@ +import { bench } from "./lib/bench.ts"; + +// Array.prototype.sort with a comparator: exercises comparator-call overhead + +// the engine's sort (V8/JSC TimSort vs .NET introsort). Data is generated with +// a seeded Park-Miller LCG (products stay < 2^53, exact in IEEE doubles) so +// SharpTS, Node, and Bun all sort byte-identical inputs. + +function makeNumbers(n: number): number[] { + const out: number[] = []; + let state: number = 123456789; + for (let i: number = 0; i < n; i++) { + state = (state * 48271) % 2147483647; + out.push(state); + } + return out; +} + +function makeRecords(n: number): { key: number; tag: string }[] { + const out: { key: number; tag: string }[] = []; + let state: number = 987654321; + for (let i: number = 0; i < n; i++) { + state = (state * 48271) % 2147483647; + out.push({ key: state, tag: "t" + (state % 1000) }); + } + return out; +} + +// Copy a FRESH (unsorted) array each call — sorting an already-sorted array +// collapses to O(n) and would misrepresent the cost. The slice() is the one +// intentional per-call allocation. +function sortNumbers(src: number[]): number { + const c: number[] = src.slice(); + c.sort((a: number, b: number): number => a - b); + return c[0] + c[c.length - 1]; +} + +function sortRecords(src: { key: number; tag: string }[]): number { + const c = src.slice(); + c.sort((a: { key: number; tag: string }, b: { key: number; tag: string }): number => a.key - b.key); + return c[0].key; +} + +const params: number[] = [100, 1000, 10000]; +for (let p: number = 0; p < params.length; p++) { + const n: number = params[p]; + const nums: number[] = makeNumbers(n); + const recs = makeRecords(n); + bench("sort", n, () => sortNumbers(nums) + sortRecords(recs)); +} diff --git a/benchmarks/scripts/typed-arrays.ts b/benchmarks/scripts/typed-arrays.ts new file mode 100644 index 00000000..22c3178c --- /dev/null +++ b/benchmarks/scripts/typed-arrays.ts @@ -0,0 +1,8 @@ +import { typedArrayKernel } from "./lib/algorithms.ts"; +import { bench } from "./lib/bench.ts"; + +const params: number[] = [1000, 100000, 1000000]; +for (let p: number = 0; p < params.length; p++) { + const n: number = params[p]; + bench("typed-arrays", n, () => typedArrayKernel(n)); +}