From e79b1cd4381d887c449a49b55d89250174aa18ce Mon Sep 17 00:00:00 2001 From: Amayyas Date: Tue, 21 Jul 2026 18:57:27 +0200 Subject: [PATCH 1/2] refactor(distribution): extract the "Top X%" maths into lib, and test it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The distribution panel's standing calculation lived inline in the component, so the one number the app asserts publicly about someone — "TOP 0.01% of GitHub" — was the only piece of statistics in the codebase with no tests. lib/ is where the pure, framework-agnostic logic lives, so it moves there. Behaviour is unchanged: the extracted module was diffed against the old inline version across all 99 ratings on both shipped samples (all accounts and active-only) and the labels are identical, and the rendered panel still reads TOP 0.01% / TOP 0.05% for torvalds in the browser. Tests cover the parts that were easy to get wrong and impossible to notice: the at-or-above count is inclusive of the card's own bucket, a rating below the histogram floor takes the whole sample, and the rule-of-three fallback (nothing observed at or above → the honest claim is "< 3/n", not "top 0%") is marked and labelled with a "<". Plus a monotonicity check over the real histogram and an empty-sample guard, so n = 0 can't produce an Infinity. --- components/DistributionPanel.tsx | 16 ++--- lib/distribution.ts | 48 ++++++++++++++ tests/distribution.test.ts | 106 +++++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+), 11 deletions(-) create mode 100644 lib/distribution.ts create mode 100644 tests/distribution.test.ts diff --git a/components/DistributionPanel.tsx b/components/DistributionPanel.tsx index 67ecce9..a78062b 100644 --- a/components/DistributionPanel.tsx +++ b/components/DistributionPanel.tsx @@ -1,6 +1,7 @@ "use client"; import { DIST_ACTIVE_COUNTS, DIST_ACTIVE_N, DIST_COUNTS, DIST_MIN, DIST_N } from "@/lib/distribution-data"; +import { standing } from "@/lib/distribution"; import { resolveResultTheme } from "./finishTheme"; import { Tip } from "./ScoutReport"; import type { Card } from "@/lib/scoring/types"; @@ -24,17 +25,10 @@ export default function DistributionPanel({ card }: { card: Card }) { const meX = xFor(card.overall + 0.5); - // "Top X%" = share of the population rated at least this card's overall. When - // nothing in the sample reaches it, the honest claim is the rule-of-three - // bound: at 95% confidence the true share is below 3/n. - const fmtPct = (p: number) => (p >= 10 ? `${Math.round(p)}` : p >= 1 ? p.toFixed(1) : p.toFixed(2)); - const top = (counts: number[], n: number) => { - const atOrAbove = counts.reduce((s, c, i) => s + (DIST_MIN + i >= card.overall ? c : 0), 0); - const pct = atOrAbove > 0 ? (100 * atOrAbove) / n : (100 * 3) / n; - return { atOrAbove, label: `Top ${atOrAbove > 0 ? "" : "< "}${fmtPct(pct)}%` }; - }; - const all = top(DIST_COUNTS, DIST_N); - const act = top(DIST_ACTIVE_COUNTS, DIST_ACTIVE_N); + // "Top X%" = share of the population rated at least this card's overall (see + // lib/distribution for the rule-of-three bound when nothing reaches it). + const all = standing(DIST_COUNTS, DIST_N, card.overall); + const act = standing(DIST_ACTIVE_COUNTS, DIST_ACTIVE_N, card.overall); const tipText = `Higher than ${(DIST_N - all.atOrAbove).toLocaleString()} of ${DIST_N.toLocaleString()} randomly sampled GitHub users, ` + `and ${(DIST_ACTIVE_N - act.atOrAbove).toLocaleString()} of the ${DIST_ACTIVE_N.toLocaleString()} who were active in the past year.`; diff --git a/lib/distribution.ts b/lib/distribution.ts new file mode 100644 index 0000000..a446fa2 --- /dev/null +++ b/lib/distribution.ts @@ -0,0 +1,48 @@ +import { DIST_MIN } from "./distribution-data"; + +// Where a card stands in the sampled rating histogram — the "TOP 0.02% of GitHub" +// claim under the scouting metrics. Pure and framework-agnostic so the statistic +// can be tested on its own: it's a number we assert publicly about someone, and +// the histogram is a snapshot that gets regenerated (scripts/distribution-runner), +// so the maths has to hold for a sample of any size. + +export interface Standing { + /** Sampled accounts rated at or above this card. */ + atOrAbove: number; + /** Share of the sample at or above, in percent. An upper bound when `bounded`. */ + pct: number; + /** True when NOTHING in the sample reached this rating, so pct is a bound. */ + bounded: boolean; + /** Display label, e.g. "Top 3.2%" or "Top < 0.02%". */ + label: string; +} + +// Percent → display string: whole numbers from 10 up, one decimal from 1, two +// below that. +export function formatPct(p: number): string { + if (p >= 10) return String(Math.round(p)); + if (p >= 1) return p.toFixed(1); + return p.toFixed(2); +} + +/** + * The card's standing in a histogram where `counts[i]` holds the number of + * sampled accounts rated `min + i`. + * + * When nothing in the sample reaches the card's rating, the honest claim isn't + * "top 0%" — it's the rule-of-three bound: with none observed in n samples, the + * true share is below 3/n at 95% confidence. That case is marked `bounded` and + * labelled with a "<". + */ +export function standing( + counts: readonly number[], + n: number, + overall: number, + min: number = DIST_MIN, +): Standing { + const atOrAbove = counts.reduce((sum, c, i) => sum + (min + i >= overall ? c : 0), 0); + const bounded = atOrAbove === 0; + // Guard n: an empty sample has no claim to make rather than an Infinity/NaN one. + const pct = n > 0 ? (100 * (bounded ? 3 : atOrAbove)) / n : 0; + return { atOrAbove, pct, bounded, label: `Top ${bounded ? "< " : ""}${formatPct(pct)}%` }; +} diff --git a/tests/distribution.test.ts b/tests/distribution.test.ts new file mode 100644 index 0000000..fec170a --- /dev/null +++ b/tests/distribution.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; +import { formatPct, standing } from "@/lib/distribution"; +import { DIST_ACTIVE_COUNTS, DIST_ACTIVE_N, DIST_COUNTS, DIST_MIN, DIST_N } from "@/lib/distribution-data"; + +// The "TOP 0.02% of GitHub" line under the scouting metrics. It's a claim the app +// makes publicly about someone, computed from a histogram snapshot that gets +// regenerated (scripts/distribution-runner) — so the maths has to stay right for +// a sample of any size, not just today's. + +// A tiny histogram: ratings 50..54, one account at each of 50–53 and none at 54. +const COUNTS = [10, 5, 3, 2, 0]; // n = 20 +const N = 20; + +describe("formatPct", () => { + it("shows whole numbers from 10 up", () => { + expect(formatPct(10)).toBe("10"); + expect(formatPct(42.4)).toBe("42"); + expect(formatPct(99.6)).toBe("100"); + }); + + it("shows one decimal from 1 to just under 10", () => { + expect(formatPct(9.96)).toBe("10.0"); // still the one-decimal band + expect(formatPct(3.25)).toBe("3.3"); + expect(formatPct(1)).toBe("1.0"); + }); + + it("shows two decimals below 1", () => { + expect(formatPct(0.99)).toBe("0.99"); + expect(formatPct(0.016)).toBe("0.02"); + }); +}); + +describe("standing", () => { + it("counts the sample at or above the rating (inclusive)", () => { + // ratings 52,53 → 3 + 2 = 5 of 20 = 25% + expect(standing(COUNTS, N, 52, 50)).toMatchObject({ atOrAbove: 5, pct: 25, bounded: false }); + }); + + it("includes the card's own bucket, not just the ones above it", () => { + expect(standing(COUNTS, N, 53, 50).atOrAbove).toBe(2); // the 53 bucket itself + }); + + it("counts the whole sample for a rating below the histogram floor", () => { + expect(standing(COUNTS, N, 1, 50)).toMatchObject({ atOrAbove: 20, pct: 100 }); + }); + + it("labels a normal standing without the '<'", () => { + expect(standing(COUNTS, N, 52, 50).label).toBe("Top 25%"); + }); + + // Nothing observed at or above → "top 0%" would be a lie. The honest claim is + // the rule-of-three upper bound, 3/n at 95% confidence. + describe("when nothing in the sample reaches the rating", () => { + it("falls back to the rule-of-three bound and marks it bounded", () => { + const s = standing(COUNTS, N, 54, 50); + expect(s).toMatchObject({ atOrAbove: 0, bounded: true }); + expect(s.pct).toBeCloseTo((100 * 3) / N); // 15% + }); + + it("shows the bound with a '<' so it doesn't read as measured", () => { + expect(standing(COUNTS, N, 54, 50).label).toBe("Top < 15%"); + }); + }); + + it("makes no claim on an empty sample instead of dividing by zero", () => { + const s = standing([], 0, 90); + expect(s.pct).toBe(0); + expect(Number.isFinite(s.pct)).toBe(true); + }); + + it("defaults the histogram floor to DIST_MIN", () => { + expect(standing(DIST_COUNTS, DIST_N, 50)).toEqual(standing(DIST_COUNTS, DIST_N, 50, DIST_MIN)); + }); +}); + +describe("standing — against the shipped sample", () => { + it("puts a floor-rated card at the top of nothing (100% of the sample)", () => { + expect(standing(DIST_COUNTS, DIST_N, DIST_MIN).pct).toBe(100); + }); + + it("rates an elite card above almost everyone, and bounds a rating nobody reached", () => { + const elite = standing(DIST_COUNTS, DIST_N, 90); + expect(elite.pct).toBeLessThan(1); + expect(elite.bounded).toBe(false); // the sample does reach 90 + + const unreached = standing(DIST_COUNTS, DIST_N, 99); + expect(unreached.bounded).toBe(true); // nothing in the sample is 99 + expect(unreached.label).toBe("Top < 0.02%"); + }); + + it("is stricter among active devs than across all of GitHub", () => { + // The active sample is the tougher crowd, so the same rating is rarer overall. + const all = standing(DIST_COUNTS, DIST_N, 75); + const active = standing(DIST_ACTIVE_COUNTS, DIST_ACTIVE_N, 75); + expect(active.pct).toBeGreaterThan(all.pct); + }); + + it("is monotonic — a higher rating never has a larger share above it", () => { + let prev = Infinity; + for (let ovr = DIST_MIN; ovr <= 99; ovr++) { + const { atOrAbove } = standing(DIST_COUNTS, DIST_N, ovr); + expect(atOrAbove).toBeLessThanOrEqual(prev); + prev = atOrAbove; + } + }); +}); From 50cffaee0db2ce45b4d4fa9bf98c00c1c8dc5672 Mon Sep 17 00:00:00 2001 From: Amayyas Date: Tue, 21 Jul 2026 18:58:29 +0200 Subject: [PATCH 2/2] fix(distribution): never label a card "Top < 0.00%" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When nothing in the sample reaches a card's rating, the panel falls back to the rule-of-three bound, 3/n. That share is formatted with two decimals, so any sample past ~60k accounts rounds it to "0.00" and the card is labelled "Top < 0.00%" — a claim about nobody. Today's sample is n = 18,107, so the bound prints as 0.02% and the bug is invisible. It only surfaces when the histogram is regenerated bigger, which is a documented workflow (scripts/distribution-runner.ts) — so it's the kind of thing that breaks later, in someone else's PR. Floors the displayed percentage at 0.01. Still honest: that number is already an upper bound, so showing "< 0.01%" understates nothing. --- lib/distribution.ts | 6 ++++-- tests/distribution.test.ts | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/distribution.ts b/lib/distribution.ts index a446fa2..32d829d 100644 --- a/lib/distribution.ts +++ b/lib/distribution.ts @@ -18,11 +18,13 @@ export interface Standing { } // Percent → display string: whole numbers from 10 up, one decimal from 1, two -// below that. +// below that. Never renders "0.00": two-decimal rounding flattens anything under +// 0.005, and "Top < 0.00%" is a claim about nobody. A floor of 0.01 keeps the +// smallest claim meaningful and still true (it's already an upper bound). export function formatPct(p: number): string { if (p >= 10) return String(Math.round(p)); if (p >= 1) return p.toFixed(1); - return p.toFixed(2); + return Math.max(p, 0.01).toFixed(2); } /** diff --git a/tests/distribution.test.ts b/tests/distribution.test.ts index fec170a..8cad0e3 100644 --- a/tests/distribution.test.ts +++ b/tests/distribution.test.ts @@ -28,6 +28,14 @@ describe("formatPct", () => { expect(formatPct(0.99)).toBe("0.99"); expect(formatPct(0.016)).toBe("0.02"); }); + + // Two-decimal rounding flattens anything under 0.005 to "0.00", and a label + // reading "Top < 0.00%" is a claim about nobody. + it("floors the display at 0.01 instead of rendering 0.00", () => { + expect(formatPct(0.004)).toBe("0.01"); + expect(formatPct(0.0000001)).toBe("0.01"); + expect(formatPct(0)).toBe("0.01"); + }); }); describe("standing", () => { @@ -60,6 +68,13 @@ describe("standing", () => { it("shows the bound with a '<' so it doesn't read as measured", () => { expect(standing(COUNTS, N, 54, 50).label).toBe("Top < 15%"); }); + + // The shipped sample is n = 18_107, so the bound is 0.02% today. Regenerate + // it past ~60k accounts and the bound slips under 0.005 — where two-decimal + // rounding used to print the meaningless "Top < 0.00%". + it("never prints '< 0.00%', however large the sample gets", () => { + expect(standing([1], 100_000, 99, 50).label).toBe("Top < 0.01%"); + }); }); it("makes no claim on an empty sample instead of dividing by zero", () => {