Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 52 additions & 25 deletions lib/github/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ import { tokenPool, pickToken, pickFailover, recordTokenHealth, benchToken, type
// Lifetime contributions need one contributionsCollection window per calendar
// year (each window must span ≤1yr). GitHub's resolver times out (~10s) if too
// many windows share a request, so we fetch the profile in one fast query, then
// the years in small parallel batches with a retry — and tolerate a dropped
// batch (the figure is only used log-scaled, so a missing year barely moves it).
// the years in small parallel batches. Failed batches retry year by year; an
// individually unavailable year becomes zero/inactive so the scout stays usable.
//
// GitHub also rejects contributionsCollection outright (RESOURCE_LIMITS_EXCEEDED)
// when the window holds too many contribution events — measured mid-2026 at
Expand Down Expand Up @@ -72,6 +72,7 @@ export interface RawPayload {
recentRestricted: number; // last-year private contributions (count only)
recentActiveDays: number;
lifetimeContributions: number; // all years, all types, incl. private
activeYears: number; // calendar years where GitHub reports any contribution activity
}

const ENDPOINT = "https://api.github.com/graphql";
Expand Down Expand Up @@ -146,6 +147,17 @@ interface YearContrib {
totalPullRequestContributions: number;
totalPullRequestReviewContributions: number;
restrictedContributionsCount: number;
hasAnyContributions: boolean;
}

interface LifetimeSummary {
totalContributions: number;
activeYears: number;
}

interface YearResult {
contributions: number;
active: boolean;
}

// POSTs a query, retrying transient failures. Terminal failures (bad token,
Expand Down Expand Up @@ -437,7 +449,7 @@ function lifetimeQuery(
const aliases = years
.map((y) => {
const to = y === currentYear ? nowIso : `${y}-12-31T23:59:59Z`;
return ` y${y}: contributionsCollection(from: "${y}-01-01T00:00:00Z", to: "${to}") { totalCommitContributions totalIssueContributions totalPullRequestContributions totalPullRequestReviewContributions restrictedContributionsCount }`;
return ` y${y}: contributionsCollection(from: "${y}-01-01T00:00:00Z", to: "${to}") { totalCommitContributions totalIssueContributions totalPullRequestContributions totalPullRequestReviewContributions restrictedContributionsCount hasAnyContributions }`;
})
.join("\n");
return `
Expand All @@ -461,48 +473,62 @@ const sumContrib = (c: YearContrib) =>
c.totalPullRequestReviewContributions +
c.restrictedContributionsCount;

// Sum of every year's contributions (commits + issues + PRs + reviews + private).
// Each batch is best-effort: a batch that fails after its retry contributes 0
// Summarize every year's contributions (commits + issues + PRs + reviews +
// private) while the annual boundaries still exist. Each batch is best-effort:
// a year that remains unavailable after its retry contributes 0 and is inactive
// rather than failing the scout.
async function fetchLifetime(
login: string,
tok: PoolToken,
createdYear: number,
currentYear: number,
nowIso: string,
): Promise<number> {
): Promise<LifetimeSummary> {
const years: number[] = [];
for (let y = Math.max(createdYear, GITHUB_EPOCH_YEAR); y <= currentYear; y++) years.push(y);

const yearsTotal = async (batch: number[]): Promise<number> => {
const fetchYears = async (batch: number[]): Promise<YearResult[]> => {
const { user } = await gql<Record<string, YearContrib | null>>(
lifetimeQuery(batch, currentYear, nowIso),
login,
tok,
);
if (!user) return 0;
return batch.reduce((s, y) => {
const c = user[`y${y}`];
return c ? s + sumContrib(c) : s;
}, 0);
return batch.map((year) => {
const collection = user?.[`y${year}`];
return collection
? {
contributions: sumContrib(collection),
active: collection.hasAnyContributions,
}
: { contributions: 0, active: false };
});
};

const sums = await Promise.all(
const batches = await Promise.all(
chunk(years, LIFETIME_BATCH).map(async (batch) => {
try {
return await yearsTotal(batch);
return await fetchYears(batch);
} catch {
// Aliased years pool their resource cost, so one hyperactive year sinks
// its whole batch — retry each year alone (own request, own budget) and
// let only the truly over-budget years degrade to 0.
const singles = await Promise.all(
batch.map((y) => yearsTotal([y]).catch(() => 0)),
// Aliased years pool their resource cost, so one unavailable year can
// sink its whole batch. Retry each year with its own request and budget,
// then degrade only individually unavailable years to zero/inactive.
return Promise.all(
batch.map(async (year) => {
try {
return (await fetchYears([year]))[0];
} catch {
return { contributions: 0, active: false };
}
}),
);
return singles.reduce((a, b) => a + b, 0);
}
}),
);
return sums.reduce((a, b) => a + b, 0);
const results = batches.flat();
return {
totalContributions: results.reduce((total, year) => total + year.contributions, 0),
activeYears: results.filter((year) => year.active).length,
};
}

export async function fetchProfile(
Expand Down Expand Up @@ -547,18 +573,18 @@ export async function fetchProfile(
if (!user) return fail("notfound", "No GitHub user by that name.");

const createdYear = new Date(user.createdAt).getUTCFullYear();
const lifetimeContributions = await fetchLifetime(
const lifetime = await fetchLifetime(
login,
tok,
createdYear,
now.getUTCFullYear(),
now.toISOString(),
);

return normalize(user, lifetimeContributions);
return normalize(user, lifetime);
}

function normalize(user: UserNode, lifetimeContributions: number): RawPayload {
function normalize(user: UserNode, lifetime: LifetimeSummary): RawPayload {
const repos: RawRepo[] = user.repositories.nodes.map((n) => ({
stars: n.stargazerCount ?? 0,
language: n.primaryLanguage?.name ?? null,
Expand Down Expand Up @@ -608,6 +634,7 @@ function normalize(user: UserNode, lifetimeContributions: number): RawPayload {
recentIssues: user.recent.totalIssueContributions,
recentRestricted: user.recent.restrictedContributionsCount,
recentActiveDays,
lifetimeContributions,
lifetimeContributions: lifetime.totalContributions,
activeYears: lifetime.activeYears,
};
}
16 changes: 7 additions & 9 deletions lib/github/signals.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { rankLanguages } from "./languages";
import type { Signals } from "@/lib/scoring/types";

const YEAR_MS = 31557600000;
const yearOf = (iso: string) => (iso ? new Date(iso).getUTCFullYear() : null);

// Maps the (already real, already flattened) GraphQL payload onto the scoring
// signals. No estimation — every field traces back to a GitHub number.
Expand All @@ -21,14 +20,13 @@ export function signalsFromPayload(p: RawPayload, now = Date.now()): Signals {
// styling/markup (CSS/HTML) — the #1 drives the card's language + logo.
const rankedLanguages = rankLanguages(p.languageRepos);

const years = new Set<number>();
for (const r of p.repos) {
const c = yearOf(r.createdAt);
const pushed = yearOf(r.pushedAt);
if (c) years.add(c);
if (pushed) years.add(pushed);
}
const active_years = Math.min(Math.max(years.size, 1), Math.ceil(account_age_years) || 1);
// GitHub reports this over the same annual contribution collections used for
// the lifetime total, so organization and private activity count even when no
// personally-owned repository changed that year.
const createdYear = new Date(p.createdAt).getUTCFullYear();
const currentYear = new Date(now).getUTCFullYear();
const accountCalendarYears = Math.max(0, currentYear - createdYear + 1);
const active_years = Math.min(Math.max(p.activeYears, 0), accountCalendarYears);

// Recent activity over the last year: every contribution type GitHub exposes,
// including the private (restricted) count, so it matches the profile graph.
Expand Down
80 changes: 57 additions & 23 deletions scripts/distribution-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ const TOKEN = process.env.GH_TOKEN!;
const MAX_ID = 300_000_000;
const ENDPOINT = "https://api.github.com/graphql";
const CONCURRENCY = 8;
const LIFETIME_BATCH = 4;
const DIST_MIN = 50;
const FLUSH_EVERY = 100;

Expand Down Expand Up @@ -156,50 +157,82 @@ type YearContrib = {
totalPullRequestContributions: number;
totalPullRequestReviewContributions: number;
restrictedContributionsCount: number;
hasAnyContributions: boolean;
};

type YearResult = {
contributions: number;
active: boolean;
};

type LifetimeSummary = {
totalContributions: number;
activeYears: number;
};

function lifetimeQuery(years: number[], currentYear: number, nowIso: string): string {
const aliases = years
.map((y) => {
const to = y === currentYear ? nowIso : `${y}-12-31T23:59:59Z`;
return `y${y}: contributionsCollection(from: "${y}-01-01T00:00:00Z", to: "${to}") { totalCommitContributions totalIssueContributions totalPullRequestContributions totalPullRequestReviewContributions restrictedContributionsCount }`;
return `y${y}: contributionsCollection(from: "${y}-01-01T00:00:00Z", to: "${to}") { totalCommitContributions totalIssueContributions totalPullRequestContributions totalPullRequestReviewContributions restrictedContributionsCount hasAnyContributions }`;
})
.join("\n");
return `query Lifetime($login: String!) { user(login: $login) { ${aliases} } }`;
}

async function fetchLifetime(login: string, createdYear: number): Promise<number> {
async function fetchLifetime(login: string, createdYear: number): Promise<LifetimeSummary> {
const now = new Date();
const currentYear = now.getUTCFullYear();
const years: number[] = [];
for (let y = Math.max(createdYear, 2008); y <= currentYear; y++) years.push(y);
const batches: number[][] = [];
for (let i = 0; i < years.length; i += 5) batches.push(years.slice(i, i + 5));
const sums = await Promise.all(
batches.map(async (batch) => {
const u = await gql<Record<string, YearContrib | null>>(lifetimeQuery(batch, currentYear, now.toISOString()), login);
if (!u) return 0;
return batch.reduce((s, y) => {
const c = u[`y${y}`];
return c
? s +
c.totalCommitContributions +
c.totalIssueContributions +
c.totalPullRequestContributions +
c.totalPullRequestReviewContributions +
c.restrictedContributionsCount
: s;
}, 0);
}),
);
return sums.reduce((a, b) => a + b, 0);
for (let i = 0; i < years.length; i += LIFETIME_BATCH) batches.push(years.slice(i, i + LIFETIME_BATCH));

const fetchYears = async (batch: number[]): Promise<YearResult[] | null> => {
const user = await gql<Record<string, YearContrib | null>>(
lifetimeQuery(batch, currentYear, now.toISOString()),
login,
);
if (!user) return null;
return batch.map((year) => {
const collection = user[`y${year}`];
return collection
? {
contributions:
collection.totalCommitContributions +
collection.totalIssueContributions +
collection.totalPullRequestContributions +
collection.totalPullRequestReviewContributions +
collection.restrictedContributionsCount,
active: collection.hasAnyContributions,
}
: { contributions: 0, active: false };
});
};

const results = (
await Promise.all(
batches.map(async (batch) => {
const grouped = await fetchYears(batch);
if (grouped) return grouped;
return Promise.all(
batch.map(async (year) => (await fetchYears([year]))?.[0] ?? { contributions: 0, active: false }),
);
}),
)
).flat();

return {
totalContributions: results.reduce((total, year) => total + year.contributions, 0),
activeYears: results.filter((year) => year.active).length,
};
}

async function fetchPayload(login: string): Promise<RawPayload | null> {
const user = await gql<UserNode>(PROFILE_QUERY, login);
if (!user) return null;
const createdYear = new Date(user.createdAt).getUTCFullYear();
const lifetimeContributions = await fetchLifetime(login, createdYear);
const lifetime = await fetchLifetime(login, createdYear);
const repos: RawRepo[] = user.repositories.nodes.map((n) => ({
stars: n.stargazerCount ?? 0,
language: n.primaryLanguage?.name ?? null,
Expand Down Expand Up @@ -241,7 +274,8 @@ async function fetchPayload(login: string): Promise<RawPayload | null> {
recentIssues: user.recent.totalIssueContributions,
recentRestricted: user.recent.restrictedContributionsCount,
recentActiveDays,
lifetimeContributions,
lifetimeContributions: lifetime.totalContributions,
activeYears: lifetime.activeYears,
};
}

Expand Down
Loading