diff --git a/.github/workflows/seed-popular.yml b/.github/workflows/seed-popular.yml index ce961d4..b71daa4 100644 --- a/.github/workflows/seed-popular.yml +++ b/.github/workflows/seed-popular.yml @@ -2,7 +2,8 @@ name: Seed popular repos on: schedule: - - cron: '0 3 * * *' # 03:00 UTC daily + # Weekly, away from the top-of-hour GitHub Actions load spike. + - cron: '17 3 * * 0' # 03:17 UTC every Sunday workflow_dispatch: inputs: daily_limit: @@ -13,6 +14,10 @@ on: description: 'Repos to inspect for tool intelligence' required: false default: '250' + max_additions: + description: 'Abort-before-write repository addition bound (0-100)' + required: false + default: '100' concurrency: group: starboard-embedding @@ -47,7 +52,8 @@ jobs: - name: Run seed script run: pnpm db:seed-popular env: - # The runner token has enough GitHub Search quota for a bounded walk. + # The runner token has enough GitHub Search quota for one complete, + # rate-paced catalog reconciliation. # Keep this independent of long-lived PAT rotation so a stale secret # cannot break a scheduled seed run with 401 Bad credentials. GITHUB_TOKEN: ${{ github.token }} @@ -55,8 +61,9 @@ jobs: D1_DATABASE_ID: ${{ vars.D1_DATABASE_ID }} CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} SEED_EMBED_MODE: 'worker' - # Operational row-read circuit breaker. The script also hard-caps this at 25. - SEED_METADATA_PAGE_LIMIT: '10' + # Fail before repository writes when the source delta is anomalous. + SEED_MAX_ADDITIONS: ${{ github.event.inputs.max_additions || '100' }} + SEED_MIN_SOURCE_REPOS: '5000' MIN_STARS_FLOOR: '5000' - name: Embed refreshed repos through Worker bindings diff --git a/AGENTS.md b/AGENTS.md index 19266ba..31dc0d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,7 @@ pnpm test:e2e # playwright pnpm lint # biome check . pnpm db:migrate # apply migrations/* to isolated local D1 pnpm db:migrate:remote # validate config + apply migrations/* to remote D1 (approval required) -pnpm db:seed-popular # cold-seed popular repos (≥5k stars) — used by daily GH Action +pnpm db:seed-popular # reconcile missing popular repos (≥5k stars) — weekly GH Action pnpm db:seed-embeddings# backfill repo_embeddings pnpm docs:check # validate docs/ links + structure pnpm docs:dev # blume dev (local docs site; requires pnpm add -D blume) diff --git a/PROJECT_STATUS.md b/PROJECT_STATUS.md index 8749316..8342d21 100644 --- a/PROJECT_STATUS.md +++ b/PROJECT_STATUS.md @@ -1,6 +1,6 @@ # starboard — PROJECT STATUS -Last updated: 2026-08-09 +Last updated: 2026-08-13 ## Why/What @@ -28,7 +28,7 @@ installation, alerts, reports, digest email, and stack generation. | Client state | SWR (data), nuqs (URL-backed filters/sort) | | AI / search | Cloudflare Workers AI `@cf/baai/bge-base-en-v1.5` (768d); optional `knowledgebase` Worker via service binding | | Deploy | Cloudflare Workers via OpenNext (`@opennextjs/cloudflare`) | -| CI | GitHub Actions — push CI + manual SHA-tagged deploy + daily bounded seed/enrich/embed | +| CI | GitHub Actions — push CI + manual SHA-tagged deploy + weekly complete additions-only seed/enrich/embed | **Local dev:** `pnpm install && cp .env.example .env.local && pnpm dev` → http://localhost:3000 @@ -66,12 +66,21 @@ provenance. The workflow is free and has no billing or entitlement gate. | Secrets | `AUTH_SECRET`, `AUTH_GITHUB_ID`, `AUTH_GITHUB_SECRET`; `AI_GATEWAY_API_KEY` for authenticated operator jobs; `RAG_SERVICE_KEY` for relevance RAG. Any legacy unused `TURSO_*` bindings are separate credential-cleanup work. | | Embedding model | `@cf/baai/bge-base-en-v1.5` — change model, dimension, and replacement Vectorize index together | | Project connections | Additive `0003_user_projects.sql`; remote migration requires explicit approval before application rollout | -| Data refresh jobs | Daily bounded `seed-popular` at 03:00 UTC plus manual seed/enrich/embed dispatches | +| Data refresh jobs | Weekly complete additions-only `seed-popular` at Sunday 03:17 UTC plus manual seed/enrich/embed dispatches | | Deploy | `pnpm deploy:cf` or manual `deploy.yml` dispatch; both attach the full Git SHA | | Smoke | `pnpm test` + `pnpm build`; for search/DB changes also `pnpm db:migrate` and `pnpm build:cf` | ## Timeline +- **2026-08-13 (weekly catalog reconciliation implemented locally)** — Replaced + the multi-day popular-repository cursor walk with one complete weekly GitHub + identity reconciliation. Immutable creation-date partitions fit in single + Search responses, all stored D1 IDs are read once, and only source-only + additions are fetched and inserted. Incomplete or unstable source evidence + and deltas above 100 fail before writes; deletion and existing-row refresh + remain out of scope. Focused reconciliation, D1 regression, and refresh + evidence tests pass; production activation awaits normal review and push. + - **2026-08-09 (discovery entry and cache isolation complete locally)** — Made Discover the default generic sign-in destination, removed the 100-repository GitHub picker cap through safe pagination, tightened Discover card sizing and @@ -199,7 +208,7 @@ provenance. The workflow is free and has no billing or entitlement gate. | Repo intelligence | Shared-shell repository detail (`/explore`), similar projects, tool evidence, public shared lists, legal/marketing shell | | Semantic search | knowledgebase Worker integration for relevance search; README-backed sync ingest; local embeddings retained for non-RAG Starboard features | | Connected projects | Shipped public GitHub project connections, public preview, GitHub picker, and evidence-based repository and tool recommendations | -| Discovery & tools | Public Discover, daily bounded seed/enrich/embed with manual dispatch, stored growth sorting, and Tool Intelligence | +| Discovery & tools | Public Discover, weekly complete additions-only seed/enrich/embed with manual dispatch, stored growth sorting where snapshot history exists, and Tool Intelligence | | Removed 2026-08-08 | Fleet project catalog, Alerts, Reports, Stack Builder, standalone Radar, weekly digest | | Ops hardening (2026-06-20) | `.env.example`, Vitest + Playwright path, pre-push lint, self-contained TypeScript/Astro landing for green CF builds | @@ -264,8 +273,9 @@ provenance. The workflow is free and has no billing or entitlement gate. collection controls but is not required to browse, search, sort, filter, paginate, or open repo details. - Discover supports paginated 30-day growth ordering and detected-tool facets from indexed local snapshot/tool tables. -- Daily bounded GitHub Actions seed/enrich popular repos in D1 and embed through - native Worker bindings; manual dispatch remains available for operator checks. +- Weekly complete, additions-only GitHub Actions reconciliation inserts missing + popular repos in D1 and embeds through native Worker bindings; manual dispatch + remains available for operator checks. - Star history and fastest-grower APIs/surfaces: `/api/repos/[repoId]/star-history`, `/api/growth`, Discover growth sorting, and repo-detail mini history from stored `repo_star_snapshots`. - Tool Intelligence: additive `repo_tools` index, `/api/tools`, `/api/repos/[repoId]/tools`, `/tools`, and `pnpm db:enrich-tools` for bounded SBOM/tree/manifest-based detection with source/confidence labels. Repository evidence is server-filtered and paginated in 48-item pages. Accuracy disclaimer is shown in-product because manifest/SBOM evidence is stronger than README/topic/metadata inference and C/C++ monorepos vary. - SaaS Maker feedback widget integrated; product analytics run directly through PostHog. diff --git a/docs/architecture/data-flow.md b/docs/architecture/data-flow.md index 7c8305d..00b15f9 100644 --- a/docs/architecture/data-flow.md +++ b/docs/architecture/data-flow.md @@ -15,16 +15,19 @@ filtered by `session.user.githubId`. ## Public discovery ```text -daily or manually dispatched seed-popular job - → GitHub Search metadata - → repos + star snapshots in D1 +weekly or manually dispatched seed-popular job + → complete, creation-date-partitioned GitHub Search identity set + → diff against all stored D1 repository IDs + → fetch details and insert source-only additions + initial snapshots → bounded embedding and tool-enrichment jobs → Vectorize + repo_tools → public Discover and Tool Intelligence ``` -Discover requests do not synchronously call GitHub. Growth uses stored snapshot -history and remains empty when insufficient samples exist. +Discover requests do not synchronously call GitHub. The weekly catalog job is +additions-only: it does not delete or refresh existing repository rows. Growth +uses snapshots captured at ingest or user sync and remains empty when +insufficient samples exist. ## Connected projects diff --git a/docs/development/commands.md b/docs/development/commands.md index 7dd667d..4139599 100644 --- a/docs/development/commands.md +++ b/docs/development/commands.md @@ -41,7 +41,7 @@ The `--webpack` flag remains the verified OpenNext production build path. | `pnpm db:convert-turso-vectors` | Validate/convert extracted 768-d vectors to Vectorize NDJSON | | `pnpm db:snapshot-turso-logical starboard` | Produce a row-free deterministic Turso digest for final cutover reconciliation | | `pnpm db:enrich-repos` | `tsx scripts/enrich-repos.ts` — AI metadata enrichment | -| `pnpm db:seed-popular` | `tsx scripts/seed-popular.ts` — cold-seed popular repos (≥5k stars) | +| `pnpm db:seed-popular` | `tsx scripts/seed-popular.ts` — fully reconcile and add missing popular repos (≥5k stars) | | `pnpm db:enrich-tools` | `tsx scripts/enrich-tools.ts` — SBOM/tree/manifest tool detection | ## Testing diff --git a/docs/operations/data-map.md b/docs/operations/data-map.md index f3b4b04..99d67f9 100644 --- a/docs/operations/data-map.md +++ b/docs/operations/data-map.md @@ -28,7 +28,7 @@ retirement requires separate approval. | Store | Class | Owner | Reconstruction | Expected cost | Last verified | | --- | --- | --- | --- | --- | --- | | D1 `users` | irreplaceable-user | NextAuth GitHub OAuth | Not reconstructable — GitHub is the source of identity, but user records (email, created_at) must be exported | n/a — export required | 2026-08-02 | -| D1 `repos` (popular ≥5k seeded) | authoritative-source | `scripts/seed-popular.ts` | Re-walk GitHub Search ≥`MIN_STARS_FLOOR` | ~hours (rate-limited, resumable cursor) | 2026-08-02 | +| D1 `repos` (popular ≥5k seeded) | authoritative-source | `scripts/seed-popular.ts` | Fully reconcile creation-date-partitioned GitHub Search ≥`MIN_STARS_FLOOR`; insert source-only IDs | ~15–30 minutes | 2026-08-13 | | D1 `user_repos` (starred/saved state) | irreplaceable-user | GitHub sync via `/api/stars/sync` | Re-sync from GitHub starred list (ETag + HTML scrape) | ~seconds per user | 2026-08-02 | | D1 `user_projects` | irreplaceable-user | Project connection UI/API | Reconnect public GitHub repositories manually | ~seconds per user | 2026-08-08 | | D1 `user_lists`, `user_repo_lists` | irreplaceable-user | User UI actions | Not reconstructable — user-curated collections | n/a — export required | 2026-08-02 | @@ -36,8 +36,8 @@ retirement requires separate approval. | D1 `repo_embeddings` hashes + Vectorize `starboard-repos` values | derived | Worker binding embedding jobs | Re-embed from `repos` + `repo_ai_metadata` text via Workers AI | ~minutes (Workers AI quota) | 2026-08-02 | | D1 `repo_ai_metadata` | derived | `scripts/enrich-repos.ts` (free-ai gateway) | Re-enrich from `repos` metadata via AI | ~minutes per batch | 2026-08-02 | | D1 `repo_tools` | derived | `scripts/enrich-tools.ts` | Re-detect from GH tree/manifest/SBOM | ~minutes per batch | 2026-08-02 | -| D1 `repo_star_snapshots`, `repo_threshold_events` | derived | `seed-popular.ts` snapshot inserts | Re-derive from `repos` star counts over time | rebuilt on each seed run | 2026-08-02 | -| D1 `seed_cursor` | derived (walk state) | `seed-popular.ts` | Reset to defaults; walk restarts from top | seconds | 2026-08-02 | +| D1 `repo_star_snapshots`, `repo_threshold_events` | derived | Seed and user-sync snapshot inserts | Re-derive initial samples from `repos`; historical series requires repeated user sync | additions only during weekly reconciliation | 2026-08-13 | +| D1 `seed_cursor` | legacy derived state (inactive) | No active writer | Safe to ignore; retained to avoid a destructive migration | n/a | 2026-08-13 | | D1 `insight_reports`, `user_alert_preferences` | historical inactive storage | Removed product features | Retained to avoid a destructive migration; no active writers | n/a | 2026-08-08 | | Cloudflare Worker `starboard` (deployed bundle) | cache | `pnpm deploy:cf` or manual deploy workflow | Rebuild + redeploy | ~minutes | 2026-07-18 | | knowledgebase Worker RAG index (`STARBOARD_RAG_INDEX_ID`) | derived (RAG index of user repos) | `src/lib/knowledgebase.ts` ingest | Re-ingest from `repos` + README text per user | ~seconds per user | 2026-07-18 | @@ -67,10 +67,11 @@ window. ### Full popular-pool rebuild (bounded) `pnpm db:migrate:remote` → scheduled or manual `seed-popular` workflow -(resumable cursor walk of GitHub Search ≥5k stars plus bound Worker embedding backfill) → -`pnpm db:enrich-tools` (tool detection). Total runtime -is bounded by `SEED_METADATA_PAGE_LIMIT` (default 10, hard cap 25 pages/run) -and the workflow's `daily_limit` input (default 1000 embeddings/run). See +(complete one-response date partitions of GitHub Search ≥5k stars, D1 ID diff, +source-only inserts, and bounded Worker embedding backfill) → +`pnpm db:enrich-tools` (tool detection). The pre-write +`SEED_MAX_ADDITIONS` bound defaults to 100 in the workflow, and `daily_limit` +defaults to 1000 embeddings per run. See [`jobs.md`](jobs.md) §seed-popular. ### Embedding dimension drift @@ -92,15 +93,15 @@ Each scheduled or dispatched `seed-popular` GitHub Action records a structured manifest at `data/refresh-manifest.json` and copies it to the existing GitHub Actions run summary before the ephemeral runner is discarded. The manifest includes: -- `source_watermark` — GitHub Search cursor (`next_max_stars`/`next_page`) - and run timestamp -- `bounds` — `METADATA_PAGE_LIMIT`, `DAILY_LIMIT`, `MIN_STARS_FLOOR` +- `source_watermark` — verified GitHub unique-ID count +- `bounds` — source/stored/addition/stored-only counts, + `SEED_MAX_ADDITIONS`, `SEED_MIN_SOURCE_REPOS`, and `MIN_STARS_FLOOR` - `timeout` — workflow `timeout-minutes: 60` -- `idempotency` — `INSERT … ON CONFLICT(id) DO UPDATE` for `repos`; - `INSERT OR IGNORE` for `repo_star_snapshots` and `repo_threshold_events` +- `idempotency` — in-memory source/stored ID diff plus `INSERT OR IGNORE` for + additions; no existing-row update or stored-only deletion - `retries` — `withDbRetry` (4 attempts, exponential backoff) for D1; `ghSearch` (4 attempts + rate-limit sleep) for GitHub -- `output_counts` — `upsertedThisRun`, `embedded`, pool totals +- `output_counts` — inserted additions, embedded additions, and pool totals - `quality_signal` — non-zero output check + pool coverage ratio - `freshness` — run wall-clock + delta from prior success - `failure_state` — unresolved failure state within that run's manifest diff --git a/docs/operations/env.md b/docs/operations/env.md index 0e2d914..e5af38a 100644 --- a/docs/operations/env.md +++ b/docs/operations/env.md @@ -47,6 +47,12 @@ The `seed-popular` workflow deliberately uses `${{ github.token }}` for GitHub Search (not a long-lived PAT) so a stale PAT cannot break a scheduled or manual seed run with 401 Bad credentials. +The scheduled reconciliation sets `MIN_STARS_FLOOR=5000`, +`SEED_MIN_SOURCE_REPOS=5000`, and `SEED_MAX_ADDITIONS=100`. The last value is a +pre-write Cloudflare budget circuit breaker and is exposed as an explicit +manual-dispatch input that can lower the bound. Values above the code-level hard +limit of 100 are rejected before GitHub or D1 access. + ## Public keys `NEXT_PUBLIC_SAASMAKER_API_KEY` is a public key (expected to be visible in diff --git a/docs/operations/jobs.md b/docs/operations/jobs.md index 070b711..ba286af 100644 --- a/docs/operations/jobs.md +++ b/docs/operations/jobs.md @@ -6,18 +6,21 @@ annotates intent, inputs, and dependencies. ## seed-popular (`.github/workflows/seed-popular.yml`) -- **Schedule:** daily at 03:00 UTC and manual `workflow_dispatch`. Automation - was restored after the D1 cutover removed the prior Turso row-read allowance; - the bounded walk and unchanged-row protections remain in force. -- **Inputs:** `daily_limit` (default 1000), `tool_enrich_limit` (default 250). +- **Schedule:** Sundays at 03:17 UTC and manual `workflow_dispatch`. The + non-top-of-hour minute reduces GitHub Actions scheduling contention. +- **Inputs:** `daily_limit` (embedding limit, default 1000), + `tool_enrich_limit` (default 250), and `max_additions` (default 100). - **Concurrency:** shared group `starboard-embedding`, `cancel-in-progress: false`, so seed and standalone backfill cannot duplicate embedding work. - **Timeout:** 60 minutes. - **Steps:** 1. `pnpm db:migrate:remote` (approval-gated D1 migrations). - 2. `pnpm db:seed-popular` (`scripts/seed-popular.ts`) — GitHub Search for - repos ≥ `MIN_STARS_FLOOR=5000`, with a resumable cursor in `seed_cursor`. - Uses `${{ github.token }}` deliberately so a stale PAT cannot break seeding. + 2. `pnpm db:seed-popular` (`scripts/seed-popular.ts`) — completely enumerate + GitHub Search repos ≥ `MIN_STARS_FLOOR=5000` through non-overlapping + creation-date partitions that each fit one response; compare the resulting + IDs with one `SELECT id FROM repos`; fetch details and insert only IDs absent + from D1. Uses `${{ github.token }}` deliberately so a stale PAT cannot break + reconciliation. 3. Authenticated Worker operator request — Workers AI embeddings → Vectorize binding, with drift hashes written through the D1 binding. 4. `pnpm db:enrich-tools` (`scripts/enrich-tools.ts`) — SBOM/tree/manifest @@ -26,9 +29,24 @@ annotates intent, inputs, and dependencies. - **Credentials:** scoped D1 `CLOUDFLARE_API_TOKEN`, non-secret account/database variables, and the existing AI gateway key as the Worker operator bearer. GitHub does not receive Vectorize API access. -- **Safety controls:** metadata walks default to 10 GitHub Search pages and - hard-cap at 25; unchanged repos do not update or fire FTS maintenance; - snapshots are written only when star counts change. +- **Completeness controls:** any `incomplete_results`, duplicate identity, + truncated date partition, source-count drift, or unique-ID mismatch fails the + run before D1 writes. Root source counts are checked before and after the walk. +- **D1 budget controls:** the job reads all stored IDs once (currently roughly + 15,000 rows, about 0.3% of Cloudflare's 5 million free daily row-read + allowance), then applies `SEED_MAX_ADDITIONS` before detail fetches or writes. + The scheduled default is 100 additions against the 100,000 free daily + row-write allowance, and code rejects manual values above 100 before GitHub or + D1 access. Existing rows are not updated, stored-only rows are not deleted, + and new rows are inserted in batches of 50. The existing embedding and tool + enrichment steps remain bounded at 1,000 and 250 repositories respectively; + because unchanged hashes are skipped, a normal weekly run processes only new + or independently changed rows. See Cloudflare's + [current D1 pricing](https://developers.cloudflare.com/d1/platform/pricing/). +- **GitHub budget controls:** Search requests are sequential and paced at 2.1 + seconds, below the authenticated 30 requests/minute Search bucket. Complete + enumeration is expected to use roughly 250 requests, below the workflow + token's 1,000 requests/hour per-repository allowance. ## embed-pending (`.github/workflows/embed-pending.yml`) diff --git a/docs/operations/refresh-manifest.md b/docs/operations/refresh-manifest.md index 480e6ae..cece19e 100644 --- a/docs/operations/refresh-manifest.md +++ b/docs/operations/refresh-manifest.md @@ -25,14 +25,14 @@ cross-run latest-watermark/unresolved-failure view. ```json { "runs": { - "seed_walk": { - "step": "seed_walk", - "source_watermark": "cursor_after_walk", - "bounds": {"metadata_page_limit": 120, "min_stars_floor": 5000, "max_pages_per_bucket": 10}, + "seed_reconciliation": { + "step": "seed_reconciliation", + "source_watermark": "github_unique_ids:12351", + "bounds": {"min_stars_floor": 5000, "min_source_repos": 5000, "max_additions": 100, "source_count": 12351, "stored_count": 14500, "planned_additions": 42, "stored_only_count": 2191, "leaf_partitions": 128}, "timeout_s": 3600, - "idempotency": "INSERT … ON CONFLICT(id) DO UPDATE for repos; INSERT OR IGNORE for repo_star_snapshots and repo_threshold_events", + "idempotency": "Complete source and stored ID sets are diffed before INSERT OR IGNORE; existing rows are never updated and stored-only rows are never deleted", "retries": {"maxAttempts": 4, "backoffBaseMs": 1000, "used": 0}, - "output_count": 312, + "output_count": 42, "evidence_status": "produced", "quality_signal": {"expected_min_output": 0, "verified_noop_reason": null}, "quality_failed": false, @@ -56,9 +56,10 @@ its evidence is `missing`. Missing or below-minimum evidence is marked catches the "green job writes empty/poor output" failure mode that an exit code alone would miss. -For `seed-popular`, `expected_min_output` is `0` on `seed_walk` and -`seed_embed`, but zero is accepted only after their upstream query completes -and supplies the recorded verified-no-op reason. `seed_pool_coverage` requires +For `seed-popular`, `expected_min_output` is `0` on `seed_reconciliation` and +`seed_embed`, but zero is accepted only after complete source verification or a +completed pending-embedding query supplies a verified-no-op reason. +`seed_pool_coverage` requires at least one embedded repo, so an all-zero run fails. Embedding authentication failure is recorded as `failed` and also fails the job. @@ -71,7 +72,7 @@ unreadable or malformed prior evidence fails instead of being overwritten. | Step | Source | Idempotency | Expected min output | | --- | --- | --- | --- | -| `seed_walk` | GitHub Search (≥`MIN_STARS_FLOOR`) | `repos` upsert + `INSERT OR IGNORE` snapshots/events | 0 (catch-up runs are legitimate) | +| `seed_reconciliation` | Complete GitHub Search identity set (≥`MIN_STARS_FLOOR`) + all stored D1 IDs | In-memory diff + addition-only `INSERT OR IGNORE`; no updates/deletes | 0 (a verified complete no-addition run is legitimate) | | `seed_embed` | Workers AI / free-ai gateway | `repo_embeddings` upsert keyed by `text_hash` | 0 (verified no-pending-work only; auth failure fails the job) | | `seed_pool_coverage` | D1 aggregate | read-only | 1 | diff --git a/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/.openspec.yaml b/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/.openspec.yaml new file mode 100644 index 0000000..b6b2d1f --- /dev/null +++ b/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-13 diff --git a/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/design.md b/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/design.md new file mode 100644 index 0000000..65e59aa --- /dev/null +++ b/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/design.md @@ -0,0 +1,121 @@ +## Context + +See `proposal.md` for motivation. The current job pages a star-sorted GitHub +Search result across multiple daily cursor runs. GitHub caps an individual +search at 1,000 results, repository star counts can reorder paginated results, +and D1 usage is billed by rows read and written. The production table currently +contains roughly 14,500 repository IDs, while the eligible GitHub source set is +roughly 12,400 IDs. + +Cloudflare's current free D1 allowance is 5 million rows read and 100,000 rows +written per day. Reading every stored repository ID once consumes roughly +0.3% of that daily read allowance. Normal weekly growth has been under 100 +repositories, but source anomalies must be rejected before writes because FTS +triggers and indexes amplify each repository insertion. + +## Goals / Non-Goals + +**Goals:** + +- Establish one complete, reviewable source reconciliation per week. +- Avoid star-order pagination races and GitHub's 1,000-result search window. +- Perform one small D1 ID scan and write only source-only additions. +- Bound the maximum possible write burst before the first mutation. +- Preserve existing embedding and tool-enrichment follow-up behavior for new + repository rows. + +**Non-Goals:** + +- Deleting stored repositories that disappear from the source set. +- Refreshing metadata or star counts on existing repository rows. +- Replacing GitHub Search or changing the 5,000-star eligibility contract. +- Running migrations, deploying, or changing Cloudflare plan configuration. + +## Decisions + +### Partition by immutable repository creation date + +The enumerator first records GitHub's total count for `stars:>=floor`, then +recursively splits the inclusive creation-date range from 2007-01-01 through +the current UTC date. A range is accepted only when its total fits in one +100-item response; otherwise it is divided into non-overlapping UTC day ranges. +Any single day that cannot fit one response fails closed. + +This uses more requests than ten-page star buckets but stays below both the +30-search-requests/minute bucket (with the existing 2.1-second pacing) and the +workflow token's 1,000-request/hour allowance. Unlike star-count boundaries, +creation dates do not change as stars accumulate, and one-response leaves avoid +pagination reorder races. The job rechecks the root count after enumeration and +requires the before/after counts and unique-ID count to agree. + +Alternatives considered: + +- Continue star buckets: fewer requests, but ties and changing star order can + skip or duplicate IDs at result-window boundaries. +- Page all results by stars: impossible beyond GitHub's 1,000-result window. +- List every public GitHub repository: stable ID pagination, but scanning the + entire public universe to find 5,000-star repositories is not viable. + +### Diff compact identities, then fetch additions + +Enumeration retains only `id` and `full_name` in a Map. D1 supplies one +`SELECT id FROM repos` result, including repositories that entered through user +sync or project connection. The source-only entries are computed in memory. +Only those entries are fetched from GitHub's repository endpoint to obtain the +metadata needed by `repos`. + +This matches the additions-only contract, keeps memory use small, and avoids +re-reading existing D1 rows or retaining full GitHub search payloads. The detail +fetches complete and validate IDs/star eligibility before the first write. + +### Fail before writes when additions exceed 100 + +The scheduled workflow sets `SEED_MAX_ADDITIONS=100`. This accommodates the +observed current gap and normal weekly growth while sharply bounding FTS/index +write amplification. Operators can deliberately override the value for a +bootstrap or recovery dispatch after reviewing Cloudflare capacity. + +The gate is evaluated after complete enumeration and the D1 ID read but before +detail fetches or mutations. It is a safety failure, not a partial import. + +### Insert additions in small batches without conflict updates + +Repository statements use conflict-ignore semantics. Batch results identify +which rows were actually inserted before addition-only snapshots and threshold +events are written. This protects against a concurrent user sync inserting a +repository after the stored-ID snapshot. `cataloged_at` is set only on a new +row. Existing repository rows are never updated by this job. + +### Keep reconciliation evidence separate from searchable coverage + +The manifest records the complete source total and the planned/inserted +addition counts under a reconciliation step. The existing embedding coverage +gate remains, while the workflow summary exposes the full manifest. A verified +zero-addition run is successful only after source count and identity checks. + +## Risks / Trade-offs + +- **[GitHub's live corpus changes during enumeration]** → Compare root counts + before and after and fail before writes if they differ; the next scheduled or + manual run retries from scratch. +- **[A creation day contains more than 100 eligible repositories]** → Fail with + the exact date instead of silently truncating; a future timestamp-level + partition can be added with evidence from GitHub syntax. +- **[A missed weekly run extends discovery delay]** → Keep manual dispatch and + visible run evidence; schedule away from the top of the hour. +- **[Existing star counts and global momentum snapshots stop refreshing through + this job]** → This is the explicit cost of additions-only writes. User star + sync may still refresh connected rows; a separate budgeted popularity + snapshot design is future work if global growth recency remains required. +- **[Stored-only repositories accumulate]** → Report their count but do not + delete them until deletion semantics and user-owned references are designed. + +## Migration Plan + +1. Ship code, workflow, tests, and documentation together on the default branch. +2. The next scheduled run performs a full reconciliation with the 100-addition + gate; current live evidence indicates fewer than 100 missing IDs. +3. Inspect the run summary and Cloudflare D1 row metrics before considering a + higher manual limit. +4. Roll back by restoring the prior workflow/script revision. No schema or data + rollback is required because the change only inserts previously absent rows. diff --git a/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/proposal.md b/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/proposal.md new file mode 100644 index 0000000..495274c --- /dev/null +++ b/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/proposal.md @@ -0,0 +1,43 @@ +## Why + +The daily cursor walk spreads one logical GitHub catalog reconciliation across +roughly two weeks, so a green run cannot prove that Starboard currently contains +every public repository above the 5,000-star floor. The catalog is small enough +to reconcile completely in one weekly run while keeping D1 reads far below the +Cloudflare free allowance and avoiding rewrites of existing rows. + +## What Changes + +- Replace the daily resumable star cursor with a weekly complete GitHub Search + enumeration of repositories at or above the configured star floor. +- Read the existing eligible repository IDs from D1 once, diff both ID sets in + memory, and insert only repository IDs absent from D1. +- Reject incomplete or non-exhaustive GitHub results before any D1 write. +- Add a conservative additions circuit breaker that fails before writes when an + unexpectedly large source delta could threaten the D1 write budget. +- Keep deletion and routine metadata refresh out of scope; existing repository + rows remain untouched, and source-only additions continue through the existing + embedding and tool-enrichment steps. +- Publish reconciliation counts and budget bounds in refresh evidence and update + the operational documentation. + +## Capabilities + +### New Capabilities + +- `popular-catalog-reconciliation`: Complete, additions-only reconciliation of + GitHub's public repository catalog above the configured star floor. + +### Modified Capabilities + +None. + +## Impact + +- Scheduled workflow: `.github/workflows/seed-popular.yml` +- Operator job: `scripts/seed-popular.ts` +- Refresh evidence and regression tests under `src/lib/` and `src/__tests__/` +- Operational documentation and `PROJECT_STATUS.md` +- External systems: GitHub Repository Search, Cloudflare D1, Workers AI, and + Vectorize; no new production dependency, migration, deployment, or deletion + path is introduced. diff --git a/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/specs/popular-catalog-reconciliation/spec.md b/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/specs/popular-catalog-reconciliation/spec.md new file mode 100644 index 0000000..1da2b8c --- /dev/null +++ b/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/specs/popular-catalog-reconciliation/spec.md @@ -0,0 +1,86 @@ +## Purpose + +Keep Starboard's shared popular-repository catalog complete through a bounded, +auditable reconciliation that writes only newly discovered repositories. + +## ADDED Requirements + +### Requirement: Weekly complete source enumeration +Starboard SHALL enumerate the complete public GitHub repository set at or above +the configured star floor in one weekly reconciliation run. + +#### Scenario: Scheduled reconciliation runs +- **WHEN** the weekly catalog workflow starts +- **THEN** it enumerates every source partition needed to cover the configured + star floor before performing a database write + +#### Scenario: Source exceeds one search result window +- **WHEN** the eligible source corpus exceeds GitHub Search's per-query result + window +- **THEN** the reconciliation partitions the source into non-overlapping ranges + whose complete results fit within one response + +### Requirement: Source completeness is fail-closed +Starboard MUST reject source evidence that is incomplete, duplicated, unstable, +or inconsistent with GitHub's reported eligible count. + +#### Scenario: GitHub marks a response incomplete +- **WHEN** any source response reports incomplete results +- **THEN** the reconciliation fails before writing to D1 + +#### Scenario: Enumerated identities do not reconcile +- **WHEN** the unique repository ID count differs from the stable source count + or a repository appears in more than one non-overlapping partition +- **THEN** the reconciliation fails before writing to D1 + +### Requirement: Reconciliation is additions-only +Starboard SHALL compare the complete source identity set with all repository IDs +already stored in D1 and SHALL insert only source repositories absent from D1. + +#### Scenario: Repository already exists +- **WHEN** a source repository ID is already present in D1 +- **THEN** the weekly reconciliation does not update or rewrite that repository + +#### Scenario: Repository is new +- **WHEN** a source repository ID is absent from D1 +- **THEN** Starboard fetches current repository metadata and inserts it through + the existing catalog, snapshot, threshold-event, embedding, and enrichment + paths + +#### Scenario: Stored repository is absent upstream +- **WHEN** a stored repository ID is not present in the current source set +- **THEN** the reconciliation records the difference but does not delete data + +### Requirement: D1 budget is protected before writes +Starboard MUST apply a configured maximum-additions circuit breaker after the ID +diff and before fetching or writing addition details. The configured value MUST +NOT exceed the code-level Cloudflare budget safety limit. + +#### Scenario: Additions are within the bound +- **WHEN** the source-only ID count is at or below the configured bound +- **THEN** the reconciliation may insert the additions in bounded batches + +#### Scenario: Additions exceed the bound +- **WHEN** the source-only ID count exceeds the configured bound +- **THEN** the reconciliation fails without inserting, updating, or deleting a + repository + +#### Scenario: Configured bound exceeds the hard limit +- **WHEN** a scheduled or manual run configures an additions bound above the + code-level safety limit +- **THEN** the reconciliation fails before GitHub or D1 access + +### Requirement: Reconciliation evidence is explicit +Each run SHALL publish the source count, stored count, addition count, retained +stored-only count, configured bounds, and whether completeness verification +succeeded. + +#### Scenario: No additions are found +- **WHEN** a complete source enumeration matches the stored ID set +- **THEN** the run records a verified no-op rather than treating zero writes as + missing evidence + +#### Scenario: Additions are inserted +- **WHEN** complete source evidence produces source-only IDs within the bound +- **THEN** the run reports how many repositories were planned and actually + inserted diff --git a/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/tasks.md b/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/tasks.md new file mode 100644 index 0000000..4175ca7 --- /dev/null +++ b/openspec/changes/archive/2026-08-13-weekly-popular-repo-reconciliation/tasks.md @@ -0,0 +1,18 @@ +## 1. Reconciliation Core + +- [x] 1.1 Add a tested, creation-date-partitioned GitHub catalog enumerator that rejects incomplete, duplicated, truncated, or unstable source evidence. +- [x] 1.2 Replace the persisted cursor walk with one D1 repository-ID read, an in-memory additions diff, and a pre-write maximum-additions gate. +- [x] 1.3 Fetch and validate metadata only for source-only repositories, then insert actual additions in bounded conflict-safe batches. +- [x] 1.4 Record complete reconciliation counts and verified no-op evidence while retaining embedding coverage checks. + +## 2. Workflow and Operations + +- [x] 2.1 Change the schedule to a non-top-of-hour weekly run and configure the additions safety bound. +- [x] 2.2 Update regression coverage for weekly scheduling, additions-only writes, completeness failure, and the budget circuit breaker. +- [x] 2.3 Update canonical operational docs and public cadence copy to describe weekly full reconciliation and its D1 budget model. + +## 3. Verification + +- [x] 3.1 Run the smallest focused unit and regression tests for reconciliation and refresh evidence. +- [x] 3.2 Run typecheck, lint, documentation checks, and strict OpenSpec validation. +- [x] 3.3 Review the final diff for accidental deletes, existing-row updates, generated-file edits, secrets, or deployment changes. diff --git a/openspec/specs/popular-catalog-reconciliation/spec.md b/openspec/specs/popular-catalog-reconciliation/spec.md new file mode 100644 index 0000000..888a9b9 --- /dev/null +++ b/openspec/specs/popular-catalog-reconciliation/spec.md @@ -0,0 +1,86 @@ +# popular-catalog-reconciliation Specification + +## Purpose +Keep Starboard's shared popular-repository catalog complete through a bounded, +auditable reconciliation that writes only newly discovered repositories. +## Requirements +### Requirement: Weekly complete source enumeration +Starboard SHALL enumerate the complete public GitHub repository set at or above +the configured star floor in one weekly reconciliation run. + +#### Scenario: Scheduled reconciliation runs +- **WHEN** the weekly catalog workflow starts +- **THEN** it enumerates every source partition needed to cover the configured + star floor before performing a database write + +#### Scenario: Source exceeds one search result window +- **WHEN** the eligible source corpus exceeds GitHub Search's per-query result + window +- **THEN** the reconciliation partitions the source into non-overlapping ranges + whose complete results fit within one response + +### Requirement: Source completeness is fail-closed +Starboard MUST reject source evidence that is incomplete, duplicated, unstable, +or inconsistent with GitHub's reported eligible count. + +#### Scenario: GitHub marks a response incomplete +- **WHEN** any source response reports incomplete results +- **THEN** the reconciliation fails before writing to D1 + +#### Scenario: Enumerated identities do not reconcile +- **WHEN** the unique repository ID count differs from the stable source count + or a repository appears in more than one non-overlapping partition +- **THEN** the reconciliation fails before writing to D1 + +### Requirement: Reconciliation is additions-only +Starboard SHALL compare the complete source identity set with all repository IDs +already stored in D1 and SHALL insert only source repositories absent from D1. + +#### Scenario: Repository already exists +- **WHEN** a source repository ID is already present in D1 +- **THEN** the weekly reconciliation does not update or rewrite that repository + +#### Scenario: Repository is new +- **WHEN** a source repository ID is absent from D1 +- **THEN** Starboard fetches current repository metadata and inserts it through + the existing catalog, snapshot, threshold-event, embedding, and enrichment + paths + +#### Scenario: Stored repository is absent upstream +- **WHEN** a stored repository ID is not present in the current source set +- **THEN** the reconciliation records the difference but does not delete data + +### Requirement: D1 budget is protected before writes +Starboard MUST apply a configured maximum-additions circuit breaker after the ID +diff and before fetching or writing addition details. The configured value MUST +NOT exceed the code-level Cloudflare budget safety limit. + +#### Scenario: Additions are within the bound +- **WHEN** the source-only ID count is at or below the configured bound +- **THEN** the reconciliation may insert the additions in bounded batches + +#### Scenario: Additions exceed the bound +- **WHEN** the source-only ID count exceeds the configured bound +- **THEN** the reconciliation fails without inserting, updating, or deleting a + repository + +#### Scenario: Configured bound exceeds the hard limit +- **WHEN** a scheduled or manual run configures an additions bound above the + code-level safety limit +- **THEN** the reconciliation fails before GitHub or D1 access + +### Requirement: Reconciliation evidence is explicit +Each run SHALL publish the source count, stored count, addition count, retained +stored-only count, configured bounds, and whether completeness verification +succeeded. + +#### Scenario: No additions are found +- **WHEN** a complete source enumeration matches the stored ID set +- **THEN** the run records a verified no-op rather than treating zero writes as + missing evidence + +#### Scenario: Additions are inserted +- **WHEN** complete source evidence produces source-only IDs within the bound +- **THEN** the run reports how many repositories were planned and actually + inserted + diff --git a/public/catalog-updates.md b/public/catalog-updates.md index 2fea068..ed3ed22 100644 --- a/public/catalog-updates.md +++ b/public/catalog-updates.md @@ -11,7 +11,7 @@ Recently cataloged popular repositories in the shared Discover corpus (typically ## Refresh cadence -Catalogue growth comes from the bounded daily `seed-popular` GitHub Action; manual dispatch remains available for operator checks. Personal stars update when a user syncs and remain separate from this public popular pool. +Catalogue growth comes from a complete weekly, additions-only `seed-popular` GitHub Action; manual dispatch remains available for operator checks. Personal stars update when a user syncs and remain separate from this public popular pool. ## Related diff --git a/scripts/check-code-health.mjs b/scripts/check-code-health.mjs index e8c1921..512f4a6 100644 --- a/scripts/check-code-health.mjs +++ b/scripts/check-code-health.mjs @@ -28,11 +28,11 @@ const baselines = { unlisted: 0, unresolved: 0, }, - complexity: { violations: 35, maxCcn: 56, maxLength: 430, maxParams: 17 }, + complexity: { violations: 35, maxCcn: 56, maxLength: 397, maxParams: 17 }, duplication: { - clones: 35, - duplicatedLines: 542, - percentage: 2.7770661474611877, + clones: 34, + duplicatedLines: 532, + percentage: 2.698452954603094, }, suppressions: 8, dependencies: { critical: 0, highIds: 0, highFindings: 0 }, diff --git a/scripts/seed-popular.ts b/scripts/seed-popular.ts index 0aa520a..5385a83 100644 --- a/scripts/seed-popular.ts +++ b/scripts/seed-popular.ts @@ -1,18 +1,21 @@ /** - * Daily seed/refresh of popular GitHub repos into our `repos` + `repo_embeddings` tables. + * Weekly additions-only reconciliation of popular GitHub repositories. * * Each run: - * 1. Walk a bounded slice of GH search across `stars >= MIN_STARS_FLOOR`. - * Only changed repos are updated, so unchanged rows do not fire the FTS - * maintenance trigger. Cursor resumes between runs and resets after a - * complete corpus pass. - * 2. In direct/local mode, embed up to SEED_DAILY_LIMIT pending repos. The + * 1. Enumerate the complete GitHub `stars >= MIN_STARS_FLOOR` identity set + * through non-overlapping creation-date partitions that each fit in one + * Search response. + * 2. Read every stored D1 repository ID once, diff both sets in memory, and + * fail before writes if additions exceed SEED_MAX_ADDITIONS. + * 3. Fetch details and insert only genuinely new repositories. Existing rows + * are never updated and stored-only rows are never deleted. + * 4. In direct/local mode, embed up to SEED_DAILY_LIMIT pending repos. The * GitHub workflow delegates this step to the deployed Worker so Workers AI, * Vectorize, and D1 are reached through native bindings. * - * GH metadata walking is free under quota (~120 calls / 5000-per-hour). Bottleneck is - * the daily embed budget (CF Workers AI). After ~12 catch-up days the pool is fully - * embedded; subsequent runs pick up only newly-eligible repos and metadata drift. + * GitHub Search calls are paced below the authenticated 30 requests/minute + * bucket. A normal reconciliation makes roughly 250 calls, below the workflow + * GITHUB_TOKEN allowance of 1000 requests/hour per repository. * * Required env: * CLOUDFLARE_ACCOUNT_ID @@ -23,9 +26,9 @@ * GITHUB_TOKEN — fine-grained PAT, public_repo:read * Optional env: * SEED_DAILY_LIMIT — embeddings per run, default 1000 - * SEED_METADATA_PAGE_LIMIT — GitHub search pages per run, default 10 (hard cap 25) + * SEED_MAX_ADDITIONS — abort-before-write bound, default 100 + * SEED_MIN_SOURCE_REPOS — reject suspiciously small source sets, default 5000 * MIN_STARS_FLOOR — minimum stars to seed, default 5000 - * STAR_THRESHOLDS — comma-separated digest thresholds, default 5000,10000,20000,50000,100000 * SEED_EMBED_MODE — `worker` delegates embeddings to the bound Worker endpoint */ @@ -34,23 +37,33 @@ import { createD1RestClientFromEnv } from '../src/db/rest-client'; import { isRetryableDbError } from '../src/lib/db-retry'; import { buildRepoEmbeddingText, generateEmbeddings, textHash } from '../src/lib/embeddings'; +import { + enumeratePopularCatalog, + GITHUB_SEARCH_PAGE_SIZE, + planCatalogReconciliation, + type CatalogSearchResult, +} from '../src/lib/popular-catalog-reconciliation'; import { recordStep } from '../src/lib/refresh-manifest'; import { createVectorizeRestWriterFromEnv } from '../src/lib/repo-vectors-rest'; const DAILY_LIMIT = parseInt(process.env.SEED_DAILY_LIMIT || '1000', 10); -const REQUESTED_METADATA_PAGE_LIMIT = parseInt(process.env.SEED_METADATA_PAGE_LIMIT || '10', 10); -const METADATA_PAGE_LIMIT = Math.min(Math.max(REQUESTED_METADATA_PAGE_LIMIT || 0, 1), 25); const MIN_STARS_FLOOR = parseInt(process.env.MIN_STARS_FLOOR || '5000', 10); -const STAR_THRESHOLDS = (process.env.STAR_THRESHOLDS || '5000,10000,20000,50000,100000') - .split(',') - .map((value) => parseInt(value.trim(), 10)) - .filter((value) => Number.isFinite(value) && value >= MIN_STARS_FLOOR) - .sort((a, b) => a - b); -const PER_PAGE = 100; -const MAX_PAGES_PER_BUCKET = 10; // GH search caps at 1000 results +const MAX_ADDITIONS_HARD_LIMIT = 100; +const MAX_ADDITIONS = parseInt(process.env.SEED_MAX_ADDITIONS || '100', 10); +const MIN_SOURCE_REPOS = parseInt(process.env.SEED_MIN_SOURCE_REPOS || '5000', 10); const BATCH_SIZE = 50; const DB_MAX_ATTEMPTS = 4; const DB_RETRY_BASE_MS = 1_000; +const GITHUB_SEARCH_DELAY_MS = 2_100; + +if (!Number.isInteger(MAX_ADDITIONS) || MAX_ADDITIONS < 0) { + throw new Error(`SEED_MAX_ADDITIONS must be a non-negative integer; received ${MAX_ADDITIONS}`); +} +if (MAX_ADDITIONS > MAX_ADDITIONS_HARD_LIMIT) { + throw new Error( + `SEED_MAX_ADDITIONS ${MAX_ADDITIONS} exceeds hard safety limit ${MAX_ADDITIONS_HARD_LIMIT}` + ); +} interface GhRepo { id: number; @@ -69,24 +82,10 @@ interface GhRepo { interface GhSearchResponse { total_count: number; + incomplete_results: boolean; items: GhRepo[]; } -interface StoredRepo { - id: number; - name: string; - full_name: string; - owner_login: string; - owner_avatar: string; - html_url: string; - description: string | null; - language: string | null; - stargazers_count: number; - archived: number; - topics: string; - repo_updated_at: string | null; -} - async function withDbRetry(label: string, fn: () => Promise): Promise { for (let attempt = 1; ; attempt++) { try { @@ -110,100 +109,31 @@ function batchDb(db: Client, stmts: InStatement[]) { return withDbRetry('batch', () => db.batch(stmts)); } -async function loadStoredRepos(db: Client, repoIds: number[]): Promise> { - if (repoIds.length === 0) return new Map(); - - const result = await executeDb(db, { - sql: `SELECT id, - name, - full_name, - owner_login, - owner_avatar, - html_url, - description, - language, - stargazers_count, - archived, - topics, - repo_updated_at - FROM repos - WHERE id IN (SELECT CAST(value AS INTEGER) FROM json_each(?))`, - args: [JSON.stringify(repoIds)], - }); - - return new Map( - result.rows.map((row) => [ - row.id as number, - { - id: row.id as number, - name: row.name as string, - full_name: row.full_name as string, - owner_login: row.owner_login as string, - owner_avatar: row.owner_avatar as string, - html_url: row.html_url as string, - description: row.description as string | null, - language: row.language as string | null, - stargazers_count: row.stargazers_count as number, - archived: row.archived as number, - topics: row.topics as string, - repo_updated_at: row.repo_updated_at as string | null, - }, - ]) - ); +async function loadStoredRepoIds(db: Client): Promise> { + const result = await executeDb(db, 'SELECT id FROM repos'); + return new Set(result.rows.map((row) => row.id as number)); } -function storedRepoDiffers(stored: StoredRepo | undefined, repo: GhRepo): boolean { - if (!stored) return true; - - return ( - stored.name !== repo.name || - stored.full_name !== repo.full_name || - stored.owner_login !== repo.owner.login || - stored.owner_avatar !== repo.owner.avatar_url || - stored.html_url !== repo.html_url || - stored.description !== repo.description || - stored.language !== repo.language || - stored.stargazers_count !== repo.stargazers_count || - stored.archived !== (repo.archived ? 1 : 0) || - stored.topics !== JSON.stringify(repo.topics ?? []) || - stored.repo_updated_at !== repo.updated_at - ); -} - -function buildThresholdEventStatements( - repos: GhRepo[], - previousStarCounts: Map -): InStatement[] { - const stmts: InStatement[] = []; - - for (const repo of repos) { - const previousStars = previousStarCounts.get(repo.id); - - for (const threshold of STAR_THRESHOLDS) { - const crossed = - previousStars === undefined - ? threshold === MIN_STARS_FLOOR && repo.stargazers_count >= threshold - : previousStars < threshold && repo.stargazers_count >= threshold; +let lastGitHubSearchAt = 0; - if (!crossed) continue; - - stmts.push({ - sql: `INSERT OR IGNORE INTO repo_threshold_events - (repo_id, threshold, previous_stars, current_stars) - VALUES (?, ?, ?, ?)`, - args: [repo.id, threshold, previousStars ?? null, repo.stargazers_count], - }); - } +async function paceGitHubSearch(): Promise { + const sinceLastSearch = Date.now() - lastGitHubSearchAt; + if (lastGitHubSearchAt > 0 && sinceLastSearch < GITHUB_SEARCH_DELAY_MS) { + await new Promise((resolve) => setTimeout(resolve, GITHUB_SEARCH_DELAY_MS - sinceLastSearch)); } - - return stmts; + lastGitHubSearchAt = Date.now(); } -async function ghSearch(q: string, page: number, token: string): Promise { - const url = `https://api.github.com/search/repositories?q=${encodeURIComponent(q)}&sort=stars&order=desc&per_page=${PER_PAGE}&page=${page}`; +async function githubJson( + url: string, + token: string, + label: string, + beforeAttempt?: () => Promise +): Promise { const maxAttempts = 4; let lastError: unknown; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + await beforeAttempt?.(); let res: Response; try { res = await fetch(url, { @@ -217,106 +147,109 @@ async function ghSearch(q: string, page: number, token: string): Promise setTimeout(r, waitMs)); continue; } if (res.status >= 500 && attempt < maxAttempts) { const waitMs = 2 ** (attempt - 1) * 1000; - console.warn(`GitHub search returned ${res.status}. Retrying in ${waitMs / 1000}s...`); + console.warn(`${label} returned ${res.status}. Retrying in ${waitMs / 1000}s...`); await new Promise((r) => setTimeout(r, waitMs)); continue; } - if (res.status === 403 || res.status === 429) { + if ((res.status === 403 || res.status === 429) && attempt < maxAttempts) { + const retryAfter = res.headers.get('retry-after'); const reset = res.headers.get('x-ratelimit-reset'); - const waitMs = reset ? parseInt(reset, 10) * 1000 - Date.now() : 60_000; - console.warn(`Rate limited. Sleeping ${Math.round(waitMs / 1000)}s...`); - await new Promise((r) => setTimeout(r, Math.max(waitMs, 1000))); - return ghSearch(q, page, token); + const requestedWaitMs = retryAfter + ? parseInt(retryAfter, 10) * 1000 + : reset + ? parseInt(reset, 10) * 1000 - Date.now() + : 60_000; + const waitMs = Math.min(Math.max(requestedWaitMs, 1_000), 60_000); + console.warn(`${label} rate limited. Sleeping ${Math.round(waitMs / 1000)}s...`); + await new Promise((r) => setTimeout(r, waitMs)); + continue; } if (!res.ok) { - throw new Error(`GH search failed: ${res.status} ${await res.text()}`); + throw new Error(`${label} failed: ${res.status} ${await res.text()}`); } return res.json(); } - throw new Error(`GH search request failed after ${maxAttempts} attempts: ${String(lastError)}`); + throw new Error(`${label} failed after ${maxAttempts} attempts: ${String(lastError)}`); +} + +async function ghSearch(q: string, token: string): Promise { + const url = `https://api.github.com/search/repositories?q=${encodeURIComponent(q)}&sort=stars&order=desc&per_page=${GITHUB_SEARCH_PAGE_SIZE}&page=1`; + const result = await githubJson(url, token, 'GitHub search', paceGitHubSearch); + return { + totalCount: result.total_count, + incomplete: result.incomplete_results, + repos: result.items.map((repo) => ({ id: repo.id, fullName: repo.full_name })), + }; +} + +function ghRepo(fullName: string, token: string): Promise { + const url = `https://api.github.com/repos/${encodeURIComponent(fullName).replace('%2F', '/')}`; + return githubJson(url, token, `GitHub repository ${fullName}`); } -async function upsertRepos(db: Client, repos: GhRepo[]): Promise { +async function insertNewRepos(db: Client, repos: GhRepo[]): Promise { if (repos.length === 0) return []; - const storedRepos = await loadStoredRepos( - db, - repos.map((repo) => repo.id) - ); - const changedRepos = repos.filter((repo) => storedRepoDiffers(storedRepos.get(repo.id), repo)); - if (changedRepos.length === 0) return []; - - const previousStarCounts = new Map( - changedRepos.flatMap((repo) => { - const stored = storedRepos.get(repo.id); - return stored ? [[repo.id, stored.stargazers_count] as const] : []; - }) - ); - const stmts: InStatement[] = changedRepos.map((r) => ({ - sql: `INSERT INTO repos (id, name, full_name, owner_login, owner_avatar, html_url, - description, language, stargazers_count, archived, topics, repo_created_at, repo_updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(id) DO UPDATE SET - name = excluded.name, - full_name = excluded.full_name, - owner_login = excluded.owner_login, - owner_avatar = excluded.owner_avatar, - html_url = excluded.html_url, - description = excluded.description, - language = excluded.language, - stargazers_count = excluded.stargazers_count, - archived = excluded.archived, - topics = excluded.topics, - repo_updated_at = excluded.repo_updated_at - WHERE repos.name IS NOT excluded.name - OR repos.full_name IS NOT excluded.full_name - OR repos.owner_login IS NOT excluded.owner_login - OR repos.owner_avatar IS NOT excluded.owner_avatar - OR repos.html_url IS NOT excluded.html_url - OR repos.description IS NOT excluded.description - OR repos.language IS NOT excluded.language - OR repos.stargazers_count IS NOT excluded.stargazers_count - OR repos.archived IS NOT excluded.archived - OR repos.topics IS NOT excluded.topics - OR repos.repo_updated_at IS NOT excluded.repo_updated_at`, - args: [ - r.id, - r.name, - r.full_name, - r.owner.login, - r.owner.avatar_url, - r.html_url, - r.description, - r.language, - r.stargazers_count, - r.archived ? 1 : 0, - JSON.stringify(r.topics ?? []), - r.created_at, - r.updated_at, - ], - })); - const snapshotStmts: InStatement[] = changedRepos - .filter( - (repo) => - !storedRepos.has(repo.id) || - storedRepos.get(repo.id)!.stargazers_count !== repo.stargazers_count - ) - .map((repo) => ({ - sql: `INSERT OR IGNORE INTO repo_star_snapshots (repo_id, stargazers_count) - VALUES (?, ?)`, - args: [repo.id, repo.stargazers_count], - })); - const thresholdEventStmts = buildThresholdEventStatements(changedRepos, previousStarCounts); + const insertedIds: number[] = []; + + for (let offset = 0; offset < repos.length; offset += BATCH_SIZE) { + const batch = repos.slice(offset, offset + BATCH_SIZE); + const insertResults = await batchDb( + db, + batch.map((repo) => ({ + sql: `INSERT OR IGNORE INTO repos + (id, name, full_name, owner_login, owner_avatar, html_url, + description, language, stargazers_count, archived, topics, + repo_created_at, repo_updated_at, cataloged_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`, + args: [ + repo.id, + repo.name, + repo.full_name, + repo.owner.login, + repo.owner.avatar_url, + repo.html_url, + repo.description, + repo.language, + repo.stargazers_count, + repo.archived ? 1 : 0, + JSON.stringify(repo.topics ?? []), + repo.created_at, + repo.updated_at, + ], + })) + ); + const inserted = batch.filter((_, index) => (insertResults[index]?.rowsAffected ?? 0) > 0); + insertedIds.push(...inserted.map((repo) => repo.id)); + + if (inserted.length > 0) { + await batchDb( + db, + inserted.flatMap((repo): InStatement[] => [ + { + sql: `INSERT OR IGNORE INTO repo_star_snapshots (repo_id, stargazers_count) + VALUES (?, ?)`, + args: [repo.id, repo.stargazers_count], + }, + { + sql: `INSERT OR IGNORE INTO repo_threshold_events + (repo_id, threshold, previous_stars, current_stars) + VALUES (?, ?, NULL, ?)`, + args: [repo.id, MIN_STARS_FLOOR, repo.stargazers_count], + }, + ]) + ); + } + } - await batchDb(db, [...stmts, ...snapshotStmts, ...thresholdEventStmts]); - return changedRepos.map((r) => r.id); + return insertedIds; } async function embedPending(db: Client, limit: number): Promise { @@ -391,91 +324,13 @@ function isEmbeddingAuthError(err: unknown): boolean { return /Embedding API error 401|invalid_api_key|Unauthorized/i.test(err.message); } -async function loadCursor(db: Client) { - const r = await executeDb(db, 'SELECT * FROM seed_cursor WHERE id = 1'); - if (r.rows.length === 0) { - await executeDb(db, 'INSERT INTO seed_cursor (id) VALUES (1)'); - return { next_max_stars: 999999999, next_page: 1 }; - } - return { - next_max_stars: r.rows[0]!.next_max_stars as number, - next_page: r.rows[0]!.next_page as number, - }; -} - -async function saveCursor(db: Client, next_max_stars: number, next_page: number) { - await executeDb(db, { - sql: `UPDATE seed_cursor - SET next_max_stars = ?, next_page = ?, updated_at = datetime('now') - WHERE id = 1`, - args: [next_max_stars, next_page], - }); -} - -/** - * Walk GH search from `cursor.next_max_stars` down to MIN_STARS_FLOOR. Persists cursor - * between pages so a crash mid-run resumes cleanly. When the walk completes (we've gone - * below the floor), reset cursor for the next pass — that's how new repos crossing - * threshold get discovered on subsequent runs. - */ -async function walkAndUpsert(db: Client, ghToken: string): Promise { - const cursor = await loadCursor(db); - console.info(`[walk] resume cursor: max_stars=${cursor.next_max_stars} page=${cursor.next_page}`); - - let max_stars = cursor.next_max_stars; - let page = cursor.next_page; - let lowestSeenInBucket = max_stars; - let upsertedThisRun = 0; - let pagesProcessed = 0; - - while (max_stars >= MIN_STARS_FLOOR && pagesProcessed < METADATA_PAGE_LIMIT) { - const q = `stars:${MIN_STARS_FLOOR}..${max_stars}`; - console.info(`[walk] q="${q}" page=${page}`); - const result = await ghSearch(q, page, ghToken); - pagesProcessed += 1; - - if (result.items.length === 0) { - if (page === 1) break; - const newMax = lowestSeenInBucket - 1; - if (newMax < MIN_STARS_FLOOR || newMax === max_stars) break; - max_stars = newMax; - page = 1; - lowestSeenInBucket = newMax; - await saveCursor(db, max_stars, page); - continue; - } - - upsertedThisRun += (await upsertRepos(db, result.items)).length; - const minStarsInPage = result.items[result.items.length - 1].stargazers_count; - lowestSeenInBucket = Math.min(lowestSeenInBucket, minStarsInPage); - page++; - - if (page > MAX_PAGES_PER_BUCKET) { - max_stars = lowestSeenInBucket - 1; - page = 1; - lowestSeenInBucket = max_stars; - } - - await saveCursor(db, max_stars, page); - // GH search caps authenticated users at 30 req/min (1 per 2.0s). - // 2100ms keeps us safely under without idling too much. - await new Promise((r) => setTimeout(r, 2100)); - } - - if (pagesProcessed >= METADATA_PAGE_LIMIT && max_stars >= MIN_STARS_FLOOR) { - console.info( - `[walk] paused after bounded ${pagesProcessed}-page run. upserted ${upsertedThisRun} repo rows; cursor preserved at max_stars=${max_stars} page=${page}.` - ); - return upsertedThisRun; - } - - // Walk complete. Reset cursor so the next run rediscovers from the top — - // catches new ≥5k repos and refreshes star counts on existing rows. - await saveCursor(db, 999999999, 1); - console.info( - `[walk] complete after ${pagesProcessed} pages. upserted ${upsertedThisRun} repo rows. cursor reset.` - ); - return upsertedThisRun; +interface ReconciliationResult { + sourceCount: number; + storedCount: number; + plannedAdditions: number; + insertedAdditions: number; + storedOnlyCount: number; + leafPartitions: number; } async function main() { @@ -484,25 +339,51 @@ async function main() { const db = createD1RestClientFromEnv(); - const upserted = await walkAndUpsert(db, ghToken); + let reconciliation: ReconciliationResult; + try { + reconciliation = await reconcileCatalog(db, ghToken); + } catch (error) { + const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error); + recordStep({ + step: 'seed_reconciliation', + sourceWatermark: null, + bounds: { + min_stars_floor: MIN_STARS_FLOOR, + min_source_repos: MIN_SOURCE_REPOS, + max_additions: MAX_ADDITIONS, + }, + timeoutS: 60 * 60, + idempotency: + 'Complete source and stored ID sets are diffed before INSERT OR IGNORE; existing rows are never updated and stored-only rows are never deleted', + outputCount: 0, + expectedMinOutput: 0, + error: message, + }); + throw error; + } - // A zero-row walk is a verified no-op only because walkAndUpsert returns - // after successfully querying GitHub and preserving/resetting its cursor. recordStep({ - step: 'seed_walk', - sourceWatermark: `cursor_after_walk`, + step: 'seed_reconciliation', + sourceWatermark: `github_unique_ids:${reconciliation.sourceCount}`, bounds: { - metadata_page_limit: METADATA_PAGE_LIMIT, min_stars_floor: MIN_STARS_FLOOR, - max_pages_per_bucket: MAX_PAGES_PER_BUCKET, + min_source_repos: MIN_SOURCE_REPOS, + max_additions: MAX_ADDITIONS, + source_count: reconciliation.sourceCount, + stored_count: reconciliation.storedCount, + planned_additions: reconciliation.plannedAdditions, + stored_only_count: reconciliation.storedOnlyCount, + leaf_partitions: reconciliation.leafPartitions, }, timeoutS: 60 * 60, idempotency: - 'Stored-row comparison skips unchanged repos; changed repos use INSERT … ON CONFLICT(id) DO UPDATE; snapshots are written only for star-count changes', - outputCount: upserted, + 'Complete source and stored ID sets are diffed before INSERT OR IGNORE; existing rows are never updated and stored-only rows are never deleted', + outputCount: reconciliation.insertedAdditions, expectedMinOutput: 0, verifiedNoopReason: - upserted === 0 ? 'GitHub search walk completed and cursor state was preserved' : undefined, + reconciliation.insertedAdditions === 0 + ? `Complete GitHub catalog reconciled at ${reconciliation.sourceCount} unique IDs with no additions` + : undefined, }); console.info(`[embed] generating up to ${DAILY_LIMIT} embeddings`); @@ -596,6 +477,55 @@ async function main() { } } +const reconcileCatalog = async (db: Client, ghToken: string): Promise => { + console.info(`[reconcile] enumerating complete GitHub catalog at ≥${MIN_STARS_FLOOR} stars`); + const source = await enumeratePopularCatalog((query) => ghSearch(query, ghToken), { + minStars: MIN_STARS_FLOOR, + minExpectedRepos: MIN_SOURCE_REPOS, + }); + const storedIds = await loadStoredRepoIds(db); + const plan = planCatalogReconciliation(source.repos, storedIds, MAX_ADDITIONS); + + console.info( + `[reconcile] source=${source.sourceCount} stored=${storedIds.size} ` + + `additions=${plan.additions.length} stored_only=${plan.storedOnlyCount} ` + + `leaf_partitions=${source.leafPartitions}` + ); + + // Resolve every source-only repository before the first write. A removed, + // renamed, or newly ineligible repository therefore fails closed instead of + // leaving a partial reconciliation. + const additionDetails: GhRepo[] = []; + for (const identity of plan.additions) { + const repo = await ghRepo(identity.fullName, ghToken); + if (repo.id !== identity.id) { + throw new Error( + `GitHub repository identity changed for ${identity.fullName}: ${identity.id} -> ${repo.id}` + ); + } + if (repo.stargazers_count < MIN_STARS_FLOOR) { + throw new Error( + `GitHub repository ${repo.full_name} fell below ${MIN_STARS_FLOOR} stars during reconciliation` + ); + } + additionDetails.push(repo); + } + + const insertedIds = await insertNewRepos(db, additionDetails); + console.info( + `[reconcile] inserted ${insertedIds.length}/${plan.additions.length} planned additions` + ); + + return { + sourceCount: source.sourceCount, + storedCount: storedIds.size, + plannedAdditions: plan.additions.length, + insertedAdditions: insertedIds.length, + storedOnlyCount: plan.storedOnlyCount, + leafPartitions: source.leafPartitions, + }; +}; + main().catch((err) => { console.error('Seed run failed:', err); process.exit(1); diff --git a/src/__tests__/d1-query-limits.test.ts b/src/__tests__/d1-query-limits.test.ts index 629d930..b6c4420 100644 --- a/src/__tests__/d1-query-limits.test.ts +++ b/src/__tests__/d1-query-limits.test.ts @@ -6,7 +6,6 @@ const files = [ 'src/app/api/discover/route.ts', 'src/app/api/stars/route.ts', 'src/app/api/stars/sync/route.ts', - 'scripts/seed-popular.ts', ]; describe('D1 bound-parameter regression guards', () => { diff --git a/src/__tests__/db-regression.test.ts b/src/__tests__/db-regression.test.ts index 5ca91de..2fa40f9 100644 --- a/src/__tests__/db-regression.test.ts +++ b/src/__tests__/db-regression.test.ts @@ -42,22 +42,25 @@ describe('db row-read regression guards', () => { expect(schemaSql).not.toMatch(/INSERT INTO (\w+)\(\1\)\s*VALUES\('rebuild'\)/); }); - it('seed-popular skips unchanged repo updates and snapshots', () => { - expect(seedPopularTs).toContain('storedRepoDiffers'); - expect(seedPopularTs).toMatch( - /const changedRepos = repos\.filter\(\(repo\) => storedRepoDiffers\(storedRepos\.get\(repo\.id\), repo\)\)/ - ); - expect(seedPopularTs).toContain('WHERE repos.name IS NOT excluded.name'); - expect(seedPopularTs).toMatch(/const snapshotStmts:[\s\S]*changedRepos[\s\S]*stargazers_count/); + it('seed-popular reads IDs once and writes additions without updating existing repos', () => { + expect(seedPopularTs).toContain("executeDb(db, 'SELECT id FROM repos')"); + expect(seedPopularTs).toContain('planCatalogReconciliation'); + expect(seedPopularTs).toContain('INSERT OR IGNORE INTO repos'); + expect(seedPopularTs).not.toContain('stargazers_count = excluded.stargazers_count'); + expect(seedPopularTs).not.toContain('DELETE FROM repos'); + expect(seedPopularTs).not.toContain('UPDATE seed_cursor'); }); - it('daily seed runs stay scheduled and operationally bounded', () => { - expect(seedWorkflow).toContain("cron: '0 3 * * *'"); + it('weekly full reconciliation stays scheduled and operationally bounded', () => { + expect(seedWorkflow).toContain("cron: '17 3 * * 0'"); expect(seedWorkflow).toContain('timeout-minutes: 60'); expect(seedWorkflow).toContain('cancel-in-progress: false'); - expect(seedWorkflow).toContain("SEED_METADATA_PAGE_LIMIT: '10'"); - expect(seedPopularTs).toContain("process.env.SEED_METADATA_PAGE_LIMIT || '10'"); - expect(seedPopularTs).toContain(', 25);'); + expect(seedWorkflow).toContain("github.event.inputs.max_additions || '100'"); + expect(seedWorkflow).toContain("SEED_MIN_SOURCE_REPOS: '5000'"); + expect(seedPopularTs).toContain("process.env.SEED_MAX_ADDITIONS || '100'"); + expect(seedPopularTs).toContain('const MAX_ADDITIONS_HARD_LIMIT = 100'); + expect(seedPopularTs).toContain('exceeds hard safety limit'); + expect(seedPopularTs).toContain("process.env.SEED_MIN_SOURCE_REPOS || '5000'"); }); it('no Turso-backed workflow remains', () => { diff --git a/src/__tests__/popular-catalog-reconciliation.test.ts b/src/__tests__/popular-catalog-reconciliation.test.ts new file mode 100644 index 0000000..0374bff --- /dev/null +++ b/src/__tests__/popular-catalog-reconciliation.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, it } from 'vitest'; + +import { + enumeratePopularCatalog, + planCatalogReconciliation, + type CatalogRepoIdentity, + type CatalogSearch, +} from '@/lib/popular-catalog-reconciliation'; + +interface FakeRepo extends CatalogRepoIdentity { + createdAt: string; +} + +function fakeSearch(repos: FakeRepo[], options: { incompleteQuery?: string } = {}): CatalogSearch { + return async (query) => { + const range = /created:(\d{4}-\d{2}-\d{2})\.\.(\d{4}-\d{2}-\d{2})/.exec(query); + const matches = range + ? repos.filter((repo) => repo.createdAt >= range[1]! && repo.createdAt <= range[2]!) + : repos; + return { + totalCount: matches.length, + incomplete: options.incompleteQuery === query, + repos: matches.slice(0, 100), + }; + }; +} + +describe('enumeratePopularCatalog', () => { + it('splits immutable creation-date ranges until every repository fits one response', async () => { + const repos: FakeRepo[] = Array.from({ length: 101 }, (_, index) => ({ + id: index + 1, + fullName: `owner/repo-${index + 1}`, + createdAt: index < 50 ? '2025-01-01' : '2025-01-03', + })); + + const result = await enumeratePopularCatalog(fakeSearch(repos), { + minStars: 5000, + minExpectedRepos: 1, + startDate: '2025-01-01', + endDate: '2025-01-03', + }); + + expect(result.sourceCount).toBe(101); + expect(result.repos.size).toBe(101); + expect(result.leafPartitions).toBeGreaterThan(1); + }); + + it('fails before reconciliation when GitHub marks a partition incomplete', async () => { + const query = 'stars:>=5000 created:2025-01-01..2025-01-01'; + await expect( + enumeratePopularCatalog( + fakeSearch([{ id: 1, fullName: 'owner/repo', createdAt: '2025-01-01' }], { + incompleteQuery: query, + }), + { + minStars: 5000, + minExpectedRepos: 1, + startDate: '2025-01-01', + endDate: '2025-01-01', + } + ) + ).rejects.toThrow('incomplete catalog evidence'); + }); + + it('fails when the root source count changes during enumeration', async () => { + let rootCalls = 0; + const search: CatalogSearch = async (query) => { + if (!query.includes('created:')) { + rootCalls += 1; + return { + totalCount: rootCalls === 1 ? 1 : 2, + incomplete: false, + repos: [{ id: 1, fullName: 'owner/repo' }], + }; + } + return { + totalCount: 1, + incomplete: false, + repos: [{ id: 1, fullName: 'owner/repo' }], + }; + }; + + await expect( + enumeratePopularCatalog(search, { + minStars: 5000, + minExpectedRepos: 1, + startDate: '2025-01-01', + endDate: '2025-01-01', + }) + ).rejects.toThrow('source count changed'); + }); + + it('fails when a supposedly complete leaf is truncated', async () => { + const search: CatalogSearch = async (query) => ({ + totalCount: 1, + incomplete: false, + repos: query.includes('created:') ? [] : [{ id: 1, fullName: 'owner/repo' }], + }); + + await expect( + enumeratePopularCatalog(search, { + minStars: 5000, + minExpectedRepos: 1, + startDate: '2025-01-01', + endDate: '2025-01-01', + }) + ).rejects.toThrow('returned 0/1 repositories'); + }); + + it('fails when GitHub repeats an identity in the complete source set', async () => { + const search: CatalogSearch = async () => ({ + totalCount: 2, + incomplete: false, + repos: [ + { id: 1, fullName: 'owner/repo' }, + { id: 1, fullName: 'owner/repo' }, + ], + }); + + await expect( + enumeratePopularCatalog(search, { + minStars: 5000, + minExpectedRepos: 1, + startDate: '2025-01-01', + endDate: '2025-01-01', + }) + ).rejects.toThrow('duplicate repository ID 1'); + }); + + it('fails when one creation day exceeds a single response', async () => { + const repos: FakeRepo[] = Array.from({ length: 101 }, (_, index) => ({ + id: index + 1, + fullName: `owner/repo-${index + 1}`, + createdAt: '2025-01-01', + })); + + await expect( + enumeratePopularCatalog(fakeSearch(repos), { + minStars: 5000, + minExpectedRepos: 1, + startDate: '2025-01-01', + endDate: '2025-01-01', + }) + ).rejects.toThrow('one-response enumeration cannot prove completeness'); + }); +}); + +describe('planCatalogReconciliation', () => { + const source = new Map([ + [1, { id: 1, fullName: 'owner/one' }], + [2, { id: 2, fullName: 'owner/two' }], + ]); + + it('returns only source IDs absent from D1 and counts stored-only IDs', () => { + const plan = planCatalogReconciliation(source, new Set([1, 3]), 1); + expect(plan.additions).toEqual([{ id: 2, fullName: 'owner/two' }]); + expect(plan.storedOnlyCount).toBe(1); + }); + + it('rejects an unsafe delta before writes can start', () => { + expect(() => planCatalogReconciliation(source, new Set(), 1)).toThrow( + 'Catalog additions 2 exceed safety bound 1; no writes performed' + ); + }); +}); diff --git a/src/lib/catalog-updates.ts b/src/lib/catalog-updates.ts index 7b3eedb..b4bad65 100644 --- a/src/lib/catalog-updates.ts +++ b/src/lib/catalog-updates.ts @@ -131,7 +131,7 @@ export async function loadCatalogUpdates( newestCatalogedAt: (newestResult.rows[0]?.newest as string | null) ?? null, changesReturned: changes.length, refreshCadence: - 'Daily bounded seed-popular GitHub Action with manual dispatch; user star sync is on-demand.', + 'Weekly complete additions-only seed-popular GitHub Action with manual dispatch; user star sync is on-demand.', }, }; } diff --git a/src/lib/popular-catalog-reconciliation.ts b/src/lib/popular-catalog-reconciliation.ts new file mode 100644 index 0000000..28e353d --- /dev/null +++ b/src/lib/popular-catalog-reconciliation.ts @@ -0,0 +1,176 @@ +const DAY_MS = 24 * 60 * 60 * 1000; + +export const GITHUB_SEARCH_PAGE_SIZE = 100; +const GITHUB_REPOSITORY_EPOCH = '2007-01-01'; + +export interface CatalogRepoIdentity { + id: number; + fullName: string; +} + +export interface CatalogSearchResult { + totalCount: number; + incomplete: boolean; + repos: CatalogRepoIdentity[]; +} + +export type CatalogSearch = (query: string) => Promise; + +export interface CatalogEnumeration { + repos: Map; + sourceCount: number; + leafPartitions: number; +} + +export interface CatalogReconciliationPlan { + additions: CatalogRepoIdentity[]; + storedOnlyCount: number; +} + +interface UtcDayRange { + start: number; + end: number; +} + +function parseUtcDay(value: string): number { + const timestamp = Date.parse(`${value}T00:00:00Z`); + if (!Number.isFinite(timestamp) || timestamp % DAY_MS !== 0) { + throw new Error(`Invalid UTC date: ${value}`); + } + return timestamp / DAY_MS; +} + +function formatUtcDay(day: number): string { + return new Date(day * DAY_MS).toISOString().slice(0, 10); +} + +function validateSearchResult(result: CatalogSearchResult, label: string): void { + if (result.incomplete) { + throw new Error(`GitHub returned incomplete catalog evidence for ${label}`); + } + if (!Number.isInteger(result.totalCount) || result.totalCount < 0) { + throw new Error(`GitHub returned an invalid total count for ${label}`); + } + if (result.repos.length > GITHUB_SEARCH_PAGE_SIZE) { + throw new Error(`GitHub returned too many repository identities for ${label}`); + } +} + +function popularCatalogQuery(minStars: number): string { + return `stars:>=${minStars}`; +} + +function popularCatalogDateQuery(minStars: number, startDate: string, endDate: string): string { + return `${popularCatalogQuery(minStars)} created:${startDate}..${endDate}`; +} + +/** + * Enumerate the full eligible catalog without paging a mutable star ordering. + * Creation-date ranges are recursively split until every leaf fits in one + * GitHub Search response, so the 1,000-result search window is never reached. + */ +export async function enumeratePopularCatalog( + search: CatalogSearch, + options: { + minStars: number; + minExpectedRepos: number; + startDate?: string; + endDate?: string; + } +): Promise { + const rootQuery = popularCatalogQuery(options.minStars); + const before = await search(rootQuery); + validateSearchResult(before, rootQuery); + if (before.totalCount < options.minExpectedRepos) { + throw new Error( + `GitHub source count ${before.totalCount} is below safety floor ${options.minExpectedRepos}` + ); + } + + const start = parseUtcDay(options.startDate ?? GITHUB_REPOSITORY_EPOCH); + const end = parseUtcDay(options.endDate ?? new Date().toISOString().slice(0, 10)); + if (start > end) throw new Error('Catalog enumeration start date is after its end date'); + + const repos = new Map(); + let leafPartitions = 0; + + async function collect(range: UtcDayRange): Promise { + const startDate = formatUtcDay(range.start); + const endDate = formatUtcDay(range.end); + const query = popularCatalogDateQuery(options.minStars, startDate, endDate); + const result = await search(query); + validateSearchResult(result, query); + + if (result.totalCount > GITHUB_SEARCH_PAGE_SIZE) { + if (range.start === range.end) { + throw new Error( + `GitHub catalog date ${startDate} has ${result.totalCount} results; ` + + `one-response enumeration cannot prove completeness` + ); + } + const midpoint = Math.floor((range.start + range.end) / 2); + await collect({ start: range.start, end: midpoint }); + await collect({ start: midpoint + 1, end: range.end }); + return; + } + + if (result.repos.length !== result.totalCount) { + throw new Error( + `GitHub catalog range ${startDate}..${endDate} returned ` + + `${result.repos.length}/${result.totalCount} repositories` + ); + } + + leafPartitions += 1; + for (const repo of result.repos) { + if (!Number.isInteger(repo.id) || repo.id <= 0 || !repo.fullName.trim()) { + throw new Error(`GitHub returned an invalid repository identity for ${query}`); + } + if (repos.has(repo.id)) { + throw new Error(`GitHub returned duplicate repository ID ${repo.id} across partitions`); + } + repos.set(repo.id, repo); + } + } + + await collect({ start, end }); + + const after = await search(rootQuery); + validateSearchResult(after, rootQuery); + if (after.totalCount !== before.totalCount) { + throw new Error( + `GitHub source count changed during enumeration: ${before.totalCount} -> ${after.totalCount}` + ); + } + if (repos.size !== before.totalCount) { + throw new Error( + `GitHub source reconciliation failed: enumerated ${repos.size}/${before.totalCount} unique IDs` + ); + } + + return { repos, sourceCount: before.totalCount, leafPartitions }; +} + +export function planCatalogReconciliation( + sourceRepos: Map, + storedIds: Set, + maxAdditions: number +): CatalogReconciliationPlan { + if (!Number.isInteger(maxAdditions) || maxAdditions < 0) { + throw new Error(`Invalid maximum additions bound: ${maxAdditions}`); + } + + const additions = [...sourceRepos.values()].filter((repo) => !storedIds.has(repo.id)); + if (additions.length > maxAdditions) { + throw new Error( + `Catalog additions ${additions.length} exceed safety bound ${maxAdditions}; no writes performed` + ); + } + + let storedOnlyCount = 0; + for (const id of storedIds) { + if (!sourceRepos.has(id)) storedOnlyCount += 1; + } + + return { additions, storedOnlyCount }; +}