refactor: scraping

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
-  "python.defaultInterpreterPath": "${workspaceFolder}/scraping-service/.venv/bin/python3",
+  "python.defaultInterpreterPath": "${workspaceFolder}/scraping/.venv/bin/python3",
   "python.analysis.extraPaths": ["${workspaceFolder}/server/src"],
   "python.terminal.activateEnvironment": true,
   "cSpell.words": [

AGENTS.md

@@ -0,0 +1,9 @@
+# Dev Environment Setup
+
+Use bun instead of node
+
+# Code Styling
+
+# Testing Preferences
+
+For golang use testify for tests

client/src/lib/components/onboarding/onboarding-chips-field.svelte

@@ -62,7 +62,11 @@
 			onclick={() => inputEl?.focus()}
 		>
 			{#each values as item, i (item + i)}
-				<button type="button" class={cn(chipClass, 'shrink-0 rounded-2xl')} onclick={() => onRemove(i)}>
+				<button
+					type="button"
+					class={cn(chipClass, 'shrink-0 rounded-2xl')}
+					onclick={() => onRemove(i)}
+				>
 					{item}
 					<span class="opacity-60">×</span>
 				</button>

client/src/lib/components/onboarding/onboarding-form.svelte

@@ -27,7 +27,9 @@
 	let step = $state(1);
 
 	function applyFieldErrors(
-		currentErrors: Parameters<typeof errors.update>[0] extends (arg: infer T) => unknown ? T : never,
+		currentErrors: Parameters<typeof errors.update>[0] extends (arg: infer T) => unknown
+			? T
+			: never,
 		fieldErrors: Record<string, string[] | undefined>
 	) {
 		const mutableErrors = currentErrors as Record<string, string[] | undefined>;

client/src/lib/components/onboarding/onboarding-step-nav.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import { Button } from "$lib/components/ui/button";
+	import { Button } from '$lib/components/ui/button';
 	import * as Form from '$lib/components/ui/form/index.js';
 	import { ArrowLeft, ArrowRight } from 'phosphor-svelte';
 
@@ -19,30 +19,18 @@
 <div
 	class="mt-12 flex items-center justify-between border-t border-zinc-200 pt-8 dark:border-zinc-800"
 >
-	<Button
-		type="button"
-    size="sm"
-		variant="ghost"
-		onclick={onBack}
-	>
+	<Button type="button" size="sm" variant="ghost" onclick={onBack}>
 		<ArrowLeft class="h-3.5 w-3.5" weight="bold" />
 		Back
 	</Button>
 
 	{#if step < 3}
-		<Button
-			type="button"
-      size="sm"
-			onclick={onNext}
-		>
+		<Button type="button" size="sm" onclick={onNext}>
 			Next
 			<ArrowRight class="h-3.5 w-3.5" weight="bold" />
 		</Button>
 	{:else}
-		<Form.Button
-  size="sm"
-			disabled={submitting}
-		>
+		<Form.Button size="sm" disabled={submitting}>
 			{submitting ? 'Saving…' : 'Finish'}
 			<ArrowRight class="h-3.5 w-3.5" weight="bold" />
 		</Form.Button>

client/src/lib/components/plans/plan.svelte

@@ -29,11 +29,11 @@
 		aria-hidden="true"
 	></div>
 
-	<a href={resolve(`/plans/${plan.id}`)} class="absolute inset-0 z-0" aria-label="Open {plan.title}"></a>
+	<a href={resolve(`/plans/${plan.id}`)} class="absolute inset-0 z-0" aria-label="Open {plan.title}"
+	></a>
 
 	<div class="flex flex-5 flex-col space-y-4">
-		<span
-			class="pointer-events-none relative z-10 text-3xl text-pretty lg:text-4xl"
+		<span class="pointer-events-none relative z-10 text-3xl text-pretty lg:text-4xl"
 			>{plan.title}</span
 		>
 	</div>

client/src/lib/types.ts

@@ -4,5 +4,5 @@ export interface Plan {
 	term: 'Fall 2026' | 'Winter 2027' | 'Spring/Summer 2027' | 'Unknown';
 	createdAt: string;
 	courseCount: number;
-  userID: string
+	userID: string;
 }

client/src/routes/+page.svelte

@@ -19,7 +19,7 @@
 
 	<!-- Hero -->
 	<section
-		class="mx-auto flex min-h-[70vh] mt-20 max-w-7xl flex-col items-center justify-center px-6 md:px-12"
+		class="mx-auto mt-20 flex min-h-[70vh] max-w-7xl flex-col items-center justify-center px-6 md:px-12"
 	>
 		<div class="max-w-3xl text-center font-gro">
 			<h1 class="mb-8 text-5xl leading-[1.1] font-bold sm:text-7xl md:text-9xl">
@@ -112,19 +112,11 @@
 					class="flex gap-6 text-[10px] font-bold tracking-[0.2em] text-zinc-900 uppercase dark:text-zinc-300"
 				>
 					{#if data.user?.onboarded}
-						<a href={resolve('/plans')} class="link"
-							>Go to Plans</a
-						>
+						<a href={resolve('/plans')} class="link">Go to Plans</a>
 					{:else if data.user}
-						<a
-							href={resolve('/onboarding')}
-							class="link"
-							>Continue Onboarding</a
-						>
+						<a href={resolve('/onboarding')} class="link">Continue Onboarding</a>
 					{:else}
-						<a href={resolve('/login')} class="link"
-							>Login</a
-						>
+						<a href={resolve('/login')} class="link">Login</a>
 					{/if}
 				</div>
 				<div class="text-[12px] tracking-[0.2em] text-zinc-800 dark:text-zinc-400">

client/src/routes/login/+page.svelte

@@ -14,9 +14,7 @@
 >
 	<Navbar />
 	<div class="flex flex-1 items-center justify-center px-6">
-		<div
-			class="w-full max-w-md  p-8 md:grid-cols-3"
-		>
+		<div class="w-full max-w-md p-8 md:grid-cols-3">
 			<h1 class="mb-3 text-3xl leading-tight font-bold tracking-tight sm:text-4xl">Sign in</h1>
 			<p class="mb-10 text-[14px] leading-relaxed tracking-wide text-zinc-900 dark:text-zinc-300">
 				Continue with your Google account to get started.

client/src/routes/plans/+layout.svelte

@@ -16,6 +16,8 @@
 	class="w-full border-t border-zinc-200 bg-[#f5f5f0]/80 py-6 backdrop-blur-sm dark:border-zinc-900 dark:bg-zinc-950/80"
 >
 	<div class="mx-auto max-w-7xl px-6 md:px-12">
-		<div class="text-[12px] tracking-[0.2em] text-zinc-600 dark:text-zinc-400">© 2026 PathWeave</div>
+		<div class="text-[12px] tracking-[0.2em] text-zinc-600 dark:text-zinc-400">
+			© 2026 PathWeave
+		</div>
 	</div>
 </footer>

client/src/routes/plans/+page.server.ts

@@ -40,14 +40,16 @@ export const load: PageServerLoad = async ({ locals, request, fetch }) => {
 			});
 
 			if (res2.ok) {
-				const plans: Plan[] = (Array.isArray(body) ? body : []).filter(Boolean).map((x: PlanResult) => ({
-					title: x.title,
-					term: x.term,
-					id: x.id,
-					userID: x.user_id,
-					createdAt: x.created_at,
-					courseCount: x.course_count
-				}));
+				const plans: Plan[] = (Array.isArray(body) ? body : [])
+					.filter(Boolean)
+					.map((x: PlanResult) => ({
+						title: x.title,
+						term: x.term,
+						id: x.id,
+						userID: x.user_id,
+						createdAt: x.created_at,
+						courseCount: x.course_count
+					}));
 				const programName = await res2.json();
 
 				return {

client/src/routes/plans/+page.svelte

@@ -10,13 +10,11 @@
 	import type { PageProps } from './$types';
 
 	let { data }: PageProps = $props();
-let selectedTerm = $state('Fall 2026');
-let termValue = $derived(
-	selectedTerm === 'Fall 2026' ? '2269' :
-	selectedTerm === 'Winter 2027' ? '2271' :
-	'2275'
-);
-let title = $state('');
+	let selectedTerm = $state('Fall 2026');
+	let termValue = $derived(
+		selectedTerm === 'Fall 2026' ? '2269' : selectedTerm === 'Winter 2027' ? '2271' : '2275'
+	);
+	let title = $state('');
 </script>
 
 <svelte:head>
@@ -43,7 +41,7 @@ let title = $state('');
 						<Dialog.Description>Choose a new name and term</Dialog.Description>
 					</Dialog.Header>
 					<form method="POST" action="?/createPlan" class="gap-4">
-						<div class="flex items-center gap-2 mb-4">
+						<div class="mb-4 flex items-center gap-2">
 							<div class="grid flex-1 gap-2">
 								<Label for="Title" class="sr-only">Title</Label>
 								<Input

scraping/.vscode/settings.json

@@ -1,4 +1,4 @@
 {
-  "python.defaultInterpreterPath": "/Users/twito/Documents/Code/projects/PathWeave/scraping-service/.venv/bin/python3",
+  "python.defaultInterpreterPath": "/Users/twito/Documents/Code/projects/PathWeave/scraping/.venv/bin/python3",
   "python.analysis.extraPaths": ["${workspaceFolder}/src"]
 }

scraping/Justfile

@@ -2,4 +2,13 @@ dev:
 	.venv/bin/python -m fastapi dev src/main.py --port 8001
 
 sync:
-	uv sync
+	uv sync
+
+publish:
+	.venv/bin/python scripts/publish_scrape_run.py
+
+publish-stage-only:
+	.venv/bin/python scripts/publish_scrape_run.py --no-promote
+
+test:
+	.venv/bin/python -m unittest discover scripts

scraping/README.md

@@ -13,11 +13,34 @@
    - `WORKOS_CLIENT_ID`
    - `WORKOS_COOKIE_PASSWORD`
    - `INTERNAL_SERVICE_TOKEN`
+   - `GO_API_BASE_URL`
 2. Start Python AI service:
    - `just dev`
 3. Start Go API:
    - `just go-dev`
 
+## Scraping Ingestion Pipeline
+
+Python still owns browser scraping and writes JSON artifacts under `data/`. The Go API now owns validation, staging, and promotion into Postgres through internal scrape-run endpoints.
+
+Recommended flow:
+
+1. Run the scraper scripts that produce:
+   - `data/all_courses.json`
+   - `data/all_programs_with_requirements.json`
+   - `data/rmp_data.json`
+   - `data/all_possible_schedules.json`
+2. Start the Go API with `INTERNAL_SERVICE_TOKEN` configured.
+3. Publish and promote the artifacts:
+   - `just publish`
+
+For inspection before promotion:
+
+- `just publish-stage-only`
+- Then call `POST /internal/scrape-runs/{run_id}/promote` with `Authorization: Bearer $INTERNAL_SERVICE_TOKEN`.
+
+`scripts/seed_db.py` is now legacy. It still exists as a compatibility fallback, but new scrape runs should go through `scripts/publish_scrape_run.py` so database writes stay in the Go server.
+
 ## Go API Commands
 
 - `cd go-api && make run`

scraping/scripts/publish_scrape_run.py

@@ -0,0 +1,115 @@
+import argparse
+import json
+import os
+import pathlib
+import sys
+import urllib.error
+import urllib.request
+from typing import Any
+from dotenv import load_dotenv
+
+SCRIPT_DIR = pathlib.Path(__file__).resolve().parent
+PROJECT_ROOT = SCRIPT_DIR.parent
+DATA_DIR = PROJECT_ROOT / "data"
+
+
+load_dotenv()
+
+def load_json(path: pathlib.Path) -> Any:
+    with path.open("r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def build_artifacts_payload(data_dir: pathlib.Path = DATA_DIR) -> dict[str, Any]:
+    rmp_data = load_json(data_dir / "rmp_data.json")
+    return {
+        "courses": load_json(data_dir / "all_courses.json"),
+        "programs": load_json(data_dir / "all_programs_with_requirements.json"),
+        "teachers": rmp_data.get("professors", []),
+        "schedules": load_json(data_dir / "all_possible_schedules.json"),
+    }
+
+
+def build_json_request(url: str, token: str, payload: dict[str, Any]) -> urllib.request.Request:
+    body = json.dumps(payload).encode("utf-8")
+    return urllib.request.Request(
+        url,
+        data=body,
+        headers={
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+        },
+        method="POST",
+    )
+
+
+def post_json(url: str, token: str, payload: dict[str, Any]) -> dict[str, Any]:
+    request = build_json_request(url, token, payload)
+    try:
+        with urllib.request.urlopen(request) as response:
+            return json.loads(response.read().decode("utf-8"))
+    except urllib.error.HTTPError as e:
+        detail = e.read().decode("utf-8")
+        raise RuntimeError(f"POST {url} failed with {e.code}: {detail}") from e
+
+
+def run_publish(
+    api_base_url: str,
+    token: str,
+    data_dir: pathlib.Path = DATA_DIR,
+    source: str = "scraping",
+    promote: bool = True,
+) -> dict[str, Any]:
+    if not token:
+        raise ValueError("INTERNAL_SERVICE_TOKEN is required")
+
+    api_base_url = api_base_url.rstrip("/")
+    run = post_json(
+        f"{api_base_url}/internal/scrape-runs",
+        token,
+        {"source": source, "metadata": {"publisher": "publish_scrape_run.py"}},
+    )
+    run_id = run["ID"] if "ID" in run else run["id"]
+
+    artifacts = build_artifacts_payload(data_dir)
+    staged = post_json(
+        f"{api_base_url}/internal/scrape-runs/{run_id}/artifacts",
+        token,
+        artifacts,
+    )
+
+    result: dict[str, Any] = {"run": run, "staged": staged}
+    if promote:
+        result["promoted"] = post_json(
+            f"{api_base_url}/internal/scrape-runs/{run_id}/promote",
+            token,
+            {},
+        )
+    return result
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Publish scraper artifacts to the Go ingest API.")
+    parser.add_argument("--api-base-url", default=os.getenv("GO_API_BASE_URL", "http://localhost:8000"))
+    parser.add_argument("--token", default=os.getenv("INTERNAL_SERVICE_TOKEN", ""))
+    parser.add_argument("--data-dir", type=pathlib.Path, default=DATA_DIR)
+    parser.add_argument("--source", default="scraping")
+    parser.add_argument("--no-promote", action="store_true", help="Stage artifacts without promoting them.")
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv or sys.argv[1:])
+    result = run_publish(
+        api_base_url=args.api_base_url,
+        token=args.token,
+        data_dir=args.data_dir,
+        source=args.source,
+        promote=not args.no_promote,
+    )
+    print(json.dumps(result, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scraping/scripts/seed_db.py

@@ -133,6 +133,10 @@ def normalize_term(term_str):
 
 
 def seed():
+    print(
+        "WARNING: seed_db.py is the legacy direct-DB seeder. "
+        "Prefer scripts/publish_scrape_run.py so the Go API owns validation and promotion."
+    )
     conn = get_db_connection()
     cur = conn.cursor()