From 6ab13b470322f3f61568c85e2b0d5c97859d8886 Mon Sep 17 00:00:00 2001 From: twitocode Date: Sun, 14 Jun 2026 15:00:13 -0400 Subject: [PATCH 1/3] feat: added simple agents.md --- AGENTS.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..e37eef3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,9 @@ +# Dev Environment Setup + +Use bun instead of node + +# Code Styling + +# Testing Preferences + +For golang use testify for tests From 64004d86cc575d8f5d082721b5881905381103d3 Mon Sep 17 00:00:00 2001 From: twitocode Date: Sun, 14 Jun 2026 15:28:09 -0400 Subject: [PATCH 2/3] refactor: scraping db actions will now be done on go server --- .vscode/settings.json | 2 +- scraping/.vscode/settings.json | 2 +- scraping/Justfile | 11 +- scraping/scripts/publish_scrape_run.py | 115 ++++++++++++++++++ scraping/scripts/seed_db.py | 4 + .../{ => tests}/test_get_all_programs.py | 0 .../{ => tests}/test_parse_antirequisites.py | 0 .../scripts/tests/test_publish_scrape_run.py | 55 +++++++++ .../test_scrape_mosaic_schedules.py | 18 ++- scraping/scripts/{ => tests}/test_seed_db.py | 0 scraping/test_fetch.py | 33 ----- scraping/test_fetch2.py | 32 ----- 12 files changed, 193 insertions(+), 79 deletions(-) create mode 100644 scraping/scripts/publish_scrape_run.py rename scraping/scripts/{ => tests}/test_get_all_programs.py (100%) rename scraping/scripts/{ => tests}/test_parse_antirequisites.py (100%) create mode 100644 scraping/scripts/tests/test_publish_scrape_run.py rename scraping/scripts/{ => tests}/test_scrape_mosaic_schedules.py (96%) rename scraping/scripts/{ => tests}/test_seed_db.py (100%) delete mode 100644 scraping/test_fetch.py delete mode 100644 scraping/test_fetch2.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 3ad0b35..4ca12c2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { - "python.defaultInterpreterPath": "${workspaceFolder}/scraping-service/.venv/bin/python3", + "python.defaultInterpreterPath": "${workspaceFolder}/scraping/.venv/bin/python3", "python.analysis.extraPaths": ["${workspaceFolder}/server/src"], "python.terminal.activateEnvironment": true, "cSpell.words": [ diff --git a/scraping/.vscode/settings.json b/scraping/.vscode/settings.json index 6c414ce..96a4d2a 100644 --- a/scraping/.vscode/settings.json +++ b/scraping/.vscode/settings.json @@ -1,4 +1,4 @@ { - "python.defaultInterpreterPath": "/Users/twito/Documents/Code/projects/PathWeave/scraping-service/.venv/bin/python3", + "python.defaultInterpreterPath": "/Users/twito/Documents/Code/projects/PathWeave/scraping/.venv/bin/python3", "python.analysis.extraPaths": ["${workspaceFolder}/src"] } diff --git a/scraping/Justfile b/scraping/Justfile index 22eef82..547d9a2 100644 --- a/scraping/Justfile +++ b/scraping/Justfile @@ -2,4 +2,13 @@ dev: .venv/bin/python -m fastapi dev src/main.py --port 8001 sync: - uv sync \ No newline at end of file + uv sync + +publish: + .venv/bin/python scripts/publish_scrape_run.py + +publish-stage-only: + .venv/bin/python scripts/publish_scrape_run.py --no-promote + +test: + .venv/bin/python -m unittest discover scripts \ No newline at end of file diff --git a/scraping/scripts/publish_scrape_run.py b/scraping/scripts/publish_scrape_run.py new file mode 100644 index 0000000..b24e3d2 --- /dev/null +++ b/scraping/scripts/publish_scrape_run.py @@ -0,0 +1,115 @@ +import argparse +import json +import os +import pathlib +import sys +import urllib.error +import urllib.request +from typing import Any +from dotenv import load_dotenv + +SCRIPT_DIR = pathlib.Path(__file__).resolve().parent +PROJECT_ROOT = SCRIPT_DIR.parent +DATA_DIR = PROJECT_ROOT / "data" + + +load_dotenv() + +def load_json(path: pathlib.Path) -> Any: + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def build_artifacts_payload(data_dir: pathlib.Path = DATA_DIR) -> dict[str, Any]: + rmp_data = load_json(data_dir / "rmp_data.json") + return { + "courses": load_json(data_dir / "all_courses.json"), + "programs": load_json(data_dir / "all_programs_with_requirements.json"), + "teachers": rmp_data.get("professors", []), + "schedules": load_json(data_dir / "all_possible_schedules.json"), + } + + +def build_json_request(url: str, token: str, payload: dict[str, Any]) -> urllib.request.Request: + body = json.dumps(payload).encode("utf-8") + return urllib.request.Request( + url, + data=body, + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + method="POST", + ) + + +def post_json(url: str, token: str, payload: dict[str, Any]) -> dict[str, Any]: + request = build_json_request(url, token, payload) + try: + with urllib.request.urlopen(request) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as e: + detail = e.read().decode("utf-8") + raise RuntimeError(f"POST {url} failed with {e.code}: {detail}") from e + + +def run_publish( + api_base_url: str, + token: str, + data_dir: pathlib.Path = DATA_DIR, + source: str = "scraping", + promote: bool = True, +) -> dict[str, Any]: + if not token: + raise ValueError("INTERNAL_SERVICE_TOKEN is required") + + api_base_url = api_base_url.rstrip("/") + run = post_json( + f"{api_base_url}/internal/scrape-runs", + token, + {"source": source, "metadata": {"publisher": "publish_scrape_run.py"}}, + ) + run_id = run["ID"] if "ID" in run else run["id"] + + artifacts = build_artifacts_payload(data_dir) + staged = post_json( + f"{api_base_url}/internal/scrape-runs/{run_id}/artifacts", + token, + artifacts, + ) + + result: dict[str, Any] = {"run": run, "staged": staged} + if promote: + result["promoted"] = post_json( + f"{api_base_url}/internal/scrape-runs/{run_id}/promote", + token, + {}, + ) + return result + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Publish scraper artifacts to the Go ingest API.") + parser.add_argument("--api-base-url", default=os.getenv("GO_API_BASE_URL", "http://localhost:8000")) + parser.add_argument("--token", default=os.getenv("INTERNAL_SERVICE_TOKEN", "")) + parser.add_argument("--data-dir", type=pathlib.Path, default=DATA_DIR) + parser.add_argument("--source", default="scraping") + parser.add_argument("--no-promote", action="store_true", help="Stage artifacts without promoting them.") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv or sys.argv[1:]) + result = run_publish( + api_base_url=args.api_base_url, + token=args.token, + data_dir=args.data_dir, + source=args.source, + promote=not args.no_promote, + ) + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scraping/scripts/seed_db.py b/scraping/scripts/seed_db.py index 23e9e9c..459587b 100644 --- a/scraping/scripts/seed_db.py +++ b/scraping/scripts/seed_db.py @@ -133,6 +133,10 @@ def normalize_term(term_str): def seed(): + print( + "WARNING: seed_db.py is the legacy direct-DB seeder. " + "Prefer scripts/publish_scrape_run.py so the Go API owns validation and promotion." + ) conn = get_db_connection() cur = conn.cursor() diff --git a/scraping/scripts/test_get_all_programs.py b/scraping/scripts/tests/test_get_all_programs.py similarity index 100% rename from scraping/scripts/test_get_all_programs.py rename to scraping/scripts/tests/test_get_all_programs.py diff --git a/scraping/scripts/test_parse_antirequisites.py b/scraping/scripts/tests/test_parse_antirequisites.py similarity index 100% rename from scraping/scripts/test_parse_antirequisites.py rename to scraping/scripts/tests/test_parse_antirequisites.py diff --git a/scraping/scripts/tests/test_publish_scrape_run.py b/scraping/scripts/tests/test_publish_scrape_run.py new file mode 100644 index 0000000..ee829e7 --- /dev/null +++ b/scraping/scripts/tests/test_publish_scrape_run.py @@ -0,0 +1,55 @@ +import importlib.util +import json +import pathlib +import tempfile +import unittest + + +SCRIPT_PATH = pathlib.Path(__file__).resolve().parent / "publish_scrape_run.py" +SPEC = importlib.util.spec_from_file_location("publish_scrape_run", SCRIPT_PATH) +assert SPEC is not None +assert SPEC.loader is not None +PUBLISHER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(PUBLISHER) + + +class BuildArtifactsPayloadTests(unittest.TestCase): + def test_builds_payload_from_existing_scraper_files(self): + with tempfile.TemporaryDirectory() as tmp: + data_dir = pathlib.Path(tmp) + (data_dir / "all_courses.json").write_text( + json.dumps([{"course_name": "COMPSCI 1DM3 - Discrete Mathematics"}]) + ) + (data_dir / "all_programs_with_requirements.json").write_text( + json.dumps([{"program_name": "Computer Science", "requirements": ["COMPSCI 1DM3"]}]) + ) + (data_dir / "rmp_data.json").write_text( + json.dumps({"professors": [{"id": "rmp_1", "name": "Jane Doe"}]}) + ) + (data_dir / "all_possible_schedules.json").write_text( + json.dumps([{"term": "Fall 2026", "courses": []}]) + ) + + payload = PUBLISHER.build_artifacts_payload(data_dir) + + self.assertEqual(payload["courses"][0]["course_name"], "COMPSCI 1DM3 - Discrete Mathematics") + self.assertEqual(payload["programs"][0]["program_name"], "Computer Science") + self.assertEqual(payload["teachers"][0]["id"], "rmp_1") + self.assertEqual(payload["schedules"][0]["term"], "Fall 2026") + + +class RequestTests(unittest.TestCase): + def test_builds_internal_authorization_request(self): + request = PUBLISHER.build_json_request( + "http://localhost:8000/internal/scrape-runs", + "secret", + {"source": "manual"}, + ) + + self.assertEqual(request.get_method(), "POST") + self.assertEqual(request.headers["Authorization"], "Bearer secret") + self.assertEqual(request.headers["Content-type"], "application/json") + + +if __name__ == "__main__": + unittest.main() diff --git a/scraping/scripts/test_scrape_mosaic_schedules.py b/scraping/scripts/tests/test_scrape_mosaic_schedules.py similarity index 96% rename from scraping/scripts/test_scrape_mosaic_schedules.py rename to scraping/scripts/tests/test_scrape_mosaic_schedules.py index 313585e..47b5621 100644 --- a/scraping/scripts/test_scrape_mosaic_schedules.py +++ b/scraping/scripts/tests/test_scrape_mosaic_schedules.py @@ -7,18 +7,14 @@ sys.modules.setdefault("dotenv", types.SimpleNamespace(load_dotenv=lambda *_args, **_kwargs: None)) sys.modules.setdefault("psycopg2", types.SimpleNamespace(connect=lambda *_args, **_kwargs: None)) -sys.modules.setdefault( - "playwright", - types.SimpleNamespace(async_api=types.SimpleNamespace(async_playwright=None)), +sys.modules["playwright"] = types.SimpleNamespace( + async_api=types.SimpleNamespace(async_playwright=None) ) -sys.modules.setdefault( - "playwright.async_api", - types.SimpleNamespace( - async_playwright=None, - Page=object, - Frame=object, - Browser=object, - ), +sys.modules["playwright.async_api"] = types.SimpleNamespace( + async_playwright=None, + Page=object, + Frame=object, + Browser=object, ) diff --git a/scraping/scripts/test_seed_db.py b/scraping/scripts/tests/test_seed_db.py similarity index 100% rename from scraping/scripts/test_seed_db.py rename to scraping/scripts/tests/test_seed_db.py diff --git a/scraping/test_fetch.py b/scraping/test_fetch.py deleted file mode 100644 index 462f793..0000000 --- a/scraping/test_fetch.py +++ /dev/null @@ -1,33 +0,0 @@ -import asyncio -from playwright.async_api import async_playwright - -async def main(): - async with async_playwright() as p: - browser = await p.chromium.launch() - context = await browser.new_context() - page = await context.new_page() - - url = "https://academiccalendars.romcmaster.ca/content.php?catoid=65&navoid=14802&filter%5Bitem_type%5D=3&filter%5Bonly_active%5D=1&filter%5B3%5D=1&filter%5Bcpage%5D=28#acalog_template_course_filter" - await page.goto(url) - - course_links = await page.evaluate(""" - () => Array.from(document.querySelectorAll("a[onclick^='showCourse']")).map(a => { - const match = a.getAttribute('onclick').match(/showCourse\('(\d+)',\s*'(\d+)'/); - return { text: a.innerText.trim(), coid: match ? match[2] : null }; - }).filter(item => item.coid !== null) - """) - - print(f"Found {len(course_links)} courses") - if course_links: - first = course_links[0] - print(f"First course: {first}") - - preview_url = f"https://academiccalendars.romcmaster.ca/ajax/preview_course.php?catoid=65&show&coid={first['coid']}" - resp = await context.request.get(preview_url) - print(f"Status: {resp.status}") - text = await resp.text() - print(f"Response: {text[:200]}") - - await browser.close() - -asyncio.run(main()) diff --git a/scraping/test_fetch2.py b/scraping/test_fetch2.py deleted file mode 100644 index f45cfd3..0000000 --- a/scraping/test_fetch2.py +++ /dev/null @@ -1,32 +0,0 @@ -import asyncio -from playwright.async_api import async_playwright - -async def main(): - async with async_playwright() as p: - browser = await p.chromium.launch() - context = await browser.new_context(ignore_https_errors=True) - page = await context.new_page() - - url = "https://academiccalendars.romcmaster.ca/content.php?catoid=65&navoid=14802&filter%5Bitem_type%5D=3&filter%5Bonly_active%5D=1&filter%5B3%5D=1&filter%5Bcpage%5D=28#acalog_template_course_filter" - await page.goto(url) - - course_links = await page.evaluate(""" - () => Array.from(document.querySelectorAll("a[onclick^='showCourse']")).map(a => { - const match = a.getAttribute('onclick').match(/showCourse\\('(\\d+)',\\s*'(\\d+)'/); - return { text: a.innerText.trim(), coid: match ? match[2] : null }; - }).filter(item => item.coid !== null) - """) - - if course_links: - first = course_links[0] - print(f"First course: {first}") - - preview_url = f"https://academiccalendars.romcmaster.ca/ajax/preview_course.php?catoid=65&show&coid={first['coid']}" - resp = await context.request.get(preview_url, ignore_https_errors=True) - print(f"Status: {resp.status}") - text = await resp.text() - print(f"Response: {text[:50]}") - - await browser.close() - -asyncio.run(main()) From 5154fb88018d0b1c78b418cbf235b6791adf0de1 Mon Sep 17 00:00:00 2001 From: twitocode Date: Sun, 14 Jun 2026 16:05:33 -0400 Subject: [PATCH 3/3] feat(scraping): implement scrape ingest service with create, get, stage, and promote functionalities --- .../onboarding/onboarding-chips-field.svelte | 6 +- .../onboarding/onboarding-form.svelte | 4 +- .../onboarding/onboarding-step-nav.svelte | 20 +- client/src/lib/components/plans/plan.svelte | 6 +- client/src/lib/types.ts | 2 +- client/src/routes/+page.svelte | 16 +- client/src/routes/login/+page.svelte | 4 +- client/src/routes/plans/+layout.svelte | 4 +- client/src/routes/plans/+page.server.ts | 18 +- client/src/routes/plans/+page.svelte | 14 +- scraping/README.md | 23 + ...260614143000_scrape_ingestion_pipeline.sql | 80 ++ server/db/query/scrape_ingest.sql | 209 +++++ server/internal/app/routes.go | 9 + server/internal/app/services.go | 35 +- server/internal/db/models.go | 47 ++ server/internal/db/scrape_ingest.sql.go | 728 ++++++++++++++++++ server/internal/handlers/scrape_ingest.go | 101 +++ server/internal/middleware/internal.go | 27 + server/internal/middleware/internal_test.go | 39 + .../service/scraping/scrape_ingest.go | 45 ++ .../scraping/scrape_ingest_normalize.go | 309 ++++++++ .../scraping/scrape_ingest_payloads.go | 106 +++ .../service/scraping/scrape_ingest_promote.go | 449 +++++++++++ .../service/scraping/scrape_ingest_stage.go | 127 +++ .../service/scraping/scrape_ingest_test.go | 51 ++ 26 files changed, 2409 insertions(+), 70 deletions(-) create mode 100644 server/db/migrations/20260614143000_scrape_ingestion_pipeline.sql create mode 100644 server/db/query/scrape_ingest.sql create mode 100644 server/internal/db/scrape_ingest.sql.go create mode 100644 server/internal/handlers/scrape_ingest.go create mode 100644 server/internal/middleware/internal.go create mode 100644 server/internal/middleware/internal_test.go create mode 100644 server/internal/service/scraping/scrape_ingest.go create mode 100644 server/internal/service/scraping/scrape_ingest_normalize.go create mode 100644 server/internal/service/scraping/scrape_ingest_payloads.go create mode 100644 server/internal/service/scraping/scrape_ingest_promote.go create mode 100644 server/internal/service/scraping/scrape_ingest_stage.go create mode 100644 server/internal/service/scraping/scrape_ingest_test.go diff --git a/client/src/lib/components/onboarding/onboarding-chips-field.svelte b/client/src/lib/components/onboarding/onboarding-chips-field.svelte index ce360d0..746aa09 100644 --- a/client/src/lib/components/onboarding/onboarding-chips-field.svelte +++ b/client/src/lib/components/onboarding/onboarding-chips-field.svelte @@ -62,7 +62,11 @@ onclick={() => inputEl?.focus()} > {#each values as item, i (item + i)} - diff --git a/client/src/lib/components/onboarding/onboarding-form.svelte b/client/src/lib/components/onboarding/onboarding-form.svelte index 7ea3498..9c25072 100644 --- a/client/src/lib/components/onboarding/onboarding-form.svelte +++ b/client/src/lib/components/onboarding/onboarding-form.svelte @@ -27,7 +27,9 @@ let step = $state(1); function applyFieldErrors( - currentErrors: Parameters[0] extends (arg: infer T) => unknown ? T : never, + currentErrors: Parameters[0] extends (arg: infer T) => unknown + ? T + : never, fieldErrors: Record ) { const mutableErrors = currentErrors as Record; diff --git a/client/src/lib/components/onboarding/onboarding-step-nav.svelte b/client/src/lib/components/onboarding/onboarding-step-nav.svelte index 1447e80..ab3e12d 100644 --- a/client/src/lib/components/onboarding/onboarding-step-nav.svelte +++ b/client/src/lib/components/onboarding/onboarding-step-nav.svelte @@ -1,5 +1,5 @@ @@ -43,7 +41,7 @@ let title = $state(''); Choose a new name and term
-
+
0 { + return raw, strings.ToUpper(parts[0]) + } + return raw, "" +} + +func parseScrapeTime(value string) (string, error) { + value = strings.TrimSpace(value) + if value == "" || strings.EqualFold(value, "TBA") { + return "", errors.New("time is empty") + } + for _, layout := range []string{"3:04PM", "3:04 PM"} { + parsed, err := time.Parse(layout, value) + if err == nil { + return parsed.Format("15:04:05"), nil + } + } + return "", fmt.Errorf("invalid time %q", value) +} + +func parseNullableScrapeTime(value string) (pgtype.Time, error) { + value = strings.TrimSpace(value) + if value == "" || strings.EqualFold(value, "TBA") { + return pgtype.Time{}, nil + } + normalized, err := parseScrapeTime(value) + if err != nil { + return pgtype.Time{}, err + } + parsed, err := time.Parse("15:04:05", normalized) + if err != nil { + return pgtype.Time{}, err + } + return pgtype.Time{ + Microseconds: int64(parsed.Hour()*3600+parsed.Minute()*60+parsed.Second()) * 1_000_000, + Valid: true, + }, nil +} + +func parseScrapeLocation(value string) (string, string) { + raw := strings.TrimSpace(value) + if raw == "" { + return "", "" + } + upper := strings.ToUpper(raw) + if upper == "IN PERSON" || upper == "IN-PERSON" { + return "", "" + } + if strings.Contains(upper, "TBD") || strings.Contains(upper, "TBA") || + strings.Contains(upper, "ANNOUNCED") || strings.Contains(upper, "SEE CLASS NOTES") { + return "TBD", "TBD" + } + if strings.Contains(upper, "ONLINE") || strings.Contains(upper, "VIRTUAL") { + return "Online", "Online" + } + parts := strings.Fields(raw) + if len(parts) >= 2 && regexp.MustCompile(`^[A-Z0-9]+$`).MatchString(parts[0]) { + room := strings.TrimSpace(strings.Join(parts[1:], " ")) + room = strings.TrimSpace(regexp.MustCompile(`(?i)lab`).ReplaceAllString(room, "")) + return parts[0], room + } + return raw, "" +} + +func getInstructorNames(value string) []string { + cleaned := strings.ReplaceAll(value, "\u00a0", " ") + names := make([]string, 0) + for _, line := range strings.Split(cleaned, "\n") { + for _, part := range strings.Split(line, ",") { + name := strings.Join(strings.Fields(part), " ") + if name != "" { + names = append(names, name) + } + } + } + return names +} + +func getAllInstructorNames(section rawSectionPayload) []string { + seen := make(map[string]struct{}) + for _, detail := range section.Details { + for _, name := range getInstructorNames(detail.Instructor) { + seen[name] = struct{}{} + } + } + names := make([]string, 0, len(seen)) + for name := range seen { + names = append(names, name) + } + sort.Strings(names) + return names +} + +func getSectionInstructorSet(section rawSectionPayload) map[string]struct{} { + names := make(map[string]struct{}) + for _, name := range getAllInstructorNames(section) { + if !strings.EqualFold(name, "Staff") { + names[name] = struct{}{} + } + } + return names +} + +func detectDeliveryMode(section rawSectionPayload) (string, bool) { + hasOnline := false + hasInPerson := false + for _, detail := range section.Details { + room := strings.ToUpper(strings.TrimSpace(detail.Room)) + switch { + case strings.Contains(room, "ONLINE") || strings.Contains(room, "VIRTUAL"): + hasOnline = true + case room == "IN PERSON" || room == "IN-PERSON": + hasInPerson = true + case room != "" && room != "TBA" && room != "TBD": + hasInPerson = true + } + } + switch { + case hasOnline && hasInPerson: + return "Blended", true + case hasOnline: + return "Online", false + case hasInPerson: + return "In Person", true + default: + return "Unknown", false + } +} + +func buildSectionReferences(sections []normalizedSection) []sectionReference { + parents := make([]normalizedSection, 0) + children := make([]normalizedSection, 0) + for _, section := range sections { + switch section.Type { + case "LEC", "SEM": + parents = append(parents, section) + case "LAB", "TUT": + children = append(children, section) + } + } + refs := make([]sectionReference, 0) + for _, parent := range parents { + for _, child := range children { + if len(child.InstructorSet) == 0 || len(parent.InstructorSet) == 0 || intersects(parent.InstructorSet, child.InstructorSet) { + refs = append(refs, sectionReference{ParentID: parent.ID, ChildID: child.ID}) + } + } + } + return refs +} + +func intersects(a, b map[string]struct{}) bool { + for key := range a { + if _, ok := b[key]; ok { + return true + } + } + return false +} + +func collectScheduleTeacherNames(schedules []rawScheduleCoursePayload) map[string]struct{} { + names := make(map[string]struct{}) + for _, course := range schedules { + for _, section := range course.Sections { + for _, name := range getAllInstructorNames(section) { + names[name] = struct{}{} + } + } + } + return names +} + +func resolveScheduleCourseCode(course rawScheduleCoursePayload, titleCodeMap map[string]string) string { + code := strings.TrimSpace(course.CourseCode) + if courseCodePattern.MatchString(code) { + return code + } + if resolved := titleCodeMap[strings.TrimSpace(course.CourseTitle)]; resolved != "" { + return resolved + } + return code +} + +func nonRMPID(name string) string { + id := strings.ToLower(strings.TrimSpace(name)) + id = regexp.MustCompile(`\s+`).ReplaceAllString(id, "_") + return "non_rmp_" + id +} + +func nullableText(value string) pgtype.Text { + value = strings.TrimSpace(value) + if value == "" { + return pgtype.Text{} + } + return pgtype.Text{String: value, Valid: true} +} + +func numericFromFloat64(value float64) pgtype.Numeric { + var numeric pgtype.Numeric + _ = numeric.Scan(fmt.Sprintf("%g", value)) + return numeric +} diff --git a/server/internal/service/scraping/scrape_ingest_payloads.go b/server/internal/service/scraping/scrape_ingest_payloads.go new file mode 100644 index 0000000..713d61f --- /dev/null +++ b/server/internal/service/scraping/scrape_ingest_payloads.go @@ -0,0 +1,106 @@ +package scraping + +import ( + "encoding/json" + + "github.com/google/uuid" +) + +type CreateScrapeRunRequest struct { + Source string `json:"source"` + Metadata json.RawMessage `json:"metadata"` +} + +type StageScrapeArtifactsRequest struct { + Courses []rawCoursePayload `json:"courses"` + Programs []rawProgramPayload `json:"programs"` + Teachers []rawTeacherPayload `json:"teachers"` + Schedules []rawScheduleTermPayload `json:"schedules"` +} + +type StageScrapeArtifactsResult struct { + RunID uuid.UUID `json:"run_id"` + CourseCount int `json:"course_count"` + ProgramCount int `json:"program_count"` + TeacherCount int `json:"teacher_count"` + ScheduleTermCount int `json:"schedule_term_count"` +} + +type PromoteScrapeRunResult struct { + RunID uuid.UUID `json:"run_id"` + Status string `json:"status"` + Promoted bool `json:"promoted"` +} + +type rawCoursePayload struct { + Code string `json:"code"` + Name string `json:"name"` + CourseName string `json:"course_name"` + Units string `json:"units"` + Description string `json:"description"` + Restrictions string `json:"restrictions"` + Prerequisites []string `json:"prerequisites"` +} + +type rawProgramPayload struct { + ProgramName string `json:"program_name"` + URL string `json:"url"` + Requirements []string `json:"requirements"` + RequirementsByLevel json.RawMessage `json:"requirements_by_level"` +} + +type rawTeacherPayload struct { + ID string `json:"id"` + Name string `json:"name"` + AvgRating float64 `json:"avgRating"` + AvgDifficulty float64 `json:"avgDifficulty"` + Department string `json:"department"` + NumRatings int32 `json:"numRatings"` + Courses []string `json:"courses"` +} + +type rawScheduleTermPayload struct { + Term string `json:"term"` + Courses []rawScheduleCoursePayload `json:"courses"` +} + +type rawScheduleCoursePayload struct { + CourseCode string `json:"course_code"` + CourseTitle string `json:"course_title"` + Term string `json:"term"` + Sections []rawSectionPayload `json:"sections"` +} + +type rawSectionPayload struct { + SectionName string `json:"section_name"` + Details []rawMeetingPayload `json:"details"` +} + +type rawMeetingPayload struct { + Days string `json:"days"` + StartTime string `json:"start_time"` + EndTime string `json:"end_time"` + Room string `json:"room"` + Instructor string `json:"instructor"` +} + +type normalizedCourseRecord struct { + Code string + Name string + Description string + Restrictions string + Prerequisites []string + Units int32 + LevelNumber *int32 +} + +type normalizedSection struct { + ID int32 + Type string + InstructorSet map[string]struct{} +} + +type sectionReference struct { + ParentID int32 + ChildID int32 +} diff --git a/server/internal/service/scraping/scrape_ingest_promote.go b/server/internal/service/scraping/scrape_ingest_promote.go new file mode 100644 index 0000000..af86446 --- /dev/null +++ b/server/internal/service/scraping/scrape_ingest_promote.go @@ -0,0 +1,449 @@ +package scraping + +import ( + "context" + "encoding/json" + "errors" + "strings" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "github.com/twitocode/pathweave/go-api/internal/db" + "go.uber.org/zap" +) + +func (s *ScrapeIngestService) PromoteRun(ctx context.Context, runID uuid.UUID) (PromoteScrapeRunResult, error) { + log := s.log.With(zap.String("run_id", runID.String())) + log.Info("scrape promote started") + + tx, err := s.pool.Begin(ctx) + if err != nil { + log.Error("scrape promote failed to begin transaction", zap.Error(err)) + return PromoteScrapeRunResult{}, err + } + + qtx := s.db.WithTx(tx) + if err := qtx.MarkScrapeRunPromoting(ctx, runID); err != nil { + _ = tx.Rollback(ctx) + log.Error("scrape promote failed to mark run promoting", zap.Error(err)) + return PromoteScrapeRunResult{}, err + } + if err := s.promoteRunInTx(ctx, qtx, runID, log); err != nil { + _ = tx.Rollback(ctx) + _ = s.db.MarkScrapeRunFailed(ctx, db.MarkScrapeRunFailedParams{ + ID: runID, + ErrorMessage: pgtype.Text{String: err.Error(), Valid: true}, + }) + log.Error("scrape promote failed", zap.Error(err)) + return PromoteScrapeRunResult{}, err + } + if err := qtx.MarkScrapeRunSucceeded(ctx, runID); err != nil { + _ = tx.Rollback(ctx) + log.Error("scrape promote failed to mark run succeeded", zap.Error(err)) + return PromoteScrapeRunResult{}, err + } + if err := tx.Commit(ctx); err != nil { + log.Error("scrape promote failed to commit transaction", zap.Error(err)) + return PromoteScrapeRunResult{}, err + } + log.Info("scrape promote completed") + return PromoteScrapeRunResult{RunID: runID, Status: "succeeded", Promoted: true}, nil +} + +func (s *ScrapeIngestService) promoteRunInTx(ctx context.Context, qtx *db.Queries, runID uuid.UUID, log *zap.Logger) error { + log.Info("scrape promote loading staged payloads") + courses, err := loadPayloads[rawCoursePayload](ctx, qtx.ListStagedCoursePayloads, runID) + if err != nil { + return err + } + programs, err := loadPayloads[rawProgramPayload](ctx, qtx.ListStagedProgramPayloads, runID) + if err != nil { + return err + } + teachers, err := loadPayloads[rawTeacherPayload](ctx, qtx.ListStagedTeacherPayloads, runID) + if err != nil { + return err + } + schedules, err := loadPayloads[rawScheduleCoursePayload](ctx, qtx.ListStagedSchedulePayloads, runID) + if err != nil { + return err + } + terms, err := qtx.ListStagedScheduleTerms(ctx, runID) + if err != nil { + return err + } + log.Info("scrape promote loaded staged payloads", + zap.Int("courses", len(courses)), + zap.Int("programs", len(programs)), + zap.Int("teachers", len(teachers)), + zap.Int("schedule_courses", len(schedules)), + zap.Strings("terms", terms), + ) + + log.Info("scrape promote upserting courses", zap.Int("count", len(courses))) + normalizedCourses, titleCodeMap, err := promoteCourses(ctx, qtx, courses) + if err != nil { + return err + } + log.Info("scrape promote upserted courses", zap.Int("count", len(normalizedCourses))) + + log.Info("scrape promote loading course ids") + courseIDs, err := loadCourseIDs(ctx, qtx) + if err != nil { + return err + } + log.Info("scrape promote loaded course ids", zap.Int("count", len(courseIDs))) + + scheduleTeacherNames := collectScheduleTeacherNames(schedules) + log.Info("scrape promote upserting teachers", + zap.Int("rmp_teachers", len(teachers)), + zap.Int("schedule_teacher_names", len(scheduleTeacherNames)), + ) + if err := promoteTeachers(ctx, qtx, teachers, scheduleTeacherNames); err != nil { + return err + } + log.Info("scrape promote upserted teachers") + + log.Info("scrape promote loading teacher ids") + teacherIDs, err := loadTeacherIDsByName(ctx, qtx) + if err != nil { + return err + } + log.Info("scrape promote loaded teacher ids", zap.Int("count", len(teacherIDs))) + + log.Info("scrape promote upserting programs", zap.Int("count", len(programs))) + if err := promotePrograms(ctx, qtx, programs, courseIDs); err != nil { + return err + } + log.Info("scrape promote upserted programs", zap.Int("count", len(programs))) + + log.Info("scrape promote linking course teachers") + if err := promoteCourseTeacherLinks(ctx, qtx, teachers, schedules, courseIDs, teacherIDs, titleCodeMap); err != nil { + return err + } + log.Info("scrape promote linked course teachers") + + log.Info("scrape promote deleting sections for terms", zap.Strings("terms", terms)) + if err := deleteSectionsForTerms(ctx, qtx, terms); err != nil { + return err + } + log.Info("scrape promote deleted sections for terms") + + log.Info("scrape promote upserting sections", zap.Int("schedule_courses", len(schedules))) + if err := promoteSections(ctx, qtx, schedules, courseIDs, teacherIDs, titleCodeMap); err != nil { + return err + } + log.Info("scrape promote upserted sections") + + log.Info("scrape promote linking teacher programs") + if err := promoteTeacherProgramLinks(ctx, qtx); err != nil { + return err + } + log.Info("scrape promote linked teacher programs") + return nil +} + +func clearRunStaging(ctx context.Context, q *db.Queries, runID uuid.UUID) error { + if err := q.ClearScrapeRunStagedSchedules(ctx, runID); err != nil { + return err + } + if err := q.ClearScrapeRunStagedTeachers(ctx, runID); err != nil { + return err + } + if err := q.ClearScrapeRunStagedPrograms(ctx, runID); err != nil { + return err + } + return q.ClearScrapeRunStagedCourses(ctx, runID) +} + +func loadPayloads[T any](ctx context.Context, list func(context.Context, uuid.UUID) ([][]byte, error), runID uuid.UUID) ([]T, error) { + payloads, err := list(ctx, runID) + if err != nil { + return nil, err + } + items := make([]T, 0, len(payloads)) + for _, payload := range payloads { + var item T + if err := json.Unmarshal(payload, &item); err != nil { + return nil, err + } + items = append(items, item) + } + return items, nil +} + +func promoteCourses(ctx context.Context, q *db.Queries, courses []rawCoursePayload) ([]normalizedCourseRecord, map[string]string, error) { + records := make([]normalizedCourseRecord, 0, len(courses)) + titleCodeMap := make(map[string]string, len(courses)) + for _, course := range courses { + record, err := normalizeCourseRecord(course) + if err != nil { + return nil, nil, err + } + records = append(records, record) + if record.Name != "" { + titleCodeMap[record.Name] = record.Code + } + var level pgtype.Int4 + if record.LevelNumber != nil { + level = pgtype.Int4{Int32: *record.LevelNumber, Valid: true} + } + if err := q.UpsertScrapeCourse(ctx, db.UpsertScrapeCourseParams{ + Code: record.Code, + Name: record.Name, + Description: record.Description, + Restrictions: record.Restrictions, + Prerequisites: record.Prerequisites, + Units: record.Units, + LevelNumber: level, + }); err != nil { + return nil, nil, err + } + } + return records, titleCodeMap, nil +} + +func promoteTeachers(ctx context.Context, q *db.Queries, teachers []rawTeacherPayload, scheduleTeacherNames map[string]struct{}) error { + knownNames := make(map[string]struct{}, len(teachers)) + for _, teacher := range teachers { + name := strings.TrimSpace(teacher.Name) + if name == "" { + continue + } + rmpID := strings.TrimSpace(teacher.ID) + if rmpID == "" { + return errors.New("teacher id is required") + } + department := strings.TrimSpace(teacher.Department) + if department == "" { + department = "Unknown" + } + if err := q.UpsertScrapeTeacher(ctx, db.UpsertScrapeTeacherParams{ + Name: name, + AvgRating: numericFromFloat64(teacher.AvgRating), + AvgDifficulty: numericFromFloat64(teacher.AvgDifficulty), + Department: department, + RmpID: rmpID, + NumRatings: teacher.NumRatings, + }); err != nil { + return err + } + knownNames[strings.ToLower(name)] = struct{}{} + } + + for name := range scheduleTeacherNames { + if _, ok := knownNames[strings.ToLower(name)]; ok { + continue + } + rmpID := nonRMPID(name) + if err := q.UpsertScrapeTeacher(ctx, db.UpsertScrapeTeacherParams{ + Name: name, + AvgRating: numericFromFloat64(0), + AvgDifficulty: numericFromFloat64(0), + Department: "Unknown", + RmpID: rmpID, + NumRatings: 0, + }); err != nil { + return err + } + } + return nil +} + +func promotePrograms(ctx context.Context, q *db.Queries, programs []rawProgramPayload, courseIDs map[string]int64) error { + for _, program := range programs { + name := strings.TrimSpace(program.ProgramName) + if name == "" { + return errors.New("program_name is required") + } + requirementsByLevel := program.RequirementsByLevel + if len(requirementsByLevel) == 0 { + requirementsByLevel = json.RawMessage(`[]`) + } + requirementCodes := normalizeProgramRequirementCodes(program.Requirements) + programID, err := q.UpsertScrapeProgram(ctx, db.UpsertScrapeProgramParams{ + ProgramName: name, + SourceUrl: nullableText(program.URL), + RequirementCodes: requirementCodes, + RequirementsByLevel: []byte(requirementsByLevel), + }) + if err != nil { + return err + } + if err := q.DeleteScrapeProgramCourses(ctx, programID); err != nil { + return err + } + for _, code := range requirementCodes { + courseID, ok := courseIDs[code] + if !ok { + continue + } + if err := q.InsertScrapeProgramCourse(ctx, db.InsertScrapeProgramCourseParams{ + ProgramID: programID, + CourseID: courseID, + }); err != nil { + return err + } + } + } + return nil +} + +func promoteCourseTeacherLinks(ctx context.Context, q *db.Queries, teachers []rawTeacherPayload, schedules []rawScheduleCoursePayload, courseIDs map[string]int64, teacherIDs map[string]int32, titleCodeMap map[string]string) error { + for _, teacher := range teachers { + teacherID, ok := teacherIDs[strings.TrimSpace(teacher.Name)] + if !ok { + continue + } + for _, code := range teacher.Courses { + courseID, ok := courseIDs[strings.TrimSpace(code)] + if !ok { + continue + } + if err := q.InsertScrapeCourseTeacher(ctx, db.InsertScrapeCourseTeacherParams{ + CourseID: courseID, + TeacherID: teacherID, + }); err != nil { + return err + } + } + } + + for _, course := range schedules { + courseID, ok := courseIDs[resolveScheduleCourseCode(course, titleCodeMap)] + if !ok { + continue + } + for _, section := range course.Sections { + for _, name := range getAllInstructorNames(section) { + teacherID, ok := teacherIDs[name] + if !ok { + continue + } + if err := q.InsertScrapeCourseTeacher(ctx, db.InsertScrapeCourseTeacherParams{ + CourseID: courseID, + TeacherID: teacherID, + }); err != nil { + return err + } + } + } + } + return nil +} + +func deleteSectionsForTerms(ctx context.Context, q *db.Queries, terms []string) error { + if len(terms) == 0 { + return nil + } + return q.DeleteScrapeSectionsForTerms(ctx, terms) +} + +func promoteSections(ctx context.Context, q *db.Queries, schedules []rawScheduleCoursePayload, courseIDs map[string]int64, teacherIDs map[string]int32, titleCodeMap map[string]string) error { + for _, course := range schedules { + term := normalizeTerm(course.Term) + courseID, ok := courseIDs[resolveScheduleCourseCode(course, titleCodeMap)] + if !ok { + continue + } + insertedSections := make([]normalizedSection, 0, len(course.Sections)) + for _, section := range course.Sections { + name, sectionType := parseSectionName(section.SectionName) + if name == "" { + continue + } + mode, isInPerson := detectDeliveryMode(section) + sectionID, err := q.CreateScrapeSection(ctx, db.CreateScrapeSectionParams{ + CourseID: courseID, + Name: name, + Type: sectionType, + Term: term, + Mode: mode, + IsInPerson: isInPerson, + }) + if err != nil { + return err + } + for _, detail := range section.Details { + startTime, err := parseNullableScrapeTime(detail.StartTime) + if err != nil { + return err + } + endTime, err := parseNullableScrapeTime(detail.EndTime) + if err != nil { + return err + } + building, room := parseScrapeLocation(detail.Room) + if err := q.CreateScrapeSectionMeeting(ctx, db.CreateScrapeSectionMeetingParams{ + SectionID: sectionID, + Days: strings.TrimSpace(detail.Days), + StartTime: startTime, + EndTime: endTime, + Building: building, + Room: room, + }); err != nil { + return err + } + } + linkedTeacherIDs := make(map[int32]struct{}) + for _, instructor := range getAllInstructorNames(section) { + teacherID, ok := teacherIDs[instructor] + if !ok { + continue + } + if _, seen := linkedTeacherIDs[teacherID]; seen { + continue + } + linkedTeacherIDs[teacherID] = struct{}{} + if err := q.InsertScrapeSectionTeacher(ctx, db.InsertScrapeSectionTeacherParams{ + SectionID: sectionID, + TeacherID: teacherID, + }); err != nil { + return err + } + } + insertedSections = append(insertedSections, normalizedSection{ + ID: sectionID, + Type: sectionType, + InstructorSet: getSectionInstructorSet(section), + }) + } + for _, ref := range buildSectionReferences(insertedSections) { + if err := q.InsertScrapeSectionReference(ctx, db.InsertScrapeSectionReferenceParams{ + ParentSectionID: ref.ParentID, + ChildSectionID: ref.ChildID, + }); err != nil { + return err + } + } + } + return nil +} + +func promoteTeacherProgramLinks(ctx context.Context, q *db.Queries) error { + return q.InsertScrapeTeacherProgramLinks(ctx) +} + +func loadCourseIDs(ctx context.Context, q *db.Queries) (map[string]int64, error) { + rows, err := q.ListScrapeCourseIDs(ctx) + if err != nil { + return nil, err + } + ids := make(map[string]int64, len(rows)) + for _, row := range rows { + ids[row.Code] = row.ID + } + return ids, nil +} + +func loadTeacherIDsByName(ctx context.Context, q *db.Queries) (map[string]int32, error) { + rows, err := q.ListScrapeTeacherIDsByName(ctx) + if err != nil { + return nil, err + } + ids := make(map[string]int32, len(rows)) + for _, row := range rows { + ids[row.Name] = row.ID + } + return ids, nil +} diff --git a/server/internal/service/scraping/scrape_ingest_stage.go b/server/internal/service/scraping/scrape_ingest_stage.go new file mode 100644 index 0000000..c12cbb0 --- /dev/null +++ b/server/internal/service/scraping/scrape_ingest_stage.go @@ -0,0 +1,127 @@ +package scraping + +import ( + "context" + "encoding/json" + "errors" + "strings" + + "github.com/google/uuid" + "github.com/twitocode/pathweave/go-api/internal/db" +) + +func (s *ScrapeIngestService) StageArtifacts(ctx context.Context, runID uuid.UUID, req StageScrapeArtifactsRequest) (StageScrapeArtifactsResult, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return StageScrapeArtifactsResult{}, err + } + defer func() { _ = tx.Rollback(ctx) }() + + qtx := s.db.WithTx(tx) + if err := clearRunStaging(ctx, qtx, runID); err != nil { + return StageScrapeArtifactsResult{}, err + } + + for _, course := range req.Courses { + record, err := normalizeCourseRecord(course) + if err != nil { + return StageScrapeArtifactsResult{}, err + } + payload, err := json.Marshal(course) + if err != nil { + return StageScrapeArtifactsResult{}, err + } + if err := qtx.StageCoursePayload(ctx, db.StageCoursePayloadParams{ + RunID: runID, + CourseCode: record.Code, + Payload: payload, + }); err != nil { + return StageScrapeArtifactsResult{}, err + } + } + + for _, program := range req.Programs { + name := strings.TrimSpace(program.ProgramName) + if name == "" { + return StageScrapeArtifactsResult{}, errors.New("program_name is required") + } + payload, err := json.Marshal(program) + if err != nil { + return StageScrapeArtifactsResult{}, err + } + if err := qtx.StageProgramPayload(ctx, db.StageProgramPayloadParams{ + RunID: runID, + ProgramName: name, + Payload: payload, + }); err != nil { + return StageScrapeArtifactsResult{}, err + } + } + + for _, teacher := range req.Teachers { + rmpID := strings.TrimSpace(teacher.ID) + if rmpID == "" { + return StageScrapeArtifactsResult{}, errors.New("teacher id is required") + } + payload, err := json.Marshal(teacher) + if err != nil { + return StageScrapeArtifactsResult{}, err + } + if err := qtx.StageTeacherPayload(ctx, db.StageTeacherPayloadParams{ + RunID: runID, + RmpID: rmpID, + Payload: payload, + }); err != nil { + return StageScrapeArtifactsResult{}, err + } + } + + terms := make(map[string]struct{}) + for _, termPayload := range req.Schedules { + term := normalizeTerm(termPayload.Term) + if term == "Unknown" { + return StageScrapeArtifactsResult{}, errors.New("schedule term is required") + } + terms[term] = struct{}{} + for _, course := range termPayload.Courses { + if course.Term == "" { + course.Term = term + } + courseCode := strings.TrimSpace(course.CourseCode) + if courseCode == "" { + return StageScrapeArtifactsResult{}, errors.New("schedule course_code is required") + } + payload, err := json.Marshal(course) + if err != nil { + return StageScrapeArtifactsResult{}, err + } + if err := qtx.StageSchedulePayload(ctx, db.StageSchedulePayloadParams{ + RunID: runID, + Term: term, + CourseCode: courseCode, + Payload: payload, + }); err != nil { + return StageScrapeArtifactsResult{}, err + } + } + } + + result := StageScrapeArtifactsResult{ + RunID: runID, + CourseCount: len(req.Courses), + ProgramCount: len(req.Programs), + TeacherCount: len(req.Teachers), + ScheduleTermCount: len(terms), + } + if err := qtx.MarkScrapeRunStaged(ctx, db.MarkScrapeRunStagedParams{ + ID: runID, + CourseCount: int32(result.CourseCount), + ProgramCount: int32(result.ProgramCount), + TeacherCount: int32(result.TeacherCount), + ScheduleTermCount: int32(result.ScheduleTermCount), + }); err != nil { + return StageScrapeArtifactsResult{}, err + } + + return result, tx.Commit(ctx) +} diff --git a/server/internal/service/scraping/scrape_ingest_test.go b/server/internal/service/scraping/scrape_ingest_test.go new file mode 100644 index 0000000..523aec0 --- /dev/null +++ b/server/internal/service/scraping/scrape_ingest_test.go @@ -0,0 +1,51 @@ +package scraping + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestScrapeIngestNormalizeCourseRecord(t *testing.T) { + record, err := normalizeCourseRecord(rawCoursePayload{ + CourseName: "COMPSCI 1DM3 - Discrete Mathematics for Computer Science", + Units: "3 unit(s)", + Description: "Sets and logic", + Restrictions: "Antirequisite(s): MATH 1DM3", + Prerequisites: []string{"MATH 1ZA3"}, + }) + require.NoError(t, err) + require.Equal(t, "COMPSCI 1DM3", record.Code) + require.Equal(t, "Discrete Mathematics for Computer Science", record.Name) + require.Equal(t, int32(3), record.Units) + require.NotNil(t, record.LevelNumber) + require.Equal(t, int32(1), *record.LevelNumber) +} + +func TestScrapeIngestParseMeetingDetails(t *testing.T) { + start, err := parseScrapeTime("4:30PM") + require.NoError(t, err) + require.Equal(t, "16:30:00", start) + + building, room := parseScrapeLocation("BSB B156") + require.Equal(t, "BSB", building) + require.Equal(t, "B156", room) + + building, room = parseScrapeLocation("Online") + require.Equal(t, "Online", building) + require.Equal(t, "Online", room) +} + +func TestScrapeIngestBuildSectionReferences(t *testing.T) { + sections := []normalizedSection{ + {ID: 1, Type: "LEC", InstructorSet: map[string]struct{}{"Jane Doe": {}}}, + {ID: 2, Type: "LAB", InstructorSet: map[string]struct{}{"Jane Doe": {}}}, + {ID: 3, Type: "TUT", InstructorSet: map[string]struct{}{}}, + {ID: 4, Type: "LAB", InstructorSet: map[string]struct{}{"Other Person": {}}}, + } + + refs := buildSectionReferences(sections) + require.Len(t, refs, 2) + require.Equal(t, sectionReference{ParentID: 1, ChildID: 2}, refs[0]) + require.Equal(t, sectionReference{ParentID: 1, ChildID: 3}, refs[1]) +}