Add program requirements parsing and optional-course shelf (backend + frontend)

app.py

@@ -42,6 +42,38 @@ def inject_supabase_config():
         supabase_key=SUPABASE_KEY # important to pass the key and not service_role_key
     )
 
+
+
+def rebuild_requirements_from_details(details):
+    buckets = {}
+    for code, course in (details or {}).items():
+        bucket_id = course.get("requirement_bucket")
+        if not bucket_id:
+            continue
+        bucket = buckets.setdefault(bucket_id, {
+            "id": bucket_id,
+            "title": course.get("requirement_bucket_title") or bucket_id,
+            "category": course.get("category", "CORE"),
+            "min_credits": course.get("requirement_min_credits", 0),
+            "max_credits": course.get("requirement_max_credits"),
+            "courses": [],
+        })
+        bucket["courses"].append(code)
+    return {"buckets": list(buckets.values())}
+
+
+def merge_program_requirements(current, incoming):
+    merged = dict(current or {})
+    existing_buckets = {bucket.get("id"): bucket for bucket in merged.get("buckets", [])}
+    for bucket in (incoming or {}).get("buckets", []):
+        bucket_id = bucket.get("id")
+        if not bucket_id or bucket_id in existing_buckets:
+            continue
+        existing_buckets[bucket_id] = bucket
+    merged["buckets"] = list(existing_buckets.values())
+    merged["course_to_bucket"] = {**(current or {}).get("course_to_bucket", {}), **(incoming or {}).get("course_to_bucket", {})}
+    return merged
+
 courses_info = {}
 with open('static/json/courses_info.json', 'r', encoding='utf-8') as f:
     courses_info = json.load(f)
@@ -89,6 +121,7 @@ def scrape_form():
                 #Saving to session
                 session['prereqs_data'] = prereqs_data
                 session['details_data'] = details_data
+                session['program_requirements'] = rebuild_requirements_from_details(details_data)
                 session['graph_data_available'] = True
 
                 return redirect("graph")
@@ -97,11 +130,23 @@ def scrape_form():
     major = request.args.get("programSearch")
 
     if action == "Visualize Program": 
-        courses_prereqs_data, processed_details_data = get_information_for_major.process_program_data(url, major) #scrape the major's site and grab all info
+        program_data = get_information_for_major.process_program_data(url, major)
+        if not isinstance(program_data, tuple):
+            return render_template('scrape_form.html', error="Unable to scrape that program. Please try again later.")
+
+        if len(program_data) == 2:
+            courses_prereqs_data, processed_details_data = program_data
+            requirements_data = {}
+        else:
+            courses_prereqs_data, processed_details_data, requirements_data = program_data #scrape the major's site and grab all info
+        if courses_prereqs_data is None or processed_details_data is None:
+            return render_template('scrape_form.html', error="Unable to scrape that program. Please try again later.")
 
         #Saving the variable to session
         session['prereqs_data'] = courses_prereqs_data
         session['details_data'] = processed_details_data
+        session['program_requirements'] = requirements_data or {}
+        session['credit_requirements'] = (requirements_data or {}).get('credit_requirements', {'core': 0, 'comp': 0, 'elec': 0})
         session['graph_data_available'] = True
 
         return redirect("graph")
@@ -113,6 +158,7 @@ def scrape_form():
             session.pop('prereqs_data', None)
             session.pop('details_data', None)
             session.pop('graph_data_available', None)
+            session.pop('program_requirements', None)
             # This is an initial GET request to the form
             return render_template('scrape_form.html')
         else:
@@ -127,7 +173,8 @@ def graph():
         prereqs = session.get('prereqs_data', {})
         details = session.get('details_data', {})
         logging.log_entry(request, "displaying graph")
-        return render_template('graph.html', prereqs=prereqs, details=details)
+        requirements = session.get('program_requirements') or rebuild_requirements_from_details(details)
+        return render_template('graph.html', prereqs=prereqs, details=details, requirements=requirements)
     else:
         # If no data, redirect back to home page
         return redirect(url_for('scrape_form'))
@@ -151,7 +198,14 @@ def add_program_to_graph():
         logging.log_entry(request, f"adding program {new_program_name} (url: {new_program_url}) to graph")    
 
         # Fetch and process data for the new program
-        new_prereqs, new_details = get_information_for_major.process_program_data(new_program_url, new_program_name)
+        new_program_data = get_information_for_major.process_program_data(new_program_url, new_program_name)
+        if len(new_program_data) == 2:
+            new_prereqs, new_details = new_program_data
+            new_requirements = {}
+        else:
+            new_prereqs, new_details, new_requirements = new_program_data
+        if new_prereqs is None or new_details is None:
+            raise ValueError("Unable to scrape that program.")
 
         # Retrieve current graph data from session
         current_prereqs = session.get('prereqs_data', {})
@@ -174,14 +228,16 @@ def add_program_to_graph():
 
         session['details_data'].update(new_details)
         session['prereqs_data'].update(new_prereqs)
+        session['program_requirements'] = merge_program_requirements(session.get('program_requirements', {}), new_requirements or {})
         session['graph_data_available'] = True
 
         # NEW: If JavaScript asked for this, send the raw JSON data back!
         if is_ajax:
             return jsonify({
                 "status": "success", 
                 "new_details": new_details, 
-                "new_prereqs": new_prereqs
+                "new_prereqs": new_prereqs,
+                "new_requirements": new_requirements or {}
             })
 
         return redirect(url_for('graph'))
@@ -394,6 +450,7 @@ def load_graph():
             # Load the data into the Flask session
             session['prereqs_data'] = graph.get('prereqs_data', {})
             session['details_data'] = graph.get('details_data', {})
+            session['program_requirements'] = rebuild_requirements_from_details(session['details_data'])
             session['schedule_id'] = graph.get('id')
             session['graph_data_available'] = True
 

scripts/Getting_Info_For_Major/get_courses_of_major.py

@@ -1,24 +1,85 @@
+import re
 import requests
 from bs4 import BeautifulSoup
 
-#Given url of the major, it scrapes the site and gets all the courses of the major.
-#Eventually this could be ran on all programs and be hard-coded instead of running on every query
+# Given url of the major, it scrapes the site and gets all the courses of the major.
+# Eventually this could be run on all programs and be hard-coded instead of running on every query.
 
-def get_program_codes(url):
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+}
+
+COURSE_CODE_RE = re.compile(r"\b[A-Z]{3,5}\s?\d{3}[A-Z0-9]?\b")
+
+
+def get_program_soup(url):
+    # Passing a header to mimic a real user instead.
+    res = requests.get(url, headers=HEADERS)
+    res.raise_for_status()
+    return BeautifulSoup(res.text, "html.parser")
 
-    #Passing a header to mimic a real user instead
-    headers = {
-        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-    }
 
-    res = requests.get(url, headers=headers)
-    soup = BeautifulSoup(res.text, 'html.parser')
+def get_program_codes(url, soup=None):
+    soup = soup or get_program_soup(url)
 
-    #finding all required courses: 
+    # On McGill pages, program course rows use <td class="codecol">COMP 250</td>.
     course_codes = soup.find_all("td", class_="codecol")
-    course_codes = [code.get_text(strip=True) for code in course_codes]
-
-    return course_codes
+    return [code.get_text(strip=True) for code in course_codes]
+
+
+def get_course_metadata_from_program_page(code, soup):
+    """Best-effort extraction of title/credits/terms from a program page table row.
+
+    This is only a fallback for courses missing from static/json/courses_info.json.
+    It does NOT replace the Gemini prerequisite pipeline.
+    """
+    # Locate the correct code cell
+    code_cell = soup.find("td", class_="codecol", string=lambda txt: txt and code in txt)
+    if not code_cell:
+        for candidate in soup.find_all("td", class_="codecol"):
+            if candidate.get_text(strip=True) == code:
+                code_cell = candidate
+                break
 
-
-
+    if not code_cell:
+        return None
+
+    # Get the parent row of the code cell
+    row = code_cell.find_parent("tr")
+    if not row:
+        return None
+
+    title = ""
+    credits = "N/A"
+    semesters_offered = "Unknown"
+
+    # Extract Title
+    title_cell = code_cell.find_next_sibling("td")
+    if title_cell:
+        title = title_cell.get_text(strip=True).rstrip(".")
+
+    # Extract Credits
+    if title_cell:
+        hours_cell = title_cell.find_next_sibling("td")
+        if hours_cell:
+            hours_text = hours_cell.get_text(strip=True)
+            credit_match = re.search(r"(\d+(?:\.\d+)?)", hours_text)
+            if credit_match:
+                credits = credit_match.group(1)
+
+    # Extract Semesters Offered from the adjacent bubbledrawer row
+    # find_next_sibling find the next <tr> tag directly below the current course row
+    drawer_row = row.find_next_sibling("tr", class_="bubbledrawer")
+    if drawer_row:
+        terms_paragraph = drawer_row.find("p", class_="bubbledrawer__terms")
+        if terms_paragraph:
+            # Extract text and strip out the "Terms offered:" prefix label
+            raw_terms = terms_paragraph.get_text(" ", strip=True)
+            semesters_offered = raw_terms.replace("Terms offered:", "").strip().strip('"')
+
+    return {
+        "code": code,
+        "title": title or f"{code} (Details not found)",
+        "credits": credits,
+        "semesters_offered": semesters_offered,
+    }

scripts/Getting_Info_For_Major/get_information_for_major.py

@@ -1,6 +1,7 @@
-from scripts.Getting_Info_For_Major import get_courses_of_major, get_prereqs
+from scripts.Getting_Info_For_Major import get_courses_of_major, get_prereqs, program_requirements
 from scripts import utils
 import json
+import re
 
 #Given the link from provided from the programs.json file, it grabs all the courses of the major, and compiles all the info for each course.
 
@@ -15,10 +16,25 @@
 
 def process_program_data(program_url, major):
     try:
-        course_codes = get_courses_of_major.get_program_codes(program_url)
+        program_soup = get_courses_of_major.get_program_soup(program_url)
+        course_codes = get_courses_of_major.get_program_codes(program_url, soup=program_soup)
+        requirements_data = program_requirements.extract_program_requirements(program_soup)
+        major_slug = re.sub(r"[^a-z0-9]+", "-", (major or "program").lower()).strip("-") or "program"
+        for idx, bucket in enumerate(requirements_data.get("buckets", []), start=1):
+            bucket["program_name"] = major
+            bucket_id = bucket.get("id") or f"bucket-{idx}"
+            if not str(bucket_id).startswith(f"{major_slug}-"):
+                bucket["id"] = f"{major_slug}-{bucket_id}"
+
+        requirements_data["course_to_bucket"] = {
+            code: (f"{major_slug}-{bucket_id}" if bucket_id and not str(bucket_id).startswith(f"{major_slug}-") else bucket_id)
+            for code, bucket_id in requirements_data.get("course_to_bucket", {}).items()
+        }
+        rule_texts=[b.get("constraints_text") for b in requirements_data.get("buckets",[]) if b.get("constraints_text")]
+        requirements_data["parsed_rules"] = get_prereqs.parse_requirement_rules(major, rule_texts) if rule_texts else []
         if not course_codes:
             print(f"No course codes found for URL: {program_url}")
-            return None, None
+            return None, None, None
 
         course_details_data = {}
         gemini_data = {}
@@ -37,8 +53,8 @@ def process_program_data(program_url, major):
                 }
             else:
                 print(f"Warning: Course code {code} from program {major} not found in global courses_info.json")
-                # Add a placeholder if not found in global courses_info
-                course_details_data[code] = {
+                fallback_metadata = get_courses_of_major.get_course_metadata_from_program_page(code, program_soup)
+                course_details_data[code] = fallback_metadata or {
                     "title": f"{code} (Details not found)",
                     "credits": "N/A",
                     "semesters_offered": "Unknown",
@@ -49,6 +65,9 @@ def process_program_data(program_url, major):
         courses_prereqs_data = {}
         if gemini_data:
             courses_prereqs_data = get_prereqs.get_prereq_list(major, gemini_data)
+            if not isinstance(courses_prereqs_data, dict):
+                print(f"Warning: Gemini prereq response was not a dict for program {major}. Got type={type(courses_prereqs_data).__name__}")
+                courses_prereqs_data = {}
         else:
             print(f"No data prepared for Gemini for program: {major}")
 
@@ -58,8 +77,19 @@ def process_program_data(program_url, major):
         for code, details in course_details_data.items():
             default_detail = {"title": "Unknown Title", "credits": "N/A", "semesters_offered": "Unknown"}
             actual_details = {**default_detail, **details}
+            bucket_id = requirements_data.get("course_to_bucket", {}).get(code)
+            bucket = next(
+                (bucket for bucket in requirements_data.get("buckets", []) if bucket.get("id") == bucket_id),
+                None,
+            )
             course_details_full[code] = {
                 **actual_details,
+                "category": "CORE" if (bucket and bucket.get("category") == "CORE") else "COMPLEMENTARY",
+                "requirement_bucket": bucket_id,
+                "requirement_bucket_title": bucket.get("title") if bucket else None,
+                "requirement_min_credits": bucket.get("min_credits") if bucket else 0,
+                "requirement_max_credits": bucket.get("max_credits") if bucket else None,
+                "include_in_graph": True if (bucket and bucket.get("category") == "CORE") else False,
                 "color": utils.parse_semester_to_color(actual_details.get("semesters_offered", ""))
             }
 
@@ -69,7 +99,7 @@ def process_program_data(program_url, major):
                 if course_code_prereq in courses_info:
                     course_details_full[course_code_prereq] = {
 
-                            "title": f"{courses_info[course_code_prereq]["Title"]} (Prereq)",
+                            "title": f"{courses_info[course_code_prereq]['Title']} (Prereq)",
                             "credits": courses_info[course_code_prereq]["Credits"],
                             "semesters_offered": courses_info[course_code_prereq]["Terms_Offered"],
                             "color": "LightSteelBlue"
@@ -90,7 +120,7 @@ def process_program_data(program_url, major):
                 if prereq_item not in course_details_full:
                     if prereq_item in courses_info:
                         course_details_full[prereq_item] = {
-                            "title": f"{courses_info[prereq_item]["Title"]} (Prereq)",
+                            "title": f"{courses_info[prereq_item]['Title']} (Prereq)",
                             "credits": courses_info[prereq_item]["Credits"],
                             "semesters_offered": courses_info[prereq_item]["Terms_Offered"],
                             "color": "LightSteelBlue"
@@ -99,7 +129,7 @@ def process_program_data(program_url, major):
                         course_details_full[prereq_code] = {"title": "Details Not Found (Prereq)", "credits": "N/A", "semesters_offered": "Unknown", "color": "LightSteelBlue"}
 
 
-        return courses_prereqs_data, course_details_full
+        return courses_prereqs_data, course_details_full, requirements_data
     except Exception as e:
         print(f"Error in process_program_data for {program_url}: {e}")
-        return None, None
+        return None, None, None

scripts/Getting_Info_For_Major/get_prereqs.py

@@ -30,7 +30,7 @@ def get_prereq_list(major_name, major_courses_data):
                 contents=prompt
             )
         except Exception as e: 
-            return "There was an error with contacting Gemini"
+            return {}
 
         text_response = response.text.strip()
 
@@ -45,4 +45,26 @@ def get_prereq_list(major_name, major_courses_data):
         resolved_prereqs = json.loads(text_response.replace("'", '"'))
         return resolved_prereqs
     except Exception as e:
-        return f"Error: {e}\nResponse from Gemini: {response}"
+        return {}
+
+
+def parse_requirement_rules(major_name, rule_texts):
+    load_dotenv()
+    GEMINI_API_KEY = os.getenv("GEMINI_API_KEY")
+    client = genai.Client(api_key=GEMINI_API_KEY)
+    prompt = f"""
+    For McGill program '{major_name}', convert these plain-English degree rules into strict JSON array.
+    Output ONLY JSON. Schema per item: {{"rule_text": str, "allowed_prefixes": [str], "min_level": int|null, "exclude_courses": [str], "must_have_credits_at_level": {{"level": int, "credits": int}}|null}}
+    Rules:
+    {json.dumps(rule_texts)}
+    """
+    try:
+        resp = client.models.generate_content(model='gemini-2.5-flash-lite', contents=prompt)
+        txt = resp.text.strip()
+        if txt.startswith("```json"):
+            txt = txt[len("```json"):].strip()
+        if txt.endswith("```"):
+            txt = txt[:-3].strip()
+        return json.loads(txt)
+    except Exception:
+        return []