ModMatch-Orbital-Project/ModScrape.py at main · Miloepeng/ModMatch-Orbital-Project · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
import time
import re

def moduleScrape(module_code):
    # Setup headless Chrome
    options = Options()
    options.add_argument("--headless")
    options.add_argument("--no-sandbox")
    options.add_argument("--disable-dev-shm-usage")

    driver = webdriver.Chrome(options=options)

    # Load the module page
    url = f"https://nusmods.com/modules/{module_code}"
    driver.get(url)

    # Wait for iframes to load
    WebDriverWait(driver, 5).until(
        EC.presence_of_element_located((By.TAG_NAME, "iframe"))
    )

    # Find and switch to the first iframe (likely the comment one)
    iframes = driver.find_elements(By.TAG_NAME, "iframe")

    # Try the first one that looks like a comment iframe
    driver.switch_to.frame(iframes[0])  # Adjust index if needed

    # Keep clicking the "Load more comments" button until it's gone
    while True:
        try:
            load_more_button = WebDriverWait(driver, 5).until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, "a.load-more-refresh__button"))
            )
            load_more_button.click()
            time.sleep(2)
        except:
            #print("No more 'Load more' button found.")
            break

    # Wait for posts to appear inside the iframe
    WebDriverWait(driver, 5).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, "li.post"))
    )

    # Parse iframe content with BeautifulSoup
    soup = BeautifulSoup(driver.page_source, "html.parser")
    comments = soup.select("li.post")

    output = []
    ##print(f"\nFound {len(comments)} comments:\n")
    for i, comment in enumerate(comments, start=1):
        ##print(f"Comment {i}:")
        ##print(comment.get_text(strip=True))
        temp = comment.get_text(strip=True).lower()
        temp = re.sub(r'([a-z])([0-9])', r'\1 \2', temp)
        temp = re.sub(r'([0-9])([a-z])', r'\1 \2', temp)
        output.append(temp)
        ##print("-" * 50)

    driver.quit()

    answer = ""
    for item in output:
        answer += " " + item
    return answer

def descScrape(module_code):
    # Setup headless Chrome
    options = Options()
    options.add_argument("--headless")
    options.add_argument("--no-sandbox")
    options.add_argument("--disable-dev-shm-usage")

    driver = webdriver.Chrome(options=options)

    # Load the module page
    url = f"https://nusmods.com/modules/{module_code}"
    driver.get(url)

    p_element = WebDriverWait(driver, 5).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, "section.row div p")))
    #print(type(p_element.text))
    return p_element.text