TalentMatch/process_data.py at main · Sneha-Amballa/TalentMatch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
import os
import shutil
import pandas as pd

def main():
    # Setup folders
    print("Setting up directories...")
    os.makedirs("data/raw/resumes", exist_ok=True)
    os.makedirs("data/raw/jobs", exist_ok=True)
    os.makedirs("data/processed", exist_ok=True)

    # Move raw datasets if they exist in the root folder
    def move_file(src, dest):
        if os.path.exists(src):
            if os.path.exists(dest):
                print(f"Destination '{dest}' already exists. Attempting to remove redundant root file '{src}'...")
                try:
                    os.remove(src)
                except Exception as e:
                    print(f"Could not remove root file '{src}': {e}. Skipping.")
            else:
                print(f"Moving '{src}' to '{dest}'...")
                try:
                    shutil.move(src, dest)
                except Exception as e:
                    print(f"Error moving file: {e}")
        else:
            print(f"'{src}' not found in root (already moved or missing).")

    move_file("Resume.csv", "data/raw/resumes/Resume.csv")
    move_file("postings.csv", "data/raw/jobs/postings.csv")
    move_file("job_skills.csv", "data/raw/jobs/job_skills.csv")

    # Step 1: Resume dataset cleaning
    print("\n--- Step 1: Processing resumes dataset ---")
    resume_path = "data/raw/resumes/Resume.csv"
    if os.path.exists(resume_path):
        print(f"Reading resumes from {resume_path}...")
        resumes = pd.read_csv(resume_path)
        print(f"Initial shape: {resumes.shape}")
        resumes = resumes[["ID", "Resume_str", "Category"]].dropna(subset=["Resume_str"])
        resumes.rename(columns={"ID": "resume_id", "Resume_str": "resume_text", "Category": "category"}, inplace=True)
        out_path = "data/processed/resumes_clean.csv"
        resumes.to_csv(out_path, index=False)
        print(f"Resumes cleaned. Saved to {out_path}. Shape: {resumes.shape}")
    else:
        print(f"Error: {resume_path} not found.")

    # Step 2: Postings dataset cleaning and filtering
    print("\n--- Step 2: Processing job postings dataset ---")
    postings_path = "data/raw/jobs/postings.csv"
    if os.path.exists(postings_path):
        print(f"Reading postings from {postings_path}...")
        jobs = pd.read_csv(postings_path)
        print(f"Initial shape: {jobs.shape}")

        cols_to_keep = ["job_id", "title", "description", "location",
                        "formatted_experience_level", "formatted_work_type", "skills_desc"]

        # Verify columns exist
        missing_cols = [c for c in cols_to_keep if c not in jobs.columns]
        if missing_cols:
            print(f"Warning: columns {missing_cols} not found in postings.csv")

        jobs = jobs[cols_to_keep]

        # Drop postings with no title
        jobs = jobs.dropna(subset=["title"])
        jobs["description"] = jobs["description"].fillna("")  # description is optional

        # Filter to tech-relevant roles only
        tech_keywords = "Engineer|Developer|Software|Data|IT|Programmer|Analyst"
        jobs = jobs[jobs["title"].str.contains(tech_keywords, case=False, na=False)]
        print(f"Shape after filtering by tech keywords ({tech_keywords}): {jobs.shape}")

        # Subset for manageable size
        jobs = jobs.sample(n=min(5000, len(jobs)), random_state=42)
        print(f"Sampled shape: {jobs.shape}")

        out_jobs_path = "data/processed/jobs_clean.csv"
        jobs.to_csv(out_jobs_path, index=False)
        print(f"Jobs cleaned. Saved to {out_jobs_path}")

        # Build distinct title list for dropdown
        distinct_titles = jobs["title"].drop_duplicates().sort_values()
        out_titles_path = "data/processed/job_titles_dropdown.csv"
        distinct_titles.to_csv(out_titles_path, index=False)
        print(f"Distinct titles saved to {out_titles_path}. Count: {len(distinct_titles)}")
    else:
        print(f"Error: {postings_path} not found.")

    # Step 3: Job skills aggregation and merging
    print("\n--- Step 3: Merging job skills ---")
    job_skills_path = "data/raw/jobs/job_skills.csv"
    jobs_clean_path = "data/processed/jobs_clean.csv"
    if os.path.exists(job_skills_path) and os.path.exists(jobs_clean_path):
        print(f"Reading job skills from {job_skills_path}...")
        job_skills = pd.read_csv(job_skills_path)
        print(f"Initial skills rows: {len(job_skills)}")

        # Aggregate multiple skill rows into a single comma-separated string per job
        skills_agg = job_skills.groupby("job_id")["skill_abr"].apply(
            lambda x: ", ".join(x.dropna().unique())
        ).reset_index()
        skills_agg.rename(columns={"skill_abr": "skills"}, inplace=True)
        print(f"Aggregated unique jobs with skills: {len(skills_agg)}")

        # Merge into jobs_clean
        print(f"Merging skills into jobs_clean...")
        jobs = pd.read_csv(jobs_clean_path)
        jobs = jobs.merge(skills_agg, on="job_id", how="left")
        jobs["skills"] = jobs["skills"].fillna("")

        jobs.to_csv(jobs_clean_path, index=False)
        print(f"Final merged jobs dataset saved to {jobs_clean_path}. Shape: {jobs.shape}")
    else:
        print(f"Error: {job_skills_path} or {jobs_clean_path} not found.")

if __name__ == "__main__":
    main()