Executor: Cross-year student ID matching

executor/executor/artifacts.py

@@ -8,10 +8,10 @@
 import executor.config as config
 
 class JobArtifact:
-    def __init__(self, name, path):
+    def __init__(self, name, path, needs_upload=True):
         self.name = name
         self.path = path
-        self.needs_upload = True
+        self.needs_upload = needs_upload
 
 
 ROSTER = JobArtifact(
@@ -22,17 +22,27 @@ def __init__(self, name, path):
     "earthmover_results",
     "em-results.json"
 )
+# Only generated when earthmover runs for a second time as part of cross-year ID matching.
+# If a cross-year match is performed as part of a sideload-only job, EM_RESULTS is used alone
+EM_RESULTS_X_YEAR = JobArtifact(
+    "earthmover_results_x_year",
+    "em-results-x-year.json",
+    False
+)
 MATCH_RATES = JobArtifact(
     "student_id_match_rates",
     os.path.join(config.OUTPUT_DIR, ("student_id_match_rates.csv")),
 )
+# Uploading the unmatched students file impacts the UX; we must be careful to only do so
+# in the event of an otherwise successful run that has a tolerable number of unmatched input records
 UNMATCHED_STUDENTS = JobArtifact(
     "unmatched_students",
     os.path.join(config.OUTPUT_DIR, "input_no_student_id_match.csv"),
+    False
 )
 LB_SEND_RESULTS = JobArtifact(
     "lightbeam_send_results",
     "lb-send-results.json"
 )
 
-ALL = [ROSTER, EM_RESULTS, MATCH_RATES, UNMATCHED_STUDENTS, LB_SEND_RESULTS]
+ALL = [ROSTER, EM_RESULTS, EM_RESULTS_X_YEAR, MATCH_RATES, UNMATCHED_STUDENTS, LB_SEND_RESULTS]

executor/executor/config.py

@@ -1,6 +1,9 @@
 BUNDLE_DIR = 'bundles'
 OUTPUT_DIR = 'output'
+# when performing a cross-year-matching second pass: location to move earthmover output set from the first pass
+OUTPUT_DIR_FIRST_RUN = 'output-first-run'
 ROSTER_DOWNLOAD_DIR = 'roster-download-dir'
+CROSS_YEAR_ROSTER_PATH = 'cross_year_roster.jsonl'
 
 REQUIRED_ID_MATCH_RATE = 0.5
 STUDENT_ASSESSMENT_FAIL_THRESHOLD = 0.75

executor/executor/errors.py

@@ -80,6 +80,10 @@ class EarthmoverRunError(ExecutorError):
     def __init__(self, stacktrace=None):
         super().__init__("earthmover_run", stacktrace)
 
+class CrossYearRosterFetchError(ExecutorError):
+    def __init__(self, stacktrace=None):
+        super().__init__("cross_year_roster_fetch", stacktrace)
+
 class InsufficientMatchesError(ExecutorError):
     def __init__(self, match_rate, match_threshold, id_name, id_type, stacktrace=None):
         self.match_rate = match_rate