DO NOT MERGE - [PERF-286] Update pipeclean to work more generically with an Ed-Fi API

.gitattributes

@@ -1,3 +1,5 @@
+* text eol=lf
+
 *.md text
 *.txt text
 *.py text

src/edfi-performance-test/edfi_performance_test/api/client/account.py

@@ -19,9 +19,8 @@ class AccountClient(EdFiAPIClient):
     }
 
     def create_with_dependencies(self, **kwargs):
-        # TODO: this is the only reference to account_code_client anywhere in
-        # the code base, so the next line should fail if it is ever called.
-        account_code_reference = self.account_code_client.create_with_dependencies()
+        # The constructor for ed_fi_api_client dynamically creates the `xyz_client` property using `setattr`
+        account_code_reference = self.account_code_client.create_with_dependencies()  # type: ignore
         account_code_attrs = account_code_reference["attributes"]
 
         edorg_id = account_code_attrs["educationOrganizationReference"][
@@ -56,10 +55,10 @@ class _AccountDependentMixin(object):
     }
 
     def create_with_dependencies(self, **kwargs):
-        account_reference = self.account_client.create_with_dependencies()
+        account_reference = self.account_client.create_with_dependencies()  # type: ignore
         account_identifier = account_reference["attributes"]["accountIdentifier"]
 
-        return self.create_using_dependencies(
+        return self.create_using_dependencies(    # type: ignore
             account_reference,
             accountReference__accountIdentifier=account_identifier,
             **kwargs

src/edfi-performance-test/edfi_performance_test/api/client/assessment.py

@@ -18,7 +18,7 @@ class AssessmentItemClient(EdFiAPIClient):
 
     def create_with_dependencies(self, **kwargs):
         # Create new assessment
-        assessment_reference = self.assessment_client.create_with_dependencies()
+        assessment_reference = self.assessment_client.create_with_dependencies()  # type: ignore
 
         # Create assessment item
         return self.create_using_dependencies(
@@ -45,7 +45,7 @@ class ObjectiveAssessmentClient(EdFiAPIClient):
 
     def create_with_dependencies(self, **kwargs):
         # Create new assessment
-        assessment_reference = self.assessment_client.create_with_dependencies()
+        assessment_reference = self.assessment_client.create_with_dependencies()  # type: ignore
 
         # Create objective assessment
         return self.create_using_dependencies(
@@ -64,7 +64,7 @@ class StudentAssessmentClient(EdFiAPIClient):
 
     def create_with_dependencies(self, **kwargs):
         # Create new assessment
-        assessment_reference = self.assessment_client.create_with_dependencies()
+        assessment_reference = self.assessment_client.create_with_dependencies() # type: ignore
 
         # Create student assessment
         return self.create_using_dependencies(

src/edfi-performance-test/edfi_performance_test/api/client/bell_schedule.py

@@ -15,7 +15,7 @@ class BellScheduleClient(EdFiAPIClient):
 
     def create_with_dependencies(self, **kwargs):
         # Create new class period
-        class_period_reference = self.class_period_client.create_with_dependencies()
+        class_period_reference = self.class_period_client.create_with_dependencies() # type: ignore
 
         # Create bell schedule
         return self.create_using_dependencies(

src/edfi-performance-test/edfi_performance_test/api/client/calendar_date.py

@@ -22,7 +22,7 @@ def create_with_dependencies(self, **kwargs):
             "calendarCode", RandomSuffixAttribute("107SS111111")
         )
         # Create a calendar
-        calendar_reference = self.calendar_client.create_with_dependencies(
+        calendar_reference = self.calendar_client.create_with_dependencies(  # type: ignore
             schoolReference__schoolId=school_id, calendarCode=custom_calendar_code
         )
 

src/edfi-performance-test/edfi_performance_test/api/client/community.py

@@ -20,7 +20,7 @@ class CommunityProviderClient(EdFiAPIClient):
     }
 
     def create_with_dependencies(self, **kwargs):
-        org_reference = self.org_client.create_with_dependencies()
+        org_reference = self.org_client.create_with_dependencies()  # type: ignore
 
         return self.create_using_dependencies(
             org_reference,
@@ -41,7 +41,7 @@ class CommunityProviderLicenseClient(EdFiAPIClient):
     }
 
     def create_with_dependencies(self, **kwargs):
-        provider_reference = self.provider_client.create_with_dependencies()
+        provider_reference = self.provider_client.create_with_dependencies()  # type: ignore
 
         return self.create_using_dependencies(
             provider_reference,

src/edfi-performance-test/edfi_performance_test/api/client/composite.py

@@ -44,5 +44,9 @@ def _get_all(self, resource):
         response = self._get_response(
             "get", list_endpoint, headers=self.get_headers(), name=list_endpoint
         )
+
+        if not response.ok:
+            raise RuntimeError(f"Composite resource '{resource}' does not exist (404)")
+
         self.log_response(response)
         return json.loads(response.text)

src/edfi-performance-test/edfi_performance_test/api/client/course.py

@@ -24,7 +24,7 @@ class CourseTranscriptClient(EdFiAPIClient):
     def create_with_dependencies(self, **kwargs):
         school_id = kwargs.pop("schoolId", SchoolClient.shared_elementary_school_id())
 
-        record_reference = self.record_client.create_with_dependencies(
+        record_reference = self.record_client.create_with_dependencies(  # type: ignore
             schoolId=school_id
         )
 

src/edfi-performance-test/edfi_performance_test/api/client/course_offering.py

@@ -15,8 +15,9 @@ class CourseOfferingClient(EdFiAPIClient):
 
     def create_with_dependencies(self, **kwargs):
         school_id = kwargs.pop("schoolId", SchoolClient.shared_elementary_school_id())
-        school_year = kwargs.get("schoolYear", 2014)
-        session_reference = self.session_client.create_with_dependencies(
+        school_year = kwargs.pop("schoolYear", 2014)
+
+        session_reference = self.session_client.create_with_dependencies(  # type: ignore
             schoolId=school_id, schoolYear=school_year
         )
 

src/edfi-performance-test/edfi_performance_test/api/client/discipline.py

@@ -21,7 +21,7 @@ def create_with_dependencies(self, **kwargs):
         school_id = kwargs.pop("schoolId", SchoolClient.shared_elementary_school_id())
 
         # Create staff
-        staff_reference = self.staff_client.create_with_dependencies(schoolId=school_id)
+        staff_reference = self.staff_client.create_with_dependencies(schoolId=school_id)  # type: ignore
 
         # Create discipline incident
         return self.create_using_dependencies(
@@ -47,7 +47,7 @@ def create_with_dependencies(self, **kwargs):
         # Create incident and association first
         school_id = kwargs.pop("schoolId", SchoolClient.shared_elementary_school_id())
 
-        assoc_reference = self.assoc_client.create_with_dependencies(schoolId=school_id)
+        assoc_reference = self.assoc_client.create_with_dependencies(schoolId=school_id)  # type: ignore
 
         # Create discipline action
         return self.create_using_dependencies(

src/edfi-performance-test/edfi_performance_test/api/client/ed_fi_api_client.py

@@ -7,7 +7,7 @@
 import json
 import logging
 import re
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 
 import urllib3
@@ -118,6 +118,17 @@ def get_item(self, resource_id):
         self.log_response(response)
         return json.loads(response.text)
 
+    def _get_resource_id_from_location(self, headers: dict) -> Optional[str]:
+        header = ""
+        if "Location" in headers:
+            header = "Location"
+        elif "location" in headers:
+            header = "location"
+        else:
+            return None
+
+        return headers[header].split("/")[-1].strip()
+
     def create(self, unique_id_field=None, name=None, **factory_kwargs):
         # Pass in a `unique_id_field` name (e.g. 'schoolId') to have that
         # attribute returned alongside the resource_id.  Useful if you need
@@ -146,7 +157,8 @@ def create(self, unique_id_field=None, name=None, **factory_kwargs):
                 return None, None
             return None
         self.log_response(response)
-        resource_id = response.headers["Location"].split("/")[-1].strip()
+
+        resource_id = self._get_resource_id_from_location(response.headers)
         if unique_id is not None:
             return resource_id, unique_id
         return resource_id
@@ -164,8 +176,14 @@ def update(self, resource_id, **update_kwargs):
         if self.is_not_expected_result(response, [200, 204]):
             return
         self.log_response(response)
-        new_id = response.headers["Location"].split("/")[-1].strip()
-        assert new_id == resource_id
+
+        new_id = self._get_resource_id_from_location(response.headers)
+
+        # The ODS/API returns a Location header on PUT requests, but this is not a required feature
+        # of an Ed-Fi API. Therefore only apply this test if the location header was found
+        if new_id is not None:
+            assert new_id == resource_id
+
         return resource_id
 
     def delete_item(self, resource_id):
@@ -246,7 +264,7 @@ def create_using_dependencies(self, dependency_reference=None, **kwargs):
             "attributes": resource_attrs,
         }
 
-    def delete_with_dependencies(self, reference, **kwargs):
+    def delete_with_dependencies(self, reference):
         """
         Atomically delete an instance of this resource along with all
         dependencies.  The `reference` parameter will contain the necessary

src/edfi-performance-test/edfi_performance_test/api/client/education.py

@@ -26,7 +26,7 @@ class EducationOrganizationInterventionPrescriptionAssociationClient(EdFiAPIClie
     }
 
     def create_with_dependencies(self, **kwargs):
-        rx_reference = self.prescription_client.create_with_dependencies()
+        rx_reference = self.prescription_client.create_with_dependencies()  # type: ignore
 
         return self.create_using_dependencies(
             rx_reference,
@@ -52,7 +52,7 @@ class EducationOrganizationNetworkAssociationClient(EdFiAPIClient):
     }
 
     def create_with_dependencies(self, **kwargs):
-        network_reference = self.network_client.create_with_dependencies()
+        network_reference = self.network_client.create_with_dependencies()  # type: ignore
 
         return self.create_using_dependencies(
             network_reference,
@@ -73,7 +73,7 @@ class EducationOrganizationPeerAssociationClient(EdFiAPIClient):
     }
 
     def create_with_dependencies(self, **kwargs):
-        school_reference = self.school_client.create_with_dependencies()
+        school_reference = self.school_client.create_with_dependencies()  # type: ignore
 
         return self.create_using_dependencies(
             school_reference,
@@ -103,7 +103,7 @@ class LocalEducationAgencyClient(EdFiAPIClient):
     }
 
     def create_with_dependencies(self, **kwargs):
-        service_center_reference = self.service_center_client.create_with_dependencies()
+        service_center_reference = self.service_center_client.create_with_dependencies()  # type: ignore
 
         return self.create_using_dependencies(
             service_center_reference,
@@ -136,7 +136,7 @@ class FeederSchoolAssociationClient(EdFiAPIClient):
     }
 
     def create_with_dependencies(self, **kwargs):
-        feeder_school_reference = self.school_client.create_with_dependencies()
+        feeder_school_reference = self.school_client.create_with_dependencies()  # type: ignore
 
         return self.create_using_dependencies(
             feeder_school_reference,

src/edfi-performance-test/edfi_performance_test/api/client/grade.py

@@ -23,7 +23,7 @@ def create_with_dependencies(self, **kwargs):
         school_id = kwargs.pop("schoolId", SchoolClient.shared_elementary_school_id())
         course_code = kwargs.pop("courseCode", "ELA-01")
 
-        assoc_reference = self.section_assoc_client.create_with_dependencies(
+        assoc_reference = self.section_assoc_client.create_with_dependencies(  # type: ignore
             schoolId=school_id, courseCode=course_code
         )
         grade_period = assoc_reference["dependency_ids"]["section_client"][

src/edfi-performance-test/edfi_performance_test/api/client/gradebook_entries.py

@@ -16,7 +16,7 @@ class GradebookEntryClient(EdFiAPIClient):
     }
 
     def create_with_dependencies(self, **kwargs):
-        section_reference = self.section_client.create_with_dependencies()
+        section_reference = self.section_client.create_with_dependencies()  # type: ignore
         section_attrs = section_reference["attributes"]
 
         return self.create_using_dependencies(

src/edfi-performance-test/edfi_performance_test/api/client/graduation_plan.py

@@ -16,7 +16,7 @@ class GraduationPlanClient(EdFiAPIClient):
     }
 
     def create_with_dependencies(self, **kwargs):
-        school_reference = self.school_client.create_with_dependencies()
+        school_reference = self.school_client.create_with_dependencies()  # type: ignore
 
         return self.create_using_dependencies(
             school_reference,

src/edfi-performance-test/edfi_performance_test/api/client/intervention.py

@@ -21,7 +21,7 @@ class InterventionClient(EdFiAPIClient):
 
     def create_with_dependencies(self, **kwargs):
         # Create intervention prescription
-        rx_reference = self.prescription_client.create_with_dependencies()
+        rx_reference = self.prescription_client.create_with_dependencies()  # type: ignore
 
         # Create intervention
         return self.create_using_dependencies(
@@ -45,7 +45,7 @@ class InterventionStudyClient(EdFiAPIClient):
 
     def create_with_dependencies(self, **kwargs):
         # Create intervention prescription
-        rx_reference = self.prescription_client.create_with_dependencies()
+        rx_reference = self.prescription_client.create_with_dependencies()  # type: ignore
 
         # Create intervention
         return self.create_using_dependencies(

src/edfi-performance-test/edfi_performance_test/api/client/parent.py

@@ -28,12 +28,12 @@ def create_with_dependencies(self, **kwargs):
         parent_id = self.create(**parent_attrs)
 
         # Create enrolled student
-        student_reference = self.student_client.create_with_dependencies(
+        student_reference = self.student_client.create_with_dependencies(  # type: ignore
             schoolId=school_id
         )
 
         # Associate parent with student to allow updates
-        assoc_id = self.parent_assoc_client.create(
+        assoc_id = self.parent_assoc_client.create(  # type: ignore
             parentReference__parentUniqueId=parent_unique_id,
             studentReference__studentUniqueId=student_reference["attributes"][
                 "studentUniqueId"
@@ -50,8 +50,8 @@ def create_with_dependencies(self, **kwargs):
         }
 
     def delete_with_dependencies(self, reference, **kwargs):
-        self.parent_assoc_client.delete_item(reference["dependency_ids"]["assoc_id"])
-        self.student_client.delete_with_dependencies(
+        self.parent_assoc_client.delete_item(reference["dependency_ids"]["assoc_id"])  # type: ignore
+        self.student_client.delete_with_dependencies(  # type: ignore
             reference["dependency_ids"]["student_reference"]
         )
         self.delete_item(reference["resource_id"])

src/edfi-performance-test/edfi_performance_test/api/client/post_secondary.py

@@ -20,7 +20,7 @@ class PostSecondaryEventClient(EdFiAPIClient):
 
     def create_with_dependencies(self, **kwargs):
         # Create new student for association
-        institution_reference = self.institution_client.create_with_dependencies()
+        institution_reference = self.institution_client.create_with_dependencies()  # type: ignore
 
         return self.create_using_dependencies(
             institution_reference,

src/edfi-performance-test/edfi_performance_test/api/client/report_card.py

@@ -16,7 +16,7 @@ class ReportCardClient(EdFiAPIClient):
     }
 
     def create_with_dependencies(self, **kwargs):
-        period_reference = self.grading_period_client.create_with_dependencies()
+        period_reference = self.grading_period_client.create_with_dependencies()  # type: ignore
 
         return self.create_using_dependencies(
             period_reference,