From 4a6cf05486370f4b6ca50f3e60946f9d66c7ecd0 Mon Sep 17 00:00:00 2001 From: Jonavelle Cuerdo Date: Mon, 15 Jun 2026 10:02:28 -0400 Subject: [PATCH] Updates post testing digitized theses workflow Why these changes are being introduced: * Testing of the new digitized theses workflow in Dev revealed the need for these changes. How this addresses that need: * Fix Makefile * Set `aws s3 sync` logging to DEBUG level * Ensure metadata transformer uses `mit.thesis.degree` metadata field * Set collection handles for MIT Theses collections * Fix filename for Alma XML file export Side effects of this change: * None Relevant ticket(s): * https://mitlibraries.atlassian.net/browse/IN-1759 --- Makefile | 3 +- dsc/reports/digitized_theses.py | 4 +- dsc/utils/aws/s3.py | 2 +- dsc/workflows/digitized_theses/transformer.py | 8 ++-- dsc/workflows/digitized_theses/workflow.py | 46 +++++++++---------- tests/test_cli.py | 2 + .../digitized_theses/test_workflow.py | 10 ++-- 7 files changed, 38 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index b591ea9..fdf16a4 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ MINIO_COMPOSE_FILE=tests/minio/docker-compose.yaml ### and review the other commented lines in the document. ### ECR_NAME_DEV := dspace-submission-composer-dev ECR_URL_DEV := 222053980223.dkr.ecr.us-east-1.amazonaws.com/dspace-submission-composer-dev -CPU_ARCH ?= $(shell cat .aws-architecture 2>/dev/null || echo \"linux/amd64\") +CPU_ARCH ?= $(shell cat .aws-architecture 2>/dev/null || echo "linux/amd64") # FUNCTION_DEV := ### End of Terraform-generated header ### @@ -96,6 +96,7 @@ dist-dev: check-arch # Build docker container (intended for developer-based manu --tag $(ECR_URL_DEV):make-$$ARCH_TAG \ --tag $(ECR_URL_DEV):make-$(shell git describe --always) \ --tag $(ECR_NAME_DEV):$$ARCH_TAG \ + . publish-dev: dist-dev # Build, tag and push (intended for developer-based manual publish) @ARCH_TAG=$$(cat .arch_tag); \ diff --git a/dsc/reports/digitized_theses.py b/dsc/reports/digitized_theses.py index 070acfb..5a1b041 100644 --- a/dsc/reports/digitized_theses.py +++ b/dsc/reports/digitized_theses.py @@ -13,9 +13,7 @@ class DigitizedThesesFinalizeReport(FinalizeReport): Attachment( filename="FM_DSpaceURLS_Tab.txt", method_name="create_filemaker_export_text" ), - Attachment( - filename="ALMA_DSpaceURLS_Tab.xml", method_name="create_alma_export_xml" - ), + Attachment(filename="ALMA_DSpaceURLS.xml", method_name="create_alma_export_xml"), ) # ==================== diff --git a/dsc/utils/aws/s3.py b/dsc/utils/aws/s3.py index b2abe9e..7f2dae2 100644 --- a/dsc/utils/aws/s3.py +++ b/dsc/utils/aws/s3.py @@ -156,7 +156,7 @@ def run_aws_cli_sync( if process.stdout: for line in process.stdout: if line: - logger.info(line) + logger.debug(line) else: logger.info("No changes detected in source, no sync required") diff --git a/dsc/workflows/digitized_theses/transformer.py b/dsc/workflows/digitized_theses/transformer.py index aa64fdd..2b9d170 100644 --- a/dsc/workflows/digitized_theses/transformer.py +++ b/dsc/workflows/digitized_theses/transformer.py @@ -22,7 +22,7 @@ class DigitizedThesesTransformer: are based on the transformations in the MIT-customized 'marc21-to-dc.xsl' stylesheet, which itself is based on the Library of Congress MARC-to-DC crosswalk. The transformer includes crosswalks to normalize values - for select fields (dc.contributor.department and mit.theses.degree). + for select fields (dc.contributor.department and mit.thesis.degree). """ fields: Iterable[str] = [ @@ -80,7 +80,7 @@ class DigitizedThesesTransformer: "dc_subject_other", "dc_title_alternative", "dc_type", - "mit_theses_degree", + "mit_thesis_degree", ] degree_types_crosswalk: ClassVar = [ @@ -1098,7 +1098,7 @@ def _normalize_dc_type(cls, value: str) -> str: return cls.types_crosswalk.get(value, value) @classmethod - def mit_theses_degree(cls, record: etree._Element) -> list[str]: + def mit_thesis_degree(cls, record: etree._Element) -> list[str]: """MARC 502, normalized using cls.degree_types_crosswalk. If the 502 $b value does not match any of the patterns in the @@ -1119,7 +1119,7 @@ def mit_theses_degree(cls, record: etree._Element) -> list[str]: @classmethod def _normalize_degree_type(cls, value: str) -> str | None: - """Normalize 502 $b (degree type) value for mit.theses.degree. + """Normalize 502 $b (degree type) value for mit.thesis.degree. Returns the normalized degree [Bachelor, Doctoral, Engineer, Master] given cls.degree_types_crosswalk. If there are no matches, diff --git a/dsc/workflows/digitized_theses/workflow.py b/dsc/workflows/digitized_theses/workflow.py index 6ad4481..1ba08d3 100644 --- a/dsc/workflows/digitized_theses/workflow.py +++ b/dsc/workflows/digitized_theses/workflow.py @@ -30,17 +30,16 @@ CONFIG = Config() logger = logging.getLogger(__name__) -# TODO: Update collection handles for test instance once ready MIT_THESES_COLLECTION_HANDLES = { - "Bachelor": {"dev": "1721.1/test", "prod": "1721.1/131024"}, - "Engineer": {"dev": "1721.1/test", "prod": "1721.1/131023"}, - "Master": {"dev": "1721.1/test", "prod": "1721.1/131023"}, - "Doctoral": {"dev": "1721.1/test", "prod": "1721.1/131022"}, + "Bachelor": {"dev": "1721.1/131024", "prod": "1721.1/131024"}, + "Engineer": {"dev": "1721.1/131023", "prod": "1721.1/131023"}, + "Master": {"dev": "1721.1/131023", "prod": "1721.1/131023"}, + "Doctoral": {"dev": "1721.1/131022", "prod": "1721.1/131022"}, } class MITThesesCommunityUUID(StrEnum): - DEV = "" # TODO: Update to uuid for 'MIT Theses' community in test + DEV = "6fc02cc2-0d14-4023-8a6f-d9900d0c4302" PROD = "6fc02cc2-0d14-4023-8a6f-d9900d0c4302" @@ -50,7 +49,7 @@ class DigitizedTheses(Workflow): This workflow is unique to other workflows in that it relies on an additional S3 bucket that serves as a "workspace" for digitized theses. Users create a batch folder in this workspace S3 bucket and upload - theses (PDF files) named with the OCLC number: -MIT.pdf. + theses (PDF files) named with the OCLC number: .pdf. When this workflow creates a batch *without* syncing, it will first copy the contents of the original batch folder in the workspace S3 bucket @@ -474,7 +473,7 @@ def _is_replacement_thesis(item: DSpaceItem) -> bool: return True - def submit_items(self, _collection_handle: str | None = None) -> list: + def submit_items(self, collection_handle: str | None = None) -> list: """Submit items to the DSpace Submission Service according to the workflow class. This method begins by creating a manifest for the batch of item submissions. The @@ -484,7 +483,7 @@ def submit_items(self, _collection_handle: str | None = None) -> list: 1. Check if the item submission is "ready to submit" 2. Transform the item metadata to Qualified Dublin Core (QDC) - 3. Get the item's collection handle based on the mit.theses.degree field + 3. Get the item's collection handle based on the mit.thesis.degree field 4. Send a submission message based on the thesis type ("New thesis" vs. "Replacement thesis") - New theses must provide the "CollectionHandle" @@ -541,9 +540,10 @@ def submit_items(self, _collection_handle: str | None = None) -> list: item_handle=item_submission.dspace_handle, ) else: - # get collection handle based on mit.theses.degree - item_submission.collection_handle = self._get_item_collection_handle( - item_metadata + # get collection handle based on mit.thesis.degree + item_submission.collection_handle = ( + collection_handle + or self._get_item_collection_handle(item_metadata) ) response = item_submission.send_submission_message( @@ -656,7 +656,7 @@ def _get_transformed_metadata(self, source_metadata_file: str) -> dict: return transformed_metadata def _get_item_collection_handle(self, item_metadata: dict) -> str: - """Get collection handle for item based on mit.theses.degree. + """Get collection handle for item based on mit.thesis.degree. There are three collection handles for the MIT Theses Community: 1. Doctoral Theses @@ -664,34 +664,34 @@ def _get_item_collection_handle(self, item_metadata: dict) -> str: 3. Undergraduate Theses Each item submission belongs to one of the three collection handles based - on the derived mit.theses.degree value, where this field is constrained to + on the derived mit.thesis.degree value, where this field is constrained to the set: ['Bachelor', 'Engineer', 'Master', 'Doctoral']. The global variables MIT_THESES_COLLECTION_HANDLES includes mappings of - mit.theses.degree value to collection handles in the 'dev' and 'prod' + mit.thesis.degree value to collection handles in the 'dev' and 'prod' environments. """ - mit_theses_degrees = item_metadata.get("mit.theses.degree") + mit_thesis_degrees = item_metadata.get("mit.thesis.degree") - if not mit_theses_degrees: + if not mit_thesis_degrees: raise TypeError( - f"Cannot determine collection handle when mit.theses.degree={mit_theses_degrees}" # noqa: E501 + f"Cannot determine collection handle when mit.thesis.degree={mit_thesis_degrees}" # noqa: E501 ) - if len(mit_theses_degrees) > 1: + if len(mit_thesis_degrees) > 1: logger.warning( - f"Found multiple values for mit.theses.degree={mit_theses_degrees}; " + f"Found multiple values for mit.thesis.degree={mit_thesis_degrees}; " "retrieving collection handle for first match" ) - for value in mit_theses_degrees: + for value in mit_thesis_degrees: if value in MIT_THESES_COLLECTION_HANDLES: if CONFIG.workspace == "prod": return MIT_THESES_COLLECTION_HANDLES[value]["prod"] return MIT_THESES_COLLECTION_HANDLES[value]["dev"] raise ValueError( - f"No collection handles found for mit.theses.degree={mit_theses_degrees}" + f"No collection handles found for mit.thesis.degree={mit_thesis_degrees}" ) @staticmethod @@ -715,4 +715,4 @@ def _parse_record_from_sru_response(content: bytes) -> etree._Element: @staticmethod def parse_item_identifier(filename: str) -> str: - return filename.rsplit("/", maxsplit=1)[-1].removesuffix("-MIT.pdf") + return filename.rsplit("/", maxsplit=1)[-1].removesuffix(".pdf") diff --git a/tests/test_cli.py b/tests/test_cli.py index dcf4207..cdaf109 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -290,6 +290,7 @@ def test_finalize_success( def test_sync_success(caplog, runner, monkeypatch, moto_server, config_instance): """Run sync using moto stand-alone server.""" + caplog.set_level("DEBUG") monkeypatch.setenv("S3_BUCKET_SUBMISSION_ASSETS", "destination") monkeypatch.setenv("S3_BUCKET_SYNC_SOURCE", "source") monkeypatch.setenv("AWS_ENDPOINT_URL", moto_server) @@ -335,6 +336,7 @@ def test_sync_use_source_and_destination_success( caplog, runner, monkeypatch, moto_server, config_instance ): """Run sync using moto stand-alone server.""" + caplog.set_level("DEBUG") monkeypatch.setenv("S3_BUCKET_SUBMISSION_ASSETS", "destination") monkeypatch.setenv("AWS_ENDPOINT_URL", moto_server) diff --git a/tests/workflows/digitized_theses/test_workflow.py b/tests/workflows/digitized_theses/test_workflow.py index ffc4d5e..04770d4 100644 --- a/tests/workflows/digitized_theses/test_workflow.py +++ b/tests/workflows/digitized_theses/test_workflow.py @@ -393,7 +393,7 @@ def test_workflow_submit_items_success( "001": { "thesis_type": "New thesis", "metadata_file": ["001.xml"], - "bitstream_files": ["001-MIT.pdf"], + "bitstream_files": ["001.pdf"], } } mock_workflow_get_transformed_metadata.return_value = None @@ -454,7 +454,7 @@ def test_workflow_submit_items_handles_errors( "001": { "thesis_type": "New thesis", "metadata_file": ["001.xml"], - "bitstream_files": ["001-MIT.pdf"], + "bitstream_files": ["001.pdf"], } } mock_workflow_get_transformed_metadata.side_effect = Exception @@ -503,9 +503,9 @@ def test_workflow_get_item_collection_handle(): assert ( workflow._get_item_collection_handle( - item_metadata={"mit.theses.degree": ["Bachelor"]} + item_metadata={"mit.thesis.degree": ["Bachelor"]} ) - == "1721.1/test" + == "1721.1/131024" ) @@ -539,4 +539,4 @@ def test_workflow_parse_record_from_sru_response_no_records(alma_sru_response_no def test_workflow_parse_item_identifier(): workflow = DigitizedTheses(batch_id="batch-aaa") - assert workflow.parse_item_identifier("s3://bucket/prefix/123456-MIT.pdf") == "123456" + assert workflow.parse_item_identifier("s3://bucket/prefix/123456.pdf") == "123456"