Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ MINIO_COMPOSE_FILE=tests/minio/docker-compose.yaml
### and review the other commented lines in the document. ###
ECR_NAME_DEV := dspace-submission-composer-dev
ECR_URL_DEV := 222053980223.dkr.ecr.us-east-1.amazonaws.com/dspace-submission-composer-dev
CPU_ARCH ?= $(shell cat .aws-architecture 2>/dev/null || echo \"linux/amd64\")
CPU_ARCH ?= $(shell cat .aws-architecture 2>/dev/null || echo "linux/amd64")
# FUNCTION_DEV :=
### End of Terraform-generated header ###

Expand Down Expand Up @@ -96,6 +96,7 @@ dist-dev: check-arch # Build docker container (intended for developer-based manu
--tag $(ECR_URL_DEV):make-$$ARCH_TAG \
--tag $(ECR_URL_DEV):make-$(shell git describe --always) \
--tag $(ECR_NAME_DEV):$$ARCH_TAG \
.

publish-dev: dist-dev # Build, tag and push (intended for developer-based manual publish)
@ARCH_TAG=$$(cat .arch_tag); \
Expand Down
4 changes: 1 addition & 3 deletions dsc/reports/digitized_theses.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@ class DigitizedThesesFinalizeReport(FinalizeReport):
Attachment(
filename="FM_DSpaceURLS_Tab.txt", method_name="create_filemaker_export_text"
),
Attachment(
filename="ALMA_DSpaceURLS_Tab.xml", method_name="create_alma_export_xml"
),
Attachment(filename="ALMA_DSpaceURLS.xml", method_name="create_alma_export_xml"),
)

# ====================
Expand Down
2 changes: 1 addition & 1 deletion dsc/utils/aws/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ def run_aws_cli_sync(
if process.stdout:
for line in process.stdout:
if line:
logger.info(line)
logger.debug(line)
else:
logger.info("No changes detected in source, no sync required")

Expand Down
8 changes: 4 additions & 4 deletions dsc/workflows/digitized_theses/transformer.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ class DigitizedThesesTransformer:
are based on the transformations in the MIT-customized 'marc21-to-dc.xsl'
stylesheet, which itself is based on the Library of Congress MARC-to-DC
crosswalk. The transformer includes crosswalks to normalize values
for select fields (dc.contributor.department and mit.theses.degree).
for select fields (dc.contributor.department and mit.thesis.degree).
"""

fields: Iterable[str] = [
Expand Down Expand Up @@ -80,7 +80,7 @@ class DigitizedThesesTransformer:
"dc_subject_other",
"dc_title_alternative",
"dc_type",
"mit_theses_degree",
"mit_thesis_degree",
]

degree_types_crosswalk: ClassVar = [
Expand Down Expand Up @@ -1098,7 +1098,7 @@ def _normalize_dc_type(cls, value: str) -> str:
return cls.types_crosswalk.get(value, value)

@classmethod
def mit_theses_degree(cls, record: etree._Element) -> list[str]:
def mit_thesis_degree(cls, record: etree._Element) -> list[str]:
"""MARC 502, normalized using cls.degree_types_crosswalk.

If the 502 $b value does not match any of the patterns in the
Expand All @@ -1119,7 +1119,7 @@ def mit_theses_degree(cls, record: etree._Element) -> list[str]:

@classmethod
def _normalize_degree_type(cls, value: str) -> str | None:
"""Normalize 502 $b (degree type) value for mit.theses.degree.
"""Normalize 502 $b (degree type) value for mit.thesis.degree.

Returns the normalized degree [Bachelor, Doctoral, Engineer, Master]
given cls.degree_types_crosswalk. If there are no matches,
Expand Down
46 changes: 23 additions & 23 deletions dsc/workflows/digitized_theses/workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,16 @@
CONFIG = Config()
logger = logging.getLogger(__name__)

# TODO: Update collection handles for test instance once ready
MIT_THESES_COLLECTION_HANDLES = {
"Bachelor": {"dev": "1721.1/test", "prod": "1721.1/131024"},
"Engineer": {"dev": "1721.1/test", "prod": "1721.1/131023"},
"Master": {"dev": "1721.1/test", "prod": "1721.1/131023"},
"Doctoral": {"dev": "1721.1/test", "prod": "1721.1/131022"},
"Bachelor": {"dev": "1721.1/131024", "prod": "1721.1/131024"},
"Engineer": {"dev": "1721.1/131023", "prod": "1721.1/131023"},
"Master": {"dev": "1721.1/131023", "prod": "1721.1/131023"},
"Doctoral": {"dev": "1721.1/131022", "prod": "1721.1/131022"},
}


class MITThesesCommunityUUID(StrEnum):
DEV = "" # TODO: Update to uuid for 'MIT Theses' community in test
DEV = "6fc02cc2-0d14-4023-8a6f-d9900d0c4302"
PROD = "6fc02cc2-0d14-4023-8a6f-d9900d0c4302"


Expand All @@ -50,7 +49,7 @@ class DigitizedTheses(Workflow):
This workflow is unique to other workflows in that it relies on an
additional S3 bucket that serves as a "workspace" for digitized theses.
Users create a batch folder in this workspace S3 bucket and upload
theses (PDF files) named with the OCLC number: <oclc-number>-MIT.pdf.
theses (PDF files) named with the OCLC number: <oclc-number>.pdf.

When this workflow creates a batch *without* syncing, it will first
copy the contents of the original batch folder in the workspace S3 bucket
Expand Down Expand Up @@ -474,7 +473,7 @@ def _is_replacement_thesis(item: DSpaceItem) -> bool:

return True

def submit_items(self, _collection_handle: str | None = None) -> list:
def submit_items(self, collection_handle: str | None = None) -> list:
"""Submit items to the DSpace Submission Service according to the workflow class.

This method begins by creating a manifest for the batch of item submissions. The
Expand All @@ -484,7 +483,7 @@ def submit_items(self, _collection_handle: str | None = None) -> list:

1. Check if the item submission is "ready to submit"
2. Transform the item metadata to Qualified Dublin Core (QDC)
3. Get the item's collection handle based on the mit.theses.degree field
3. Get the item's collection handle based on the mit.thesis.degree field
4. Send a submission message based on the thesis type ("New thesis" vs.
"Replacement thesis")
- New theses must provide the "CollectionHandle"
Expand Down Expand Up @@ -541,9 +540,10 @@ def submit_items(self, _collection_handle: str | None = None) -> list:
item_handle=item_submission.dspace_handle,
)
else:
# get collection handle based on mit.theses.degree
item_submission.collection_handle = self._get_item_collection_handle(
item_metadata
# get collection handle based on mit.thesis.degree
item_submission.collection_handle = (
collection_handle
or self._get_item_collection_handle(item_metadata)
)

response = item_submission.send_submission_message(
Expand Down Expand Up @@ -656,42 +656,42 @@ def _get_transformed_metadata(self, source_metadata_file: str) -> dict:
return transformed_metadata

def _get_item_collection_handle(self, item_metadata: dict) -> str:
"""Get collection handle for item based on mit.theses.degree.
"""Get collection handle for item based on mit.thesis.degree.

There are three collection handles for the MIT Theses Community:
1. Doctoral Theses
2. Graduate Theses
3. Undergraduate Theses

Each item submission belongs to one of the three collection handles based
on the derived mit.theses.degree value, where this field is constrained to
on the derived mit.thesis.degree value, where this field is constrained to
the set: ['Bachelor', 'Engineer', 'Master', 'Doctoral'].

The global variables MIT_THESES_COLLECTION_HANDLES includes mappings of
mit.theses.degree value to collection handles in the 'dev' and 'prod'
mit.thesis.degree value to collection handles in the 'dev' and 'prod'
environments.
"""
mit_theses_degrees = item_metadata.get("mit.theses.degree")
mit_thesis_degrees = item_metadata.get("mit.thesis.degree")

if not mit_theses_degrees:
if not mit_thesis_degrees:
raise TypeError(
f"Cannot determine collection handle when mit.theses.degree={mit_theses_degrees}" # noqa: E501
f"Cannot determine collection handle when mit.thesis.degree={mit_thesis_degrees}" # noqa: E501
)

if len(mit_theses_degrees) > 1:
if len(mit_thesis_degrees) > 1:
logger.warning(
f"Found multiple values for mit.theses.degree={mit_theses_degrees}; "
f"Found multiple values for mit.thesis.degree={mit_thesis_degrees}; "
"retrieving collection handle for first match"
)

for value in mit_theses_degrees:
for value in mit_thesis_degrees:
if value in MIT_THESES_COLLECTION_HANDLES:
if CONFIG.workspace == "prod":
return MIT_THESES_COLLECTION_HANDLES[value]["prod"]
return MIT_THESES_COLLECTION_HANDLES[value]["dev"]

raise ValueError(
f"No collection handles found for mit.theses.degree={mit_theses_degrees}"
f"No collection handles found for mit.thesis.degree={mit_thesis_degrees}"
)

@staticmethod
Expand All @@ -715,4 +715,4 @@ def _parse_record_from_sru_response(content: bytes) -> etree._Element:

@staticmethod
def parse_item_identifier(filename: str) -> str:
return filename.rsplit("/", maxsplit=1)[-1].removesuffix("-MIT.pdf")
return filename.rsplit("/", maxsplit=1)[-1].removesuffix(".pdf")
2 changes: 2 additions & 0 deletions tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,7 @@ def test_finalize_success(

def test_sync_success(caplog, runner, monkeypatch, moto_server, config_instance):
"""Run sync using moto stand-alone server."""
caplog.set_level("DEBUG")
monkeypatch.setenv("S3_BUCKET_SUBMISSION_ASSETS", "destination")
monkeypatch.setenv("S3_BUCKET_SYNC_SOURCE", "source")
monkeypatch.setenv("AWS_ENDPOINT_URL", moto_server)
Expand Down Expand Up @@ -335,6 +336,7 @@ def test_sync_use_source_and_destination_success(
caplog, runner, monkeypatch, moto_server, config_instance
):
"""Run sync using moto stand-alone server."""
caplog.set_level("DEBUG")
monkeypatch.setenv("S3_BUCKET_SUBMISSION_ASSETS", "destination")
monkeypatch.setenv("AWS_ENDPOINT_URL", moto_server)

Expand Down
10 changes: 5 additions & 5 deletions tests/workflows/digitized_theses/test_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ def test_workflow_submit_items_success(
"001": {
"thesis_type": "New thesis",
"metadata_file": ["001.xml"],
"bitstream_files": ["001-MIT.pdf"],
"bitstream_files": ["001.pdf"],
}
}
mock_workflow_get_transformed_metadata.return_value = None
Expand Down Expand Up @@ -454,7 +454,7 @@ def test_workflow_submit_items_handles_errors(
"001": {
"thesis_type": "New thesis",
"metadata_file": ["001.xml"],
"bitstream_files": ["001-MIT.pdf"],
"bitstream_files": ["001.pdf"],
}
}
mock_workflow_get_transformed_metadata.side_effect = Exception
Expand Down Expand Up @@ -503,9 +503,9 @@ def test_workflow_get_item_collection_handle():

assert (
workflow._get_item_collection_handle(
item_metadata={"mit.theses.degree": ["Bachelor"]}
item_metadata={"mit.thesis.degree": ["Bachelor"]}
)
== "1721.1/test"
== "1721.1/131024"
)


Expand Down Expand Up @@ -539,4 +539,4 @@ def test_workflow_parse_record_from_sru_response_no_records(alma_sru_response_no
def test_workflow_parse_item_identifier():
workflow = DigitizedTheses(batch_id="batch-aaa")

assert workflow.parse_item_identifier("s3://bucket/prefix/123456-MIT.pdf") == "123456"
assert workflow.parse_item_identifier("s3://bucket/prefix/123456.pdf") == "123456"
Loading