From 268931ff0e8d5f43a283a771fac3c9d0d1fca0ad Mon Sep 17 00:00:00 2001 From: karen-hedges <133129444+karen-hedges@users.noreply.github.com> Date: Thu, 4 Jun 2026 11:54:00 +0100 Subject: [PATCH 1/9] DMP-5565 RPO - Use create record process_time in CSV reconcilliation Updated the code so that the reconciliation process uses the create record processsed time instead of the input upload processed time. Also updated the missing reponse status so it uses the most recent of the create record processed time, input upload processed time or data ingestion time as its possible any of the files could be missing --- .../ArmRpoBacklogCatchupServiceIntTest.java | 4 +- ...ExternalObjectDirectoryRepositoryTest.java | 8 +- .../AbstractArmBatchProcessResponseFiles.java | 61 ++- .../impl/ArmRpoBacklogCatchupServiceImpl.java | 19 +- ...etsToArmBatchProcessResponseFilesImpl.java | 4 +- .../entity/ExternalObjectDirectoryEntity.java | 3 + .../ExternalObjectDirectoryRepository.java | 6 +- ...lumn_create_record_process_time_to_eod.sql | 9 + .../db/reference/modernised_darts_indexes.sql | 410 +++++++++--------- .../db/reference/modernised_darts_schema.sql | 1 + .../ArmBatchProcessResponseFilesImplTest.java | 4 +- .../ArmRpoBacklogCatchupServiceImplTest.java | 10 +- 12 files changed, 300 insertions(+), 239 deletions(-) create mode 100644 src/main/resources/db/migration/common/V1_510__add_column_create_record_process_time_to_eod.sql diff --git a/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceIntTest.java b/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceIntTest.java index 1acd0c0b2e3..d9ac57fdf11 100644 --- a/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceIntTest.java +++ b/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceIntTest.java @@ -100,7 +100,7 @@ void performCatchup_withSuccessfulArmApiCalls_doesNotThrowException() { .status(dartsDatabase.getObjectRecordStatusEntity(ARM_RPO_PENDING)) .externalLocationType(dartsDatabase.getExternalLocationTypeEntity(ARM)) .externalLocation(UUID.randomUUID().toString()).build(); - armEod1.setInputUploadProcessedTs(lastModifiedDateTime); + armEod1.setCreateRecordProcessedTs(lastModifiedDateTime); armEod1.setVerificationAttempts(1); dartsPersistence.save(armEod1); @@ -154,7 +154,7 @@ void performCatchup_doesNothing_WhenOldestPendingEodIsTooNew() { .status(dartsDatabase.getObjectRecordStatusEntity(ARM_RPO_PENDING)) .externalLocationType(dartsDatabase.getExternalLocationTypeEntity(ARM)) .externalLocation(UUID.randomUUID().toString()).build(); - armEod1.setInputUploadProcessedTs(lastModifiedDateTime); + armEod1.setCreateRecordProcessedTs(lastModifiedDateTime); armEod1.setVerificationAttempts(1); dartsPersistence.save(armEod1); diff --git a/src/integrationTest/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepositoryTest.java b/src/integrationTest/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepositoryTest.java index ea29ff624f0..84aabe5de3a 100644 --- a/src/integrationTest/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepositoryTest.java +++ b/src/integrationTest/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepositoryTest.java @@ -1258,7 +1258,7 @@ private void generateDataWithMediaForInbound(int hoursBeforeCurrentTime) } @Test - void findOldestByInputUploadProcessedTsAndStatusAndLocation() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + void findOldestByCreateRecordProcessedTsAndStatusAndLocation() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { // Given OffsetDateTime pastCurrentDateTime1 = OffsetDateTime.now().minusHours(200); OffsetDateTime pastCurrentDateTime2 = OffsetDateTime.now().minusHours(2); @@ -1266,7 +1266,7 @@ void findOldestByInputUploadProcessedTsAndStatusAndLocation() throws InvocationT List matchingEods = externalObjectDirectoryStub.generateWithStatusAndMediaLocation( ExternalLocationTypeEnum.ARM, ARM_RPO_PENDING, 10, Optional.of(pastCurrentDateTime1)); matchingEods.forEach(eod -> { - eod.setInputUploadProcessedTs(pastCurrentDateTime1); + eod.setCreateRecordProcessedTs(pastCurrentDateTime1); }); dartsPersistence.saveAll(matchingEods); assertEquals(10, matchingEods.size()); @@ -1274,13 +1274,13 @@ void findOldestByInputUploadProcessedTsAndStatusAndLocation() throws InvocationT List nonMatchingEods = externalObjectDirectoryStub.generateWithStatusAndMediaLocation( ExternalLocationTypeEnum.ARM, ARM_RPO_PENDING, 4, Optional.of(pastCurrentDateTime2)); nonMatchingEods.forEach(eod -> { - eod.setInputUploadProcessedTs(pastCurrentDateTime2); + eod.setCreateRecordProcessedTs(pastCurrentDateTime2); }); dartsPersistence.saveAll(nonMatchingEods); assertEquals(4, nonMatchingEods.size()); // When - ExternalObjectDirectoryEntity result = externalObjectDirectoryRepository.findOldestByInputUploadProcessedTsAndStatusAndLocation( + ExternalObjectDirectoryEntity result = externalObjectDirectoryRepository.findOldestByCreateRecordProcessedTsAndStatusAndLocation( EodHelper.armRpoPendingStatus(), EodHelper.armLocation()); // Then diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java index 81ae2475106..ddeccb7a724 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java @@ -239,6 +239,16 @@ public OffsetDateTime getUploadFileRecordProcessTime(ArmResponseUploadFileRecord } } + public OffsetDateTime getCreateRecordProcessTime(ArmResponseCreateRecord armResponseCreateRecord) { + try { + return OffsetDateTime.parse(armResponseCreateRecord.getProcessTime(), dateTimeFormatter); + } catch (Exception e) { + log.warn("Unable to parse timestamp {} from ARM create record (CR) record {}", armResponseCreateRecord.getProcessTime(), + armResponseCreateRecord.getA360RecordId(), e); + throw new IllegalArgumentException(e); + } + } + private ArmResponseInputUploadFileRecord getResponseInputUploadFileRecordOrDelete(BatchInputUploadFileFilenameProcessor batchUploadFileFilenameProcessor, String inputUploadFileRecordStr, List externalObjectDirectoryEntities, @@ -303,14 +313,16 @@ private void processResponseFileByHashcode(BatchInputUploadFileFilenameProcessor List externalObjectDirectoryEntities = externalObjectDirectoryRepository .findAllByStatusAndManifestFile(EodHelper.armProcessingResponseFilesStatus(), manifestName); - OffsetDateTime minInputUploadProcessedTime = timeHelper.currentOffsetDateTime().minus( + OffsetDateTime minProcessedTime = timeHelper.currentOffsetDateTime().minus( armDataManagementConfiguration.getArmMissingResponseDuration()); externalObjectDirectoryEntities.forEach( externalObjectDirectoryEntity -> { - if (externalObjectDirectoryEntity.getInputUploadProcessedTs() != null - && externalObjectDirectoryEntity.getInputUploadProcessedTs().isBefore(minInputUploadProcessedTime)) { - markEodAsResponseProcessingFailed(externalObjectDirectoryEntity, userAccount); + // Use the latest date we have a process time for + OffsetDateTime processedTime = getLatestProcessedTs(externalObjectDirectoryEntity); + if (nonNull(processedTime) + && processedTime.isBefore(minProcessedTime)) { + markEodAsMissingResponseFailure(externalObjectDirectoryEntity, userAccount); } else { updateExternalObjectDirectoryStatus(externalObjectDirectoryEntity, EodHelper.armDropZoneStatus(), userAccount); } @@ -322,6 +334,16 @@ private void processResponseFileByHashcode(BatchInputUploadFileFilenameProcessor } } + private OffsetDateTime getLatestProcessedTs(ExternalObjectDirectoryEntity externalObjectDirectoryEntity) { + if (nonNull(externalObjectDirectoryEntity.getCreateRecordProcessedTs())) { + return externalObjectDirectoryEntity.getCreateRecordProcessedTs(); + } else if (nonNull(externalObjectDirectoryEntity.getInputUploadProcessedTs())) { + return externalObjectDirectoryEntity.getInputUploadProcessedTs(); + } else { + return externalObjectDirectoryEntity.getDataIngestionTs(); + } + } + protected void processBatchResponseFiles(BatchInputUploadFileFilenameProcessor batchUploadFileFilenameProcessor, ArmBatchResponses armBatchResponses, UserAccountEntity userAccount) { @@ -372,9 +394,12 @@ private void missingResponsesFiles(UserAccountEntity userAccount, ArmResponseBat OffsetDateTime minIngestionTime = timeHelper.currentOffsetDateTime().minus( armDataManagementConfiguration.getArmMissingResponseDuration()); - if (externalObjectDirectory.getInputUploadProcessedTs() != null - && externalObjectDirectory.getInputUploadProcessedTs().isBefore(minIngestionTime)) { - markEodAsResponseProcessingFailed(externalObjectDirectory, userAccount); + // Use the latest date we have a process time for + OffsetDateTime processedTime = getLatestProcessedTs(externalObjectDirectory); + + if (nonNull(processedTime) + && processedTime.isBefore(minIngestionTime)) { + markEodAsMissingResponseFailure(externalObjectDirectory, userAccount); } else { updateExternalObjectDirectoryStatus(externalObjectDirectory, EodHelper.armDropZoneStatus(), userAccount); } @@ -383,7 +408,7 @@ private void missingResponsesFiles(UserAccountEntity userAccount, ArmResponseBat } } - protected void markEodAsResponseProcessingFailed(ExternalObjectDirectoryEntity externalObjectDirectory, UserAccountEntity userAccount) { + protected void markEodAsMissingResponseFailure(ExternalObjectDirectoryEntity externalObjectDirectory, UserAccountEntity userAccount) { updateExternalObjectDirectoryStatus(externalObjectDirectory, EodHelper.armMissingResponseStatus(), userAccount); } @@ -533,6 +558,7 @@ private void readCreateRecordFile(BinaryData createRecordBinary, CreateRecordFil if (StringUtils.isNotEmpty(uploadNewFileRecord.getRelationId())) { Long externalObjectDirectoryId = Long.valueOf(uploadNewFileRecord.getRelationId()); armBatchResponses.addResponseBatchData(externalObjectDirectoryId, armResponseCreateRecord, createRecordFilenameProcessor); + setProcessTimeForCreateRecordFileRecord(armResponseCreateRecord, externalObjectDirectoryId, createRecordFilenameAndPath); } else { log.warn("Unable to get EOD id (relation id) from uploadNewFileRecord {} create record {}", armResponseCreateRecord.getInput(), createRecordFilenameAndPath); @@ -569,6 +595,25 @@ private void setEodDataIngestionTimestamp(Long externalObjectDirectoryId, Offset } } + private void setProcessTimeForCreateRecordFileRecord(ArmResponseCreateRecord armResponseCreateRecord, Long externalObjectDirectoryId, + String createRecordFilenameAndPath) { + try { + OffsetDateTime createRecordProcessTime = getCreateRecordProcessTime(armResponseCreateRecord); + setEodCreateRecordProcessTimestamp(externalObjectDirectoryId, createRecordProcessTime); + } catch (Exception e) { + log.error("Unable to set EOD create record process timestamp for EOD {} - upload file {}", + externalObjectDirectoryId, createRecordFilenameAndPath, e); + } + } + + private void setEodCreateRecordProcessTimestamp(Long externalObjectDirectoryId, OffsetDateTime createRecordProcessTime) { + ExternalObjectDirectoryEntity externalObjectDirectory = getExternalObjectDirectoryEntity(externalObjectDirectoryId); + if (nonNull(externalObjectDirectory)) { + externalObjectDirectory.setCreateRecordProcessedTs(createRecordProcessTime); + externalObjectDirectoryRepository.save(externalObjectDirectory); + } + } + private ArmResponseCreateRecord getResponseCreateRecordOrDelete(String createRecordFilenameAndPath, BinaryData createRecordBinary) throws IOException { try { diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceImpl.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceImpl.java index a0165a9d08a..a20ee55e204 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceImpl.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceImpl.java @@ -36,8 +36,8 @@ public class ArmRpoBacklogCatchupServiceImpl implements ArmRpoBacklogCatchupServ @Override public void performCatchup(Integer batchSize, Integer maxHoursEndingPoint, Integer totalCatchupHours, Duration threadSleepDuration) { var armRpoExecutionDetailEntity = armRpoService.getLatestArmRpoExecutionDetailEntity(); - var earliestEodInRpo = externalObjectDirectoryRepository.findOldestByInputUploadProcessedTsAndStatusAndLocation(EodHelper.armRpoPendingStatus(), - EodHelper.armLocation()); + var earliestEodInRpo = externalObjectDirectoryRepository.findOldestByCreateRecordProcessedTsAndStatusAndLocation(EodHelper.armRpoPendingStatus(), + EodHelper.armLocation()); // Only perform backlog catchup if the last execution is in REMOVE_PRODUCTION state or FAILED status if (!validateTaskCanBeRun(maxHoursEndingPoint, totalCatchupHours, armRpoExecutionDetailEntity, earliestEodInRpo)) { @@ -56,12 +56,15 @@ public void performCatchup(Integer batchSize, Integer maxHoursEndingPoint, Integ return; } - OffsetDateTime inputUploadProcessedTs = earliestEodInRpo.getInputUploadProcessedTs(); - // subtract 10 minutes to account for any potential delay in EOD being picked up for RPO search after the input upload processed timestamp - OffsetDateTime adjustedOldestEodDateTime = inputUploadProcessedTs.minus(PRE_AMBLE_MINUTES, ChronoUnit.MINUTES); + // Use createRecordProcessedTs if it's been set else use inputUploadProcessedTs + OffsetDateTime processedTs = isNull(earliestEodInRpo.getCreateRecordProcessedTs()) + ? earliestEodInRpo.getInputUploadProcessedTs() : earliestEodInRpo.getCreateRecordProcessedTs(); + // subtract 10 minutes to account for any potential delay in EOD being picked up for RPO search after + // the create record processed timestamp + OffsetDateTime adjustedOldestEodDateTime = processedTs.minus(PRE_AMBLE_MINUTES, ChronoUnit.MINUTES); int hoursEnd = (int) calculateHoursFromStartToNow(adjustedOldestEodDateTime.toString()); - Integer originalStaartHour = armAutomatedTaskEntity.getRpoCsvStartHour(); + Integer originalStartHour = armAutomatedTaskEntity.getRpoCsvStartHour(); Integer originalEndHour = armAutomatedTaskEntity.getRpoCsvEndHour(); try { armAutomatedTaskEntity.setRpoCsvStartHour(hoursEnd - totalCatchupHours); @@ -70,7 +73,7 @@ public void performCatchup(Integer batchSize, Integer maxHoursEndingPoint, Integ triggerArmRpoSearchService.triggerArmRpoSearch(threadSleepDuration); } catch (Exception e) { // reset the start and end hour to the original value if there are any failures - armAutomatedTaskEntity.setRpoCsvStartHour(originalStaartHour); + armAutomatedTaskEntity.setRpoCsvStartHour(originalStartHour); armAutomatedTaskEntity.setRpoCsvEndHour(originalEndHour); armAutomatedTaskRepository.save(armAutomatedTaskEntity); } @@ -98,7 +101,7 @@ private boolean validateTaskCanBeRun(Integer maxHoursEndingPoint, Integer totalC OffsetDateTime currentTime = currentTimeHelper.currentOffsetDateTime(); int amountToSubtract = maxHoursEndingPoint + totalCatchupHours; OffsetDateTime lastRunTaskDateTime = currentTime.minus(amountToSubtract, ChronoUnit.HOURS); - if (earliestEodInRpo.getInputUploadProcessedTs().isAfter(lastRunTaskDateTime)) { + if (isNull(earliestEodInRpo.getCreateRecordProcessedTs()) || earliestEodInRpo.getCreateRecordProcessedTs().isAfter(lastRunTaskDateTime)) { log.info("Earliest EODs found in ARM RPO pending status is not suitable for backlog catchup."); return false; } diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/DetsToArmBatchProcessResponseFilesImpl.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/DetsToArmBatchProcessResponseFilesImpl.java index 5311efaa4c8..53b98062e5c 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/DetsToArmBatchProcessResponseFilesImpl.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/DetsToArmBatchProcessResponseFilesImpl.java @@ -177,8 +177,8 @@ private void updateOsrIngestStatusToFailure(ObjectStateRecordEntity osr, } @Override - protected void markEodAsResponseProcessingFailed(ExternalObjectDirectoryEntity externalObjectDirectory, UserAccountEntity userAccount) { - super.markEodAsResponseProcessingFailed(externalObjectDirectory, userAccount); + protected void markEodAsMissingResponseFailure(ExternalObjectDirectoryEntity externalObjectDirectory, UserAccountEntity userAccount) { + super.markEodAsMissingResponseFailure(externalObjectDirectory, userAccount); ObjectStateRecordEntity objectStateRecordEntity = externalObjectDirectory.getObjectStateRecordEntity(); diff --git a/src/main/java/uk/gov/hmcts/darts/common/entity/ExternalObjectDirectoryEntity.java b/src/main/java/uk/gov/hmcts/darts/common/entity/ExternalObjectDirectoryEntity.java index d5fd50dd3ce..6725eeb5e32 100644 --- a/src/main/java/uk/gov/hmcts/darts/common/entity/ExternalObjectDirectoryEntity.java +++ b/src/main/java/uk/gov/hmcts/darts/common/entity/ExternalObjectDirectoryEntity.java @@ -109,6 +109,9 @@ public class ExternalObjectDirectoryEntity extends CreatedModifiedBaseEntity imp @Column(name = "is_dets", nullable = false) private boolean isDets; + @Column(name = "create_record_processed_ts") + private OffsetDateTime createRecordProcessedTs; + @Override public int getStatusId() { return getStatus().getId(); diff --git a/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java b/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java index 38b27020bd5..ae2e51bf568 100644 --- a/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java +++ b/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java @@ -731,9 +731,9 @@ List findIdsByStatusAndLastModifiedBetweenAndLocationAndLimit(@Param("stat SELECT eod FROM ExternalObjectDirectoryEntity eod WHERE eod.status = :status AND eod.externalLocationType = :locationType - ORDER BY eod.inputUploadProcessedTs ASC + ORDER BY eod.createRecordProcessedTs ASC LIMIT 1 """) - ExternalObjectDirectoryEntity findOldestByInputUploadProcessedTsAndStatusAndLocation(ObjectRecordStatusEntity status, - ExternalLocationTypeEntity locationType); + ExternalObjectDirectoryEntity findOldestByCreateRecordProcessedTsAndStatusAndLocation(ObjectRecordStatusEntity status, + ExternalLocationTypeEntity locationType); } \ No newline at end of file diff --git a/src/main/resources/db/migration/common/V1_510__add_column_create_record_process_time_to_eod.sql b/src/main/resources/db/migration/common/V1_510__add_column_create_record_process_time_to_eod.sql new file mode 100644 index 00000000000..1bbab2df6a4 --- /dev/null +++ b/src/main/resources/db/migration/common/V1_510__add_column_create_record_process_time_to_eod.sql @@ -0,0 +1,9 @@ +ALTER TABLE external_object_directory + ADD COLUMN IF NOT EXISTS create_record_processed_ts TIMESTAMP WITH TIME ZONE; + +UPDATE external_object_directory +SET create_record_processed_ts = input_upload_processed_ts +WHERE create_record_processed_ts is null + and input_upload_processed_ts is not null; + +CREATE INDEX IF NOT EXISTS eod_cr_proc_ts_idx ON external_object_directory (create_record_processed_ts); diff --git a/src/main/resources/db/reference/modernised_darts_indexes.sql b/src/main/resources/db/reference/modernised_darts_indexes.sql index 8c2fa486eb9..eaacd98e14e 100644 --- a/src/main/resources/db/reference/modernised_darts_indexes.sql +++ b/src/main/resources/db/reference/modernised_darts_indexes.sql @@ -35,279 +35,281 @@ SET ROLE DARTS_OWNER; SET SEARCH_PATH TO darts; -- v60 application -CREATE INDEX ann_cur_own_fk ON ANNOTATION(current_owner) TABLESPACE pg_default; -CREATE INDEX ann_cre_by_fk ON ANNOTATION(created_by) TABLESPACE pg_default; -CREATE INDEX ann_lst_mod_by_fk ON ANNOTATION(last_modified_by) TABLESPACE pg_default; -CREATE INDEX ann_del_by_fk ON ANNOTATION(deleted_by) TABLESPACE pg_default; +CREATE INDEX ann_cur_own_fk ON ANNOTATION (current_owner) TABLESPACE pg_default; +CREATE INDEX ann_cre_by_fk ON ANNOTATION (created_by) TABLESPACE pg_default; +CREATE INDEX ann_lst_mod_by_fk ON ANNOTATION (last_modified_by) TABLESPACE pg_default; +CREATE INDEX ann_del_by_fk ON ANNOTATION (deleted_by) TABLESPACE pg_default; -CREATE INDEX ado_ann_fk ON ANNOTATION_DOCUMENT(ann_id) TABLESPACE pg_default; -CREATE INDEX ado_del_by_fk ON ANNOTATION_DOCUMENT(deleted_by) TABLESPACE pg_default; -CREATE INDEX ado_last_mod_by_fk ON ANNOTATION_DOCUMENT(last_modified_by) TABLESPACE pg_default; -CREATE INDEX ado_up_by_fk ON ANNOTATION_DOCUMENT(uploaded_by) TABLESPACE pg_default; +CREATE INDEX ado_ann_fk ON ANNOTATION_DOCUMENT (ann_id) TABLESPACE pg_default; +CREATE INDEX ado_del_by_fk ON ANNOTATION_DOCUMENT (deleted_by) TABLESPACE pg_default; +CREATE INDEX ado_last_mod_by_fk ON ANNOTATION_DOCUMENT (last_modified_by) TABLESPACE pg_default; +CREATE INDEX ado_up_by_fk ON ANNOTATION_DOCUMENT (uploaded_by) TABLESPACE pg_default; -CREATE INDEX aat_aut_fk ON ARM_AUTOMATED_TASK(aut_id) TABLESPACE pg_default; +CREATE INDEX aat_aut_fk ON ARM_AUTOMATED_TASK (aut_id) TABLESPACE pg_default; -CREATE INDEX ard_are_fk ON ARM_RPO_EXECUTION_DETAIL(are_id) TABLESPACE pg_default; -CREATE INDEX ard_aru_fk ON ARM_RPO_EXECUTION_DETAIL(aru_id) TABLESPACE pg_default; -CREATE INDEX ard_cre_by_fk ON ARM_RPO_EXECUTION_DETAIL(created_by) TABLESPACE pg_default; -CREATE INDEX ard_lst_mod_by_fk ON ARM_RPO_EXECUTION_DETAIL(last_modified_by) TABLESPACE pg_default; +CREATE INDEX ard_are_fk ON ARM_RPO_EXECUTION_DETAIL (are_id) TABLESPACE pg_default; +CREATE INDEX ard_aru_fk ON ARM_RPO_EXECUTION_DETAIL (aru_id) TABLESPACE pg_default; +CREATE INDEX ard_cre_by_fk ON ARM_RPO_EXECUTION_DETAIL (created_by) TABLESPACE pg_default; +CREATE INDEX ard_lst_mod_by_fk ON ARM_RPO_EXECUTION_DETAIL (last_modified_by) TABLESPACE pg_default; -CREATE INDEX aud_cas_fk ON AUDIT(cas_id) TABLESPACE pg_default; -CREATE INDEX aud_aua_fk ON AUDIT(aua_id) TABLESPACE pg_default; -CREATE INDEX aud_usr_fk ON AUDIT(usr_id) TABLESPACE pg_default; -CREATE INDEX aud_cre_by_fk ON AUDIT(created_by) TABLESPACE pg_default; -CREATE INDEX aud_lst_mod_by_fk ON AUDIT(last_modified_by) TABLESPACE pg_default; +CREATE INDEX aud_cas_fk ON AUDIT (cas_id) TABLESPACE pg_default; +CREATE INDEX aud_aua_fk ON AUDIT (aua_id) TABLESPACE pg_default; +CREATE INDEX aud_usr_fk ON AUDIT (usr_id) TABLESPACE pg_default; +CREATE INDEX aud_cre_by_fk ON AUDIT (created_by) TABLESPACE pg_default; +CREATE INDEX aud_lst_mod_by_fk ON AUDIT (last_modified_by) TABLESPACE pg_default; -CREATE INDEX aua_cre_by_fk ON AUDIT_ACTIVITY(created_by) TABLESPACE pg_default; -CREATE INDEX aua_lst_mod_by_fk ON AUDIT_ACTIVITY(last_modified_by) TABLESPACE pg_default; +CREATE INDEX aua_cre_by_fk ON AUDIT_ACTIVITY (created_by) TABLESPACE pg_default; +CREATE INDEX aua_lst_mod_by_fk ON AUDIT_ACTIVITY (last_modified_by) TABLESPACE pg_default; -CREATE INDEX aut_cre_by_fk ON AUTOMATED_TASK(created_by) TABLESPACE pg_default; -CREATE INDEX aut_lst_mod_by_fk ON AUTOMATED_TASK(last_modified_by) TABLESPACE pg_default; +CREATE INDEX aut_cre_by_fk ON AUTOMATED_TASK (created_by) TABLESPACE pg_default; +CREATE INDEX aut_lst_mod_by_fk ON AUTOMATED_TASK (last_modified_by) TABLESPACE pg_default; -CREATE INDEX cad_cas_fk ON CASE_DOCUMENT(cas_id) TABLESPACE pg_default; -CREATE INDEX cad_cre_by_fk ON CASE_DOCUMENT(created_by) TABLESPACE pg_default; -CREATE INDEX cad_lst_mod_by_fk ON CASE_DOCUMENT(last_modified_by) TABLESPACE pg_default; -CREATE INDEX cad_del_by_fk ON CASE_DOCUMENT(deleted_by) TABLESPACE pg_default; +CREATE INDEX cad_cas_fk ON CASE_DOCUMENT (cas_id) TABLESPACE pg_default; +CREATE INDEX cad_cre_by_fk ON CASE_DOCUMENT (created_by) TABLESPACE pg_default; +CREATE INDEX cad_lst_mod_by_fk ON CASE_DOCUMENT (last_modified_by) TABLESPACE pg_default; +CREATE INDEX cad_del_by_fk ON CASE_DOCUMENT (deleted_by) TABLESPACE pg_default; --CREATE INDEX caj_cas_fk ON CASE_JUDGE_AE(cas_id); -CREATE INDEX caj_jud_fk ON CASE_JUDGE_AE(jud_id) TABLESPACE pg_default; +CREATE INDEX caj_jud_fk ON CASE_JUDGE_AE (jud_id) TABLESPACE pg_default; -CREATE INDEX caj_tra_fk ON CASE_TRANSCRIPTION_AE(tra_id) TABLESPACE pg_default; +CREATE INDEX caj_tra_fk ON CASE_TRANSCRIPTION_AE (tra_id) TABLESPACE pg_default; -CREATE INDEX cas_evh_fk ON COURT_CASE(evh_id) TABLESPACE pg_default; -CREATE INDEX cas_cth_fk ON COURT_CASE(cth_id) TABLESPACE pg_default; -CREATE INDEX cas_cre_by_fk ON COURT_CASE(created_by) TABLESPACE pg_default; -CREATE INDEX cas_lst_mod_by_fk ON COURT_CASE(last_modified_by) TABLESPACE pg_default; -CREATE INDEX cas_dat_anon_by_fk ON COURT_CASE(data_anonymised_by) TABLESPACE pg_default; -CREATE INDEX cas_del_by_fk ON COURT_CASE(deleted_by) TABLESPACE pg_default; +CREATE INDEX cas_evh_fk ON COURT_CASE (evh_id) TABLESPACE pg_default; +CREATE INDEX cas_cth_fk ON COURT_CASE (cth_id) TABLESPACE pg_default; +CREATE INDEX cas_cre_by_fk ON COURT_CASE (created_by) TABLESPACE pg_default; +CREATE INDEX cas_lst_mod_by_fk ON COURT_CASE (last_modified_by) TABLESPACE pg_default; +CREATE INDEX cas_dat_anon_by_fk ON COURT_CASE (data_anonymised_by) TABLESPACE pg_default; +CREATE INDEX cas_del_by_fk ON COURT_CASE (deleted_by) TABLESPACE pg_default; -CREATE INDEX cth_cre_by_fk ON COURTHOUSE(created_by) TABLESPACE pg_default; -CREATE INDEX cth_lst_mod_by_fk ON COURTHOUSE(last_modified_by) TABLESPACE pg_default; +CREATE INDEX cth_cre_by_fk ON COURTHOUSE (created_by) TABLESPACE pg_default; +CREATE INDEX cth_lst_mod_by_fk ON COURTHOUSE (last_modified_by) TABLESPACE pg_default; --CREATE INDEX cra_cth_fk ON COURTHOUSE_REGION_AE(cth_id); -CREATE INDEX cra_reg_fk ON COURTHOUSE_REGION_AE(reg_id) TABLESPACE pg_default; +CREATE INDEX cra_reg_fk ON COURTHOUSE_REGION_AE (reg_id) TABLESPACE pg_default; -CREATE INDEX ctr_cre_by_fk ON COURTROOM(created_by) TABLESPACE pg_default; +CREATE INDEX ctr_cre_by_fk ON COURTROOM (created_by) TABLESPACE pg_default; -CREATE INDEX dan_eve_fk ON DATA_ANONYMISATION(eve_id) TABLESPACE pg_default; -CREATE INDEX dan_trc_fk ON DATA_ANONYMISATION(trc_id) TABLESPACE pg_default; -CREATE INDEX dan_req_by_fk ON DATA_ANONYMISATION(requested_by) TABLESPACE pg_default; -CREATE INDEX dan_apprv_by_fk ON DATA_ANONYMISATION(approved_by) TABLESPACE pg_default; +CREATE INDEX dan_eve_fk ON DATA_ANONYMISATION (eve_id) TABLESPACE pg_default; +CREATE INDEX dan_trc_fk ON DATA_ANONYMISATION (trc_id) TABLESPACE pg_default; +CREATE INDEX dan_req_by_fk ON DATA_ANONYMISATION (requested_by) TABLESPACE pg_default; +CREATE INDEX dan_apprv_by_fk ON DATA_ANONYMISATION (approved_by) TABLESPACE pg_default; -CREATE INDEX dal_elt_fk ON DAILY_LIST(elt_id) TABLESPACE pg_default; -CREATE INDEX dal_cre_by_fk ON DAILY_LIST(created_by) TABLESPACE pg_default; -CREATE INDEX dal_lst_mod_by ON DAILY_LIST(last_modified_by) TABLESPACE pg_default; +CREATE INDEX dal_elt_fk ON DAILY_LIST (elt_id) TABLESPACE pg_default; +CREATE INDEX dal_cre_by_fk ON DAILY_LIST (created_by) TABLESPACE pg_default; +CREATE INDEX dal_lst_mod_by ON DAILY_LIST (last_modified_by) TABLESPACE pg_default; -CREATE INDEX dfc_cre_by_fk ON DEFENCE(created_by) TABLESPACE pg_default; -CREATE INDEX dfc_lst_mod_by_fk ON DEFENCE(last_modified_by) TABLESPACE pg_default; +CREATE INDEX dfc_cre_by_fk ON DEFENCE (created_by) TABLESPACE pg_default; +CREATE INDEX dfc_lst_mod_by_fk ON DEFENCE (last_modified_by) TABLESPACE pg_default; -CREATE INDEX dfd_cre_by_fk ON DEFENDANT(created_by) TABLESPACE pg_default; -CREATE INDEX dfd_lst_mod_by_fk ON DEFENDANT(last_modified_by) TABLESPACE pg_default; +CREATE INDEX dfd_cre_by_fk ON DEFENDANT (created_by) TABLESPACE pg_default; +CREATE INDEX dfd_lst_mod_by_fk ON DEFENDANT (last_modified_by) TABLESPACE pg_default; -CREATE INDEX eve_ctr_fk ON EVENT(ctr_id) TABLESPACE pg_default; -CREATE INDEX eve_evh_fk ON EVENT(evh_id) TABLESPACE pg_default; -CREATE INDEX eve_cre_by_fk ON EVENT(created_by) TABLESPACE pg_default; -CREATE INDEX eve_lst_mod_by_fk ON EVENT(last_modified_by) TABLESPACE pg_default; +CREATE INDEX eve_ctr_fk ON EVENT (ctr_id) TABLESPACE pg_default; +CREATE INDEX eve_evh_fk ON EVENT (evh_id) TABLESPACE pg_default; +CREATE INDEX eve_cre_by_fk ON EVENT (created_by) TABLESPACE pg_default; +CREATE INDEX eve_lst_mod_by_fk ON EVENT (last_modified_by) TABLESPACE pg_default; -CREATE INDEX evh_cre_by_fk ON EVENT_HANDLER(created_by) TABLESPACE pg_default; +CREATE INDEX evh_cre_by_fk ON EVENT_HANDLER (created_by) TABLESPACE pg_default; -CREATE INDEX elc_eve_fk ON EVENT_LINKED_CASE(eve_id) TABLESPACE pg_default; -CREATE INDEX elc_cas_fk ON EVENT_LINKED_CASE(cas_id) TABLESPACE pg_default; +CREATE INDEX elc_eve_fk ON EVENT_LINKED_CASE (eve_id) TABLESPACE pg_default; +CREATE INDEX elc_cas_fk ON EVENT_LINKED_CASE (cas_id) TABLESPACE pg_default; -CREATE INDEX eod_med_fk ON EXTERNAL_OBJECT_DIRECTORY(med_id) TABLESPACE pg_default; -CREATE INDEX eod_trd_fk ON EXTERNAL_OBJECT_DIRECTORY(trd_id) TABLESPACE pg_default; -CREATE INDEX eod_cad_fk ON EXTERNAL_OBJECT_DIRECTORY(cad_id) TABLESPACE pg_default; -CREATE INDEX eod_ado_fk ON EXTERNAL_OBJECT_DIRECTORY(ado_id) TABLESPACE pg_default; -CREATE INDEX eod_cre_by_fk ON EXTERNAL_OBJECT_DIRECTORY(created_by) TABLESPACE pg_default; -CREATE INDEX eod_lst_mod_by_fk ON EXTERNAL_OBJECT_DIRECTORY(last_modified_by) TABLESPACE pg_default; -CREATE INDEX eod_ors_fk ON EXTERNAL_OBJECT_DIRECTORY(ors_id) TABLESPACE pg_default; -CREATE INDEX eod_elt_fk ON EXTERNAL_OBJECT_DIRECTORY(elt_id) TABLESPACE pg_default; +CREATE INDEX eod_med_fk ON EXTERNAL_OBJECT_DIRECTORY (med_id) TABLESPACE pg_default; +CREATE INDEX eod_trd_fk ON EXTERNAL_OBJECT_DIRECTORY (trd_id) TABLESPACE pg_default; +CREATE INDEX eod_cad_fk ON EXTERNAL_OBJECT_DIRECTORY (cad_id) TABLESPACE pg_default; +CREATE INDEX eod_ado_fk ON EXTERNAL_OBJECT_DIRECTORY (ado_id) TABLESPACE pg_default; +CREATE INDEX eod_cre_by_fk ON EXTERNAL_OBJECT_DIRECTORY (created_by) TABLESPACE pg_default; +CREATE INDEX eod_lst_mod_by_fk ON EXTERNAL_OBJECT_DIRECTORY (last_modified_by) TABLESPACE pg_default; +CREATE INDEX eod_ors_fk ON EXTERNAL_OBJECT_DIRECTORY (ors_id) TABLESPACE pg_default; +CREATE INDEX eod_elt_fk ON EXTERNAL_OBJECT_DIRECTORY (elt_id) TABLESPACE pg_default; -CREATE INDEX esa_cre_by_fk ON EXTERNAL_SERVICE_AUTH_TOKEN(created_by) TABLESPACE pg_default; -CREATE INDEX esa_lst_mod_by_fk ON EXTERNAL_SERVICE_AUTH_TOKEN(last_modified_by) TABLESPACE pg_default; +CREATE INDEX eod_cr_proc_ts_idx ON EXTERNAL_OBJECT_DIRECTORY (create_record_processed_ts) TABLESPACE pg_default; -CREATE INDEX hea_ctr_fk ON HEARING(ctr_id) TABLESPACE pg_default; -CREATE INDEX hea_cre_by_fk ON HEARING(created_by) TABLESPACE pg_default; -CREATE INDEX hea_lst_mod_by_fk ON HEARING(last_modified_by) TABLESPACE pg_default; +CREATE INDEX esa_cre_by_fk ON EXTERNAL_SERVICE_AUTH_TOKEN (created_by) TABLESPACE pg_default; +CREATE INDEX esa_lst_mod_by_fk ON EXTERNAL_SERVICE_AUTH_TOKEN (last_modified_by) TABLESPACE pg_default; + +CREATE INDEX hea_ctr_fk ON HEARING (ctr_id) TABLESPACE pg_default; +CREATE INDEX hea_cre_by_fk ON HEARING (created_by) TABLESPACE pg_default; +CREATE INDEX hea_lst_mod_by_fk ON HEARING (last_modified_by) TABLESPACE pg_default; --CREATE INDEX haa_hea_fk ON HEARING_ANNOTATION_AE(hea_id); -CREATE INDEX haa_ann_fk ON HEARING_ANNOTATION_AE(ann_id) TABLESPACE pg_default; +CREATE INDEX haa_ann_fk ON HEARING_ANNOTATION_AE (ann_id) TABLESPACE pg_default; --CREATE INDEX hee_hea_fk ON HEARING_EVENT_AE(hea_id); -CREATE INDEX hee_eve_fk ON HEARING_EVENT_AE(eve_id) TABLESPACE pg_default; +CREATE INDEX hee_eve_fk ON HEARING_EVENT_AE (eve_id) TABLESPACE pg_default; --CREATE INDEX hej_hea_fk ON HEARING_JUDGE_AE(hea_id); -CREATE INDEX hej_jud_fk ON HEARING_JUDGE_AE(jud_id) TABLESPACE pg_default; +CREATE INDEX hej_jud_fk ON HEARING_JUDGE_AE (jud_id) TABLESPACE pg_default; --CREATE INDEX hem_hea_fk ON HEARING_MEDIA_AE(hea_id); -CREATE INDEX hem_med_fk ON HEARING_MEDIA_AE(med_id) TABLESPACE pg_default; +CREATE INDEX hem_med_fk ON HEARING_MEDIA_AE (med_id) TABLESPACE pg_default; -CREATE INDEX het_tra_fk ON HEARING_TRANSCRIPTION_AE(tra_id) TABLESPACE pg_default; +CREATE INDEX het_tra_fk ON HEARING_TRANSCRIPTION_AE (tra_id) TABLESPACE pg_default; -CREATE INDEX jud_cre_by_fk ON JUDGE(created_by) TABLESPACE pg_default; -CREATE INDEX jud_lst_mod_by_fk ON JUDGE(last_modified_by) TABLESPACE pg_default; +CREATE INDEX jud_cre_by_fk ON JUDGE (created_by) TABLESPACE pg_default; +CREATE INDEX jud_lst_mod_by_fk ON JUDGE (last_modified_by) TABLESPACE pg_default; -CREATE INDEX med_ctr_id ON MEDIA(ctr_id) TABLESPACE pg_default; -CREATE INDEX med_cre_by_fk ON MEDIA(created_by) TABLESPACE pg_default; -CREATE INDEX med_lst_mod_by_fk ON MEDIA(last_modified_by) TABLESPACE pg_default; -CREATE INDEX med_del_by_fk ON MEDIA(deleted_by) TABLESPACE pg_default; +CREATE INDEX med_ctr_id ON MEDIA (ctr_id) TABLESPACE pg_default; +CREATE INDEX med_cre_by_fk ON MEDIA (created_by) TABLESPACE pg_default; +CREATE INDEX med_lst_mod_by_fk ON MEDIA (last_modified_by) TABLESPACE pg_default; +CREATE INDEX med_del_by_fk ON MEDIA (deleted_by) TABLESPACE pg_default; -CREATE INDEX mlc_med_fk ON MEDIA_LINKED_CASE(med_id) TABLESPACE pg_default; -CREATE INDEX mlc_cas_fk ON MEDIA_LINKED_CASE(cas_id) TABLESPACE pg_default; -CREATE INDEX mlc_cre_by_fk ON MEDIA_LINKED_CASE(created_by) TABLESPACE pg_default; +CREATE INDEX mlc_med_fk ON MEDIA_LINKED_CASE (med_id) TABLESPACE pg_default; +CREATE INDEX mlc_cas_fk ON MEDIA_LINKED_CASE (cas_id) TABLESPACE pg_default; +CREATE INDEX mlc_cre_by_fk ON MEDIA_LINKED_CASE (created_by) TABLESPACE pg_default; -CREATE INDEX mer_hea_fk ON MEDIA_REQUEST(hea_id) TABLESPACE pg_default; -CREATE INDEX mer_cre_by_fk ON MEDIA_REQUEST(created_by) TABLESPACE pg_default; -CREATE INDEX mer_lst_mod_by_fk ON MEDIA_REQUEST(last_modified_by) TABLESPACE pg_default; -CREATE INDEX mer_req_fk ON MEDIA_REQUEST(requestor) TABLESPACE pg_default; -CREATE INDEX mer_cur_own_fk ON MEDIA_REQUEST(current_owner) TABLESPACE pg_default; +CREATE INDEX mer_hea_fk ON MEDIA_REQUEST (hea_id) TABLESPACE pg_default; +CREATE INDEX mer_cre_by_fk ON MEDIA_REQUEST (created_by) TABLESPACE pg_default; +CREATE INDEX mer_lst_mod_by_fk ON MEDIA_REQUEST (last_modified_by) TABLESPACE pg_default; +CREATE INDEX mer_req_fk ON MEDIA_REQUEST (requestor) TABLESPACE pg_default; +CREATE INDEX mer_cur_own_fk ON MEDIA_REQUEST (current_owner) TABLESPACE pg_default; -CREATE INDEX nod_ctr_fk ON NODE_REGISTER(ctr_id) TABLESPACE pg_default; -CREATE INDEX nod_cre_by_fk ON NODE_REGISTER(created_by) TABLESPACE pg_default; +CREATE INDEX nod_ctr_fk ON NODE_REGISTER (ctr_id) TABLESPACE pg_default; +CREATE INDEX nod_cre_by_fk ON NODE_REGISTER (created_by) TABLESPACE pg_default; -CREATE INDEX not_cas_fk ON NOTIFICATION(cas_id) TABLESPACE pg_default; -CREATE INDEX not_cre_by_fk ON NOTIFICATION(created_by) TABLESPACE pg_default; -CREATE INDEX not_lst_mod_fk ON NOTIFICATION(last_modified_by) TABLESPACE pg_default; +CREATE INDEX not_cas_fk ON NOTIFICATION (cas_id) TABLESPACE pg_default; +CREATE INDEX not_cre_by_fk ON NOTIFICATION (created_by) TABLESPACE pg_default; +CREATE INDEX not_lst_mod_fk ON NOTIFICATION (last_modified_by) TABLESPACE pg_default; -CREATE INDEX oaa_ado_fk ON OBJECT_ADMIN_ACTION(ado_id) TABLESPACE pg_default; -CREATE INDEX oaa_cad_fk ON OBJECT_ADMIN_ACTION(cad_id) TABLESPACE pg_default; -CREATE INDEX oaa_med_fk ON OBJECT_ADMIN_ACTION(med_id) TABLESPACE pg_default; -CREATE INDEX oaa_trd_fk ON OBJECT_ADMIN_ACTION(trd_id) TABLESPACE pg_default; -CREATE INDEX oaa_ohr_fk ON OBJECT_ADMIN_ACTION(ohr_id) TABLESPACE pg_default; -CREATE INDEX oaa_hid_by_fk ON OBJECT_ADMIN_ACTION(hidden_by) TABLESPACE pg_default; -CREATE INDEX oaa_mkd_del_by_fk ON OBJECT_ADMIN_ACTION(marked_for_manual_del_by) TABLESPACE pg_default; +CREATE INDEX oaa_ado_fk ON OBJECT_ADMIN_ACTION (ado_id) TABLESPACE pg_default; +CREATE INDEX oaa_cad_fk ON OBJECT_ADMIN_ACTION (cad_id) TABLESPACE pg_default; +CREATE INDEX oaa_med_fk ON OBJECT_ADMIN_ACTION (med_id) TABLESPACE pg_default; +CREATE INDEX oaa_trd_fk ON OBJECT_ADMIN_ACTION (trd_id) TABLESPACE pg_default; +CREATE INDEX oaa_ohr_fk ON OBJECT_ADMIN_ACTION (ohr_id) TABLESPACE pg_default; +CREATE INDEX oaa_hid_by_fk ON OBJECT_ADMIN_ACTION (hidden_by) TABLESPACE pg_default; +CREATE INDEX oaa_mkd_del_by_fk ON OBJECT_ADMIN_ACTION (marked_for_manual_del_by) TABLESPACE pg_default; -CREATE INDEX orq_med_fk ON OBJECT_RETRIEVAL_QUEUE(med_id) TABLESPACE pg_default; -CREATE INDEX orq_trd_fk ON OBJECT_RETRIEVAL_QUEUE(trd_id) TABLESPACE pg_default; -CREATE INDEX orq_cre_by_fk ON OBJECT_RETRIEVAL_QUEUE(created_by) TABLESPACE pg_default; -CREATE INDEX orq_lst_mod_by_fk ON OBJECT_RETRIEVAL_QUEUE(last_modified_by) TABLESPACE pg_default; +CREATE INDEX orq_med_fk ON OBJECT_RETRIEVAL_QUEUE (med_id) TABLESPACE pg_default; +CREATE INDEX orq_trd_fk ON OBJECT_RETRIEVAL_QUEUE (trd_id) TABLESPACE pg_default; +CREATE INDEX orq_cre_by_fk ON OBJECT_RETRIEVAL_QUEUE (created_by) TABLESPACE pg_default; +CREATE INDEX orq_lst_mod_by_fk ON OBJECT_RETRIEVAL_QUEUE (last_modified_by) TABLESPACE pg_default; -CREATE INDEX prn_cre_by_fk ON PROSECUTOR(created_by) TABLESPACE pg_default; -CREATE INDEX prn_lst_mod_by_fk ON PROSECUTOR(last_modified_by) TABLESPACE pg_default; +CREATE INDEX prn_cre_by_fk ON PROSECUTOR (created_by) TABLESPACE pg_default; +CREATE INDEX prn_lst_mod_by_fk ON PROSECUTOR (last_modified_by) TABLESPACE pg_default; -CREATE INDEX rep_cre_by_fk ON REPORT(created_by) TABLESPACE pg_default; -CREATE INDEX rep_lst_mod_by_fk ON REPORT(last_modified_by) TABLESPACE pg_default; +CREATE INDEX rep_cre_by_fk ON REPORT (created_by) TABLESPACE pg_default; +CREATE INDEX rep_lst_mod_by_fk ON REPORT (last_modified_by) TABLESPACE pg_default; --CREATE INDEX tra_cas_fk ON TRANSCRIPTION(cas_id) TABLESPACE pg_default; -CREATE INDEX tra_ctr_fk ON TRANSCRIPTION(ctr_id) TABLESPACE pg_default; -CREATE INDEX tra_trs_fk ON TRANSCRIPTION(trs_id) TABLESPACE pg_default; -CREATE INDEX tra_tru_fk ON TRANSCRIPTION(tru_id) TABLESPACE pg_default; +CREATE INDEX tra_ctr_fk ON TRANSCRIPTION (ctr_id) TABLESPACE pg_default; +CREATE INDEX tra_trs_fk ON TRANSCRIPTION (trs_id) TABLESPACE pg_default; +CREATE INDEX tra_tru_fk ON TRANSCRIPTION (tru_id) TABLESPACE pg_default; --CREATE INDEX tra_hea_fk ON TRANSCRIPTION(hea_id) TABLESPACE pg_default; -CREATE INDEX tra_cre_by_fk ON TRANSCRIPTION(created_by) TABLESPACE pg_default; -CREATE INDEX tra_lst_mod_by_fk ON TRANSCRIPTION(last_modified_by) TABLESPACE pg_default; -CREATE INDEX tra_trt_fk ON TRANSCRIPTION(trt_id) TABLESPACE pg_default; -CREATE INDEX tra_del_by_fk ON TRANSCRIPTION(deleted_by) TABLESPACE pg_default; -CREATE INDEX tra_req_by_fk ON TRANSCRIPTION(requested_by) TABLESPACE pg_default; - -CREATE INDEX trc_tra_fk ON TRANSCRIPTION_COMMENT(tra_id) TABLESPACE pg_default; -CREATE INDEX trc_trw_fk ON TRANSCRIPTION_COMMENT(trw_id) TABLESPACE pg_default; -CREATE INDEX trc_auth_fk ON TRANSCRIPTION_COMMENT(author) TABLESPACE pg_default; -CREATE INDEX trc_cre_by_fk ON TRANSCRIPTION_COMMENT(created_by) TABLESPACE pg_default; -CREATE INDEX trc_lst_mod_by_fk ON TRANSCRIPTION_COMMENT(last_modified_by) TABLESPACE pg_default; - -CREATE INDEX trd_tra_fk ON TRANSCRIPTION_DOCUMENT(tra_id) TABLESPACE pg_default; -CREATE INDEX trd_del_by_fk ON TRANSCRIPTION_DOCUMENT(deleted_by) TABLESPACE pg_default; -CREATE INDEX trd_upl_by_fk ON TRANSCRIPTION_DOCUMENT(uploaded_by) TABLESPACE pg_default; -CREATE INDEX trd_lst_mod_by_fk ON TRANSCRIPTION_DOCUMENT(last_modified_by) TABLESPACE pg_default; - -CREATE INDEX tlc_tra_fk ON TRANSCRIPTION_LINKED_CASE(tra_id) TABLESPACE pg_default; -CREATE INDEX tlc_cas_fk ON TRANSCRIPTION_LINKED_CASE(cas_id) TABLESPACE pg_default; - -CREATE INDEX trw_tra_fk ON TRANSCRIPTION_WORKFLOW(tra_id) TABLESPACE pg_default; -CREATE INDEX trw_trs_fk ON TRANSCRIPTION_WORKFLOW(trs_id) TABLESPACE pg_default; -CREATE INDEX trw_wrkflw_act_fk ON TRANSCRIPTION_WORKFLOW(workflow_actor) TABLESPACE pg_default; - -CREATE INDEX trm_mer_fk ON TRANSFORMED_MEDIA(mer_id) TABLESPACE pg_default; -CREATE INDEX trm_cre_by_fk ON TRANSFORMED_MEDIA(created_by) TABLESPACE pg_default; -CREATE INDEX trm_lst_mod_by_fk ON TRANSFORMED_MEDIA(last_modified_by) TABLESPACE pg_default; - -CREATE INDEX tod_cre_by_fk ON TRANSIENT_OBJECT_DIRECTORY(created_by) TABLESPACE pg_default; -CREATE INDEX tod_lst_mod_by_fk ON TRANSIENT_OBJECT_DIRECTORY(last_modified_by) TABLESPACE pg_default; -CREATE INDEX tod_trm_fk ON TRANSIENT_OBJECT_DIRECTORY(trm_id) TABLESPACE pg_default; -CREATE INDEX tod_ors_fk ON TRANSIENT_OBJECT_DIRECTORY(ors_id) TABLESPACE pg_default; +CREATE INDEX tra_cre_by_fk ON TRANSCRIPTION (created_by) TABLESPACE pg_default; +CREATE INDEX tra_lst_mod_by_fk ON TRANSCRIPTION (last_modified_by) TABLESPACE pg_default; +CREATE INDEX tra_trt_fk ON TRANSCRIPTION (trt_id) TABLESPACE pg_default; +CREATE INDEX tra_del_by_fk ON TRANSCRIPTION (deleted_by) TABLESPACE pg_default; +CREATE INDEX tra_req_by_fk ON TRANSCRIPTION (requested_by) TABLESPACE pg_default; + +CREATE INDEX trc_tra_fk ON TRANSCRIPTION_COMMENT (tra_id) TABLESPACE pg_default; +CREATE INDEX trc_trw_fk ON TRANSCRIPTION_COMMENT (trw_id) TABLESPACE pg_default; +CREATE INDEX trc_auth_fk ON TRANSCRIPTION_COMMENT (author) TABLESPACE pg_default; +CREATE INDEX trc_cre_by_fk ON TRANSCRIPTION_COMMENT (created_by) TABLESPACE pg_default; +CREATE INDEX trc_lst_mod_by_fk ON TRANSCRIPTION_COMMENT (last_modified_by) TABLESPACE pg_default; + +CREATE INDEX trd_tra_fk ON TRANSCRIPTION_DOCUMENT (tra_id) TABLESPACE pg_default; +CREATE INDEX trd_del_by_fk ON TRANSCRIPTION_DOCUMENT (deleted_by) TABLESPACE pg_default; +CREATE INDEX trd_upl_by_fk ON TRANSCRIPTION_DOCUMENT (uploaded_by) TABLESPACE pg_default; +CREATE INDEX trd_lst_mod_by_fk ON TRANSCRIPTION_DOCUMENT (last_modified_by) TABLESPACE pg_default; + +CREATE INDEX tlc_tra_fk ON TRANSCRIPTION_LINKED_CASE (tra_id) TABLESPACE pg_default; +CREATE INDEX tlc_cas_fk ON TRANSCRIPTION_LINKED_CASE (cas_id) TABLESPACE pg_default; + +CREATE INDEX trw_tra_fk ON TRANSCRIPTION_WORKFLOW (tra_id) TABLESPACE pg_default; +CREATE INDEX trw_trs_fk ON TRANSCRIPTION_WORKFLOW (trs_id) TABLESPACE pg_default; +CREATE INDEX trw_wrkflw_act_fk ON TRANSCRIPTION_WORKFLOW (workflow_actor) TABLESPACE pg_default; + +CREATE INDEX trm_mer_fk ON TRANSFORMED_MEDIA (mer_id) TABLESPACE pg_default; +CREATE INDEX trm_cre_by_fk ON TRANSFORMED_MEDIA (created_by) TABLESPACE pg_default; +CREATE INDEX trm_lst_mod_by_fk ON TRANSFORMED_MEDIA (last_modified_by) TABLESPACE pg_default; + +CREATE INDEX tod_cre_by_fk ON TRANSIENT_OBJECT_DIRECTORY (created_by) TABLESPACE pg_default; +CREATE INDEX tod_lst_mod_by_fk ON TRANSIENT_OBJECT_DIRECTORY (last_modified_by) TABLESPACE pg_default; +CREATE INDEX tod_trm_fk ON TRANSIENT_OBJECT_DIRECTORY (trm_id) TABLESPACE pg_default; +CREATE INDEX tod_ors_fk ON TRANSIENT_OBJECT_DIRECTORY (ors_id) TABLESPACE pg_default; --v19 security -CREATE INDEX grp_rol_fk ON SECURITY_GROUP(rol_id) TABLESPACE pg_default; -CREATE INDEX grp_cre_by_fk ON SECURITY_GROUP(created_by) TABLESPACE pg_default; -CREATE INDEX grp_lst_mod_by_fk ON SECURITY_GROUP(last_modified_by) TABLESPACE pg_default; +CREATE INDEX grp_rol_fk ON SECURITY_GROUP (rol_id) TABLESPACE pg_default; +CREATE INDEX grp_cre_by_fk ON SECURITY_GROUP (created_by) TABLESPACE pg_default; +CREATE INDEX grp_lst_mod_by_fk ON SECURITY_GROUP (last_modified_by) TABLESPACE pg_default; --CREATE INDEX grc_grp_fk ON SECURITY_GROUP_COURTHOUSE_AE(grp_id); -CREATE INDEX grc_cth_fk ON SECURITY_GROUP_COURTHOUSE_AE(cth_id) TABLESPACE pg_default; +CREATE INDEX grc_cth_fk ON SECURITY_GROUP_COURTHOUSE_AE (cth_id) TABLESPACE pg_default; -CREATE INDEX gua_grp_fk ON SECURITY_GROUP_USER_ACCOUNT_AE(grp_id) TABLESPACE pg_default; +CREATE INDEX gua_grp_fk ON SECURITY_GROUP_USER_ACCOUNT_AE (grp_id) TABLESPACE pg_default; --CREATE INDEX gua_usr_fk ON SECURITY_GROUP_USER_ACCOUNT_AE(usr_id); --CREATE INDEX rop_rol_fk ON SECURITY_ROLE_PERMISSION_AE(rol_id); -CREATE INDEX rop_per_fk ON SECURITY_ROLE_PERMISSION_AE(per_id) TABLESPACE pg_default; +CREATE INDEX rop_per_fk ON SECURITY_ROLE_PERMISSION_AE (per_id) TABLESPACE pg_default; --v6 retention -CREATE INDEX car_cas_fk ON CASE_RETENTION(cas_id) TABLESPACE pg_default; -CREATE INDEX car_rpt_fk ON CASE_RETENTION(rpt_id) TABLESPACE pg_default; -CREATE INDEX car_cre_by_fk ON CASE_RETENTION(created_by) TABLESPACE pg_default; -CREATE INDEX car_lst_mod_by_fk ON CASE_RETENTION(last_modified_by) TABLESPACE pg_default; -CREATE INDEX car_sub_by_fk ON CASE_RETENTION(submitted_by) TABLESPACE pg_default; -CREATE INDEX car_cmr_fk ON CASE_RETENTION(cmr_id) TABLESPACE pg_default; +CREATE INDEX car_cas_fk ON CASE_RETENTION (cas_id) TABLESPACE pg_default; +CREATE INDEX car_rpt_fk ON CASE_RETENTION (rpt_id) TABLESPACE pg_default; +CREATE INDEX car_cre_by_fk ON CASE_RETENTION (created_by) TABLESPACE pg_default; +CREATE INDEX car_lst_mod_by_fk ON CASE_RETENTION (last_modified_by) TABLESPACE pg_default; +CREATE INDEX car_sub_by_fk ON CASE_RETENTION (submitted_by) TABLESPACE pg_default; +CREATE INDEX car_cmr_fk ON CASE_RETENTION (cmr_id) TABLESPACE pg_default; -CREATE INDEX cmr_cas_fk ON CASE_MANAGEMENT_RETENTION(cas_id) TABLESPACE pg_default; -CREATE INDEX cmr_rpt_fk ON CASE_MANAGEMENT_RETENTION(rpt_id) TABLESPACE pg_default; -CREATE INDEX cmr_eve_fk ON CASE_MANAGEMENT_RETENTION(eve_id) TABLESPACE pg_default; +CREATE INDEX cmr_cas_fk ON CASE_MANAGEMENT_RETENTION (cas_id) TABLESPACE pg_default; +CREATE INDEX cmr_rpt_fk ON CASE_MANAGEMENT_RETENTION (rpt_id) TABLESPACE pg_default; +CREATE INDEX cmr_eve_fk ON CASE_MANAGEMENT_RETENTION (eve_id) TABLESPACE pg_default; -CREATE INDEX rpt_cre_by_fk ON RETENTION_POLICY_TYPE(created_by) TABLESPACE pg_default; -CREATE INDEX rpt_lst_mod_by_fk ON RETENTION_POLICY_TYPE(last_modified_by) TABLESPACE pg_default; +CREATE INDEX rpt_cre_by_fk ON RETENTION_POLICY_TYPE (created_by) TABLESPACE pg_default; +CREATE INDEX rpt_lst_mod_by_fk ON RETENTION_POLICY_TYPE (last_modified_by) TABLESPACE pg_default; -CREATE INDEX rpr_rpt_fk ON RPS_RETAINER(rpt_id) TABLESPACE pg_default; -CREATE INDEX rpr_cre_by_fk ON RPS_RETAINER(created_by) TABLESPACE pg_default; -CREATE INDEX rpr_lst_mod_by_fk ON RPS_RETAINER(last_modified_by) TABLESPACE pg_default; +CREATE INDEX rpr_rpt_fk ON RPS_RETAINER (rpt_id) TABLESPACE pg_default; +CREATE INDEX rpr_cre_by_fk ON RPS_RETAINER (created_by) TABLESPACE pg_default; +CREATE INDEX rpr_lst_mod_by_fk ON RPS_RETAINER (last_modified_by) TABLESPACE pg_default; -CREATE INDEX rah_cas_fk ON CASE_RETENTION_AUDIT_HERITAGE(cas_id) TABLESPACE pg_default; -CREATE INDEX rah_rpt_fk ON CASE_RETENTION_AUDIT_HERITAGE(rpt_id) TABLESPACE pg_default; -CREATE INDEX rah_c_username_fk ON CASE_RETENTION_AUDIT_HERITAGE(c_username) TABLESPACE pg_default; -CREATE INDEX rah_r_creator_fk ON CASE_RETENTION_AUDIT_HERITAGE(r_creator_name) TABLESPACE pg_default; -CREATE INDEX rah_r_modifier_fk ON CASE_RETENTION_AUDIT_HERITAGE(r_modifier) TABLESPACE pg_default; -CREATE INDEX rah_owner_fk ON CASE_RETENTION_AUDIT_HERITAGE(owner_name) TABLESPACE pg_default; +CREATE INDEX rah_cas_fk ON CASE_RETENTION_AUDIT_HERITAGE (cas_id) TABLESPACE pg_default; +CREATE INDEX rah_rpt_fk ON CASE_RETENTION_AUDIT_HERITAGE (rpt_id) TABLESPACE pg_default; +CREATE INDEX rah_c_username_fk ON CASE_RETENTION_AUDIT_HERITAGE (c_username) TABLESPACE pg_default; +CREATE INDEX rah_r_creator_fk ON CASE_RETENTION_AUDIT_HERITAGE (r_creator_name) TABLESPACE pg_default; +CREATE INDEX rah_r_modifier_fk ON CASE_RETENTION_AUDIT_HERITAGE (r_modifier) TABLESPACE pg_default; +CREATE INDEX rah_owner_fk ON CASE_RETENTION_AUDIT_HERITAGE (owner_name) TABLESPACE pg_default; -CREATE INDEX rcc_cre_by_fk ON RETENTION_CONFIDENCE_CATEGORY_MAPPER(created_by) TABLESPACE pg_default; -CREATE INDEX rcc_lst_mod_by_fk ON RETENTION_CONFIDENCE_CATEGORY_MAPPER(last_modified_by) TABLESPACE pg_default; +CREATE INDEX rcc_cre_by_fk ON RETENTION_CONFIDENCE_CATEGORY_MAPPER (created_by) TABLESPACE pg_default; +CREATE INDEX rcc_lst_mod_by_fk ON RETENTION_CONFIDENCE_CATEGORY_MAPPER (last_modified_by) TABLESPACE pg_default; -CREATE INDEX cao_rpt_fk ON CASE_OVERFLOW(rpt_id) TABLESPACE pg_default; -CREATE INDEX cao_cas_fk ON CASE_OVERFLOW(cas_id) TABLESPACE pg_default; +CREATE INDEX cao_rpt_fk ON CASE_OVERFLOW (rpt_id) TABLESPACE pg_default; +CREATE INDEX cao_cas_fk ON CASE_OVERFLOW (cas_id) TABLESPACE pg_default; --v2 -CREATE INDEX cth_cn_idx ON COURTHOUSE(UPPER(courthouse_name)) TABLESPACE pg_default; -CREATE INDEX ctr_cn_idx ON COURTROOM(UPPER(courtroom_name)) TABLESPACE pg_default; -CREATE INDEX dfc_dn_idx ON DEFENCE(UPPER(defence_name)) TABLESPACE pg_default; -CREATE INDEX dfd_dn_idx ON DEFENDANT(UPPER(defendant_name)) TABLESPACE pg_default; -CREATE INDEX eve_ei_ic_idx ON EVENT(event_id,is_current) TABLESPACE pg_default; -CREATE INDEX eve_ts_idx ON EVENT(event_ts) TABLESPACE pg_default; -CREATE INDEX hea_hd_idx ON HEARING(hearing_date) TABLESPACE pg_default; -CREATE INDEX jud_jn_idx ON JUDGE(UPPER(judge_name)) TABLESPACE pg_default; -CREATE INDEX med_mf_idx ON MEDIA(media_file) TABLESPACE pg_default; -CREATE INDEX med_st_et_idx ON MEDIA(start_ts,end_ts) TABLESPACE pg_default; -CREATE INDEX med_chronicle_id_idx ON MEDIA(chronicle_id) TABLESPACE pg_default; -CREATE INDEX prn_pn_idx ON PROSECUTOR(UPPER(prosecutor_name)) TABLESPACE pg_default; -CREATE INDEX usr_un_idx ON USER_ACCOUNT(user_full_name) TABLESPACE pg_default; -CREATE INDEX usr_upea_idx ON USER_ACCOUNT(UPPER(user_email_address)) TABLESPACE pg_default; -CREATE INDEX usr_ag_idx ON USER_ACCOUNT(account_guid) TABLESPACE pg_default; +CREATE INDEX cth_cn_idx ON COURTHOUSE (UPPER(courthouse_name)) TABLESPACE pg_default; +CREATE INDEX ctr_cn_idx ON COURTROOM (UPPER(courtroom_name)) TABLESPACE pg_default; +CREATE INDEX dfc_dn_idx ON DEFENCE (UPPER(defence_name)) TABLESPACE pg_default; +CREATE INDEX dfd_dn_idx ON DEFENDANT (UPPER(defendant_name)) TABLESPACE pg_default; +CREATE INDEX eve_ei_ic_idx ON EVENT (event_id, is_current) TABLESPACE pg_default; +CREATE INDEX eve_ts_idx ON EVENT (event_ts) TABLESPACE pg_default; +CREATE INDEX hea_hd_idx ON HEARING (hearing_date) TABLESPACE pg_default; +CREATE INDEX jud_jn_idx ON JUDGE (UPPER(judge_name)) TABLESPACE pg_default; +CREATE INDEX med_mf_idx ON MEDIA (media_file) TABLESPACE pg_default; +CREATE INDEX med_st_et_idx ON MEDIA (start_ts, end_ts) TABLESPACE pg_default; +CREATE INDEX med_chronicle_id_idx ON MEDIA (chronicle_id) TABLESPACE pg_default; +CREATE INDEX prn_pn_idx ON PROSECUTOR (UPPER(prosecutor_name)) TABLESPACE pg_default; +CREATE INDEX usr_un_idx ON USER_ACCOUNT (user_full_name) TABLESPACE pg_default; +CREATE INDEX usr_upea_idx ON USER_ACCOUNT (UPPER(user_email_address)) TABLESPACE pg_default; +CREATE INDEX usr_ag_idx ON USER_ACCOUNT (account_guid) TABLESPACE pg_default; --v6 CREATE UNIQUE INDEX event_handler_event_type_event_event_sub_type_unq ON darts.event_handler USING btree - (event_type COLLATE pg_catalog."default" ASC NULLS LAST, event_sub_type COLLATE pg_catalog."default" ASC NULLS LAST) + (event_type COLLATE pg_catalog."default" ASC NULLS LAST, event_sub_type COLLATE pg_catalog."default" ASC NULLS LAST) TABLESPACE pg_default WHERE active; CREATE UNIQUE INDEX retention_policy_type_type_unq ON darts.retention_policy_type USING btree - (fixed_policy_key COLLATE pg_catalog."default" ASC NULLS LAST) + (fixed_policy_key COLLATE pg_catalog."default" ASC NULLS LAST) TABLESPACE pg_default WHERE policy_end_ts IS NULL; - + CREATE INDEX eod_manifest_file_idx ON darts.external_object_directory USING btree - (manifest_file COLLATE pg_catalog."default" text_pattern_ops ASC NULLS LAST) + (manifest_file COLLATE pg_catalog."default" text_pattern_ops ASC NULLS LAST) TABLESPACE pg_default; - + CREATE INDEX dfd_dn_trgm_idx ON defendant USING gin (defendant_name gin_trgm_ops); CREATE INDEX eve_evt_trgm_idx ON event USING gin (event_text gin_trgm_ops); @@ -319,25 +321,25 @@ CREATE INDEX cas_cn_trgm_idx ON court_case USING gin (case_number gin_trgm_ops); CREATE INDEX ctr_cn_trgm_idx ON courtroom USING gin (courtroom_name gin_trgm_ops); --v14 -CREATE INDEX eve_ei_mi_idx ON event(event_id,message_id); +CREATE INDEX eve_ei_mi_idx ON event (event_id, message_id); --v15 -CREATE INDEX ard_ct_idx ON arm_rpo_execution_detail(created_ts); -CREATE INDEX aut_tn_idx ON automated_task(task_name); -CREATE INDEX mer_rs_idx ON media_request(request_status); -CREATE INDEX not_ns_idx ON notification(notification_status); -CREATE INDEX rcc_cc_idx ON retention_confidence_category_mapper(confidence_category); -CREATE INDEX rpt_fpk_idx ON retention_policy_type(fixed_policy_key); +CREATE INDEX ard_ct_idx ON arm_rpo_execution_detail (created_ts); +CREATE INDEX aut_tn_idx ON automated_task (task_name); +CREATE INDEX mer_rs_idx ON media_request (request_status); +CREATE INDEX not_ns_idx ON notification (notification_status); +CREATE INDEX rcc_cc_idx ON retention_confidence_category_mapper (confidence_category); +CREATE INDEX rpt_fpk_idx ON retention_policy_type (fixed_policy_key); --v16 -CREATE INDEX ccd_ct_idx ON cc_dets(category_type); -CREATE INDEX wcc_ct_idx ON wk_case_correction(category_type); +CREATE INDEX ccd_ct_idx ON cc_dets (category_type); +CREATE INDEX wcc_ct_idx ON wk_case_correction (category_type); --v17 -CREATE INDEX wca_cdt_idx ON wk_case_activity_data(closed_date_type); +CREATE INDEX wca_cdt_idx ON wk_case_activity_data (closed_date_type); --v18 -CREATE INDEX crd_rpt_idx ON cr_dets(rpt_id); +CREATE INDEX crd_rpt_idx ON cr_dets (rpt_id); diff --git a/src/main/resources/db/reference/modernised_darts_schema.sql b/src/main/resources/db/reference/modernised_darts_schema.sql index ce33278bdd7..3d39e13e72a 100644 --- a/src/main/resources/db/reference/modernised_darts_schema.sql +++ b/src/main/resources/db/reference/modernised_darts_schema.sql @@ -1053,6 +1053,7 @@ CREATE TABLE external_object_directory ,created_by INTEGER NOT NULL ,last_modified_ts TIMESTAMP WITH TIME ZONE NOT NULL ,last_modified_by INTEGER NOT NULL +,create_record_processed_ts TIMESTAMP WITH TIME ZONE ) TABLESPACE pg_default; COMMENT ON COLUMN external_object_directory.eod_id diff --git a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmBatchProcessResponseFilesImplTest.java b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmBatchProcessResponseFilesImplTest.java index 65d3364be30..d0227e525d5 100644 --- a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmBatchProcessResponseFilesImplTest.java +++ b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmBatchProcessResponseFilesImplTest.java @@ -138,7 +138,7 @@ void setupData() { deleteArmResponseFilesHelper, featureFlagLogApi )); - + } @BeforeAll @@ -246,7 +246,6 @@ void batchProcessResponseFiles_batchSizeLargerThanMaxContinuationBatchSize_shoul when(armDataManagementApi.listResponseBlobsUsingMarker(PREFIX, 1, null)).thenReturn(continuationTokenBlobs); when(armDataManagementApi.listResponseBlobsUsingMarker(PREFIX, 1, continuationToken)).thenReturn(continuationTokenBlobs2); - List inboundList1 = new ArrayList<>(Collections.singletonList(externalObjectDirectoryArmDropZone)); ExternalObjectDirectoryEntity externalObjectDirectoryEntity2 = mock(ExternalObjectDirectoryEntity.class); List inboundList2 = new ArrayList<>(Collections.singletonList(externalObjectDirectoryEntity2)); @@ -415,7 +414,6 @@ void processBatchResponseFiles_unableToFindResponseFiles_responseProcessingFaile doNothing().when(armBatchProcessResponseFiles).updateExternalObjectDirectoryStatus(any(), any(), any()); - armBatchProcessResponseFiles.processBatchResponseFiles(batchUploadFileFilenameProcessor, armBatchResponses, userAccount); diff --git a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceImplTest.java b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceImplTest.java index b5101178aaf..9d8f73ac8c6 100644 --- a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceImplTest.java +++ b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceImplTest.java @@ -118,7 +118,7 @@ void performCatchup_doesNothing_whenNoEodsToProcess() { armRpoExecutionDetailEntity.setArmRpoStatus(ArmRpoHelper.completedRpoStatus()); when(armRpoService.getLatestArmRpoExecutionDetailEntity()).thenReturn(armRpoExecutionDetailEntity); - when(externalObjectDirectoryRepository.findOldestByInputUploadProcessedTsAndStatusAndLocation(any(), any())) + when(externalObjectDirectoryRepository.findOldestByCreateRecordProcessedTsAndStatusAndLocation(any(), any())) .thenReturn(null); // when @@ -140,8 +140,8 @@ void performCatchup_doesNothing_whenEarliestEodTooRecent() { ExternalObjectDirectoryEntity eod = new ExternalObjectDirectoryEntity(); eod.setCreatedDateTime(OffsetDateTime.now().minusHours(1)); // recent - eod.setInputUploadProcessedTs(OffsetDateTime.now().minusHours(1)); // recent - when(externalObjectDirectoryRepository.findOldestByInputUploadProcessedTsAndStatusAndLocation(any(), any())) + eod.setCreateRecordProcessedTs(OffsetDateTime.now().minusHours(1)); // recent + when(externalObjectDirectoryRepository.findOldestByCreateRecordProcessedTsAndStatusAndLocation(any(), any())) .thenReturn(eod); when(currentTimeHelper.currentOffsetDateTime()).thenReturn(OffsetDateTime.now()); @@ -166,8 +166,8 @@ void performCatchup_runsSuccessfully_whenValid() { ExternalObjectDirectoryEntity eod = new ExternalObjectDirectoryEntity(); OffsetDateTime earliest = OffsetDateTime.now().minusHours(100); eod.setCreatedDateTime(earliest); - eod.setInputUploadProcessedTs(earliest); - when(externalObjectDirectoryRepository.findOldestByInputUploadProcessedTsAndStatusAndLocation(any(), any())) + eod.setCreateRecordProcessedTs(earliest); + when(externalObjectDirectoryRepository.findOldestByCreateRecordProcessedTsAndStatusAndLocation(any(), any())) .thenReturn(eod); when(currentTimeHelper.currentOffsetDateTime()).thenReturn(OffsetDateTime.now()); From 1b68ed34c0b6fa487bfa7cb0d073aa38e094a09b Mon Sep 17 00:00:00 2001 From: karen-hedges <133129444+karen-hedges@users.noreply.github.com> Date: Tue, 16 Jun 2026 14:40:23 +0100 Subject: [PATCH 2/9] DMP-5565 RPO - Use create record process_time in CSV reconcilliation Renamed migration --- ...l => V1_513__add_column_create_record_process_time_to_eod.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/common/{V1_510__add_column_create_record_process_time_to_eod.sql => V1_513__add_column_create_record_process_time_to_eod.sql} (100%) diff --git a/src/main/resources/db/migration/common/V1_510__add_column_create_record_process_time_to_eod.sql b/src/main/resources/db/migration/common/V1_513__add_column_create_record_process_time_to_eod.sql similarity index 100% rename from src/main/resources/db/migration/common/V1_510__add_column_create_record_process_time_to_eod.sql rename to src/main/resources/db/migration/common/V1_513__add_column_create_record_process_time_to_eod.sql From 8d5cb1232b4226d0eff3c955faf1b2841916a955 Mon Sep 17 00:00:00 2001 From: karen-hedges <133129444+karen-hedges@users.noreply.github.com> Date: Thu, 18 Jun 2026 21:41:25 +0100 Subject: [PATCH 3/9] DMP-5565 RPO - Use create record process_time in CSV reconcilliation Renumbered migration --- ...l => V1_514__add_column_create_record_process_time_to_eod.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/common/{V1_513__add_column_create_record_process_time_to_eod.sql => V1_514__add_column_create_record_process_time_to_eod.sql} (100%) diff --git a/src/main/resources/db/migration/common/V1_513__add_column_create_record_process_time_to_eod.sql b/src/main/resources/db/migration/common/V1_514__add_column_create_record_process_time_to_eod.sql similarity index 100% rename from src/main/resources/db/migration/common/V1_513__add_column_create_record_process_time_to_eod.sql rename to src/main/resources/db/migration/common/V1_514__add_column_create_record_process_time_to_eod.sql From e7398d6a9477886e7514308748037af7bcb2d334 Mon Sep 17 00:00:00 2001 From: karen-hedges <133129444+karen-hedges@users.noreply.github.com> Date: Fri, 19 Jun 2026 09:38:30 +0100 Subject: [PATCH 4/9] DMP-5440 ARM - Recreation of response files after deletion Updated the code so that responses that have a different date to what was originally set to be deleted --- .../AbstractArmBatchProcessResponseFiles.java | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java index ddeccb7a724..7762525337d 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java @@ -580,18 +580,28 @@ private void setDataIngestionForUploadFileRecord(ArmResponseUploadFileRecord arm String uploadFileRecordFilenameAndPath) { try { OffsetDateTime uploadNewFileRecordProcessTime = getUploadFileRecordProcessTime(armResponseUploadFileRecord); - setEodDataIngestionTimestamp(externalObjectDirectoryId, uploadNewFileRecordProcessTime); + setEodDataIngestionTimestamp(externalObjectDirectoryId, uploadNewFileRecordProcessTime, uploadFileRecordFilenameAndPath); } catch (Exception e) { log.error("Unable to set EOD data ingestion timestamp for EOD {} - upload file {}", externalObjectDirectoryId, uploadFileRecordFilenameAndPath, e); } } - private void setEodDataIngestionTimestamp(Long externalObjectDirectoryId, OffsetDateTime uploadNewFileRecordProcessTime) { + private void setEodDataIngestionTimestamp(Long externalObjectDirectoryId, OffsetDateTime uploadNewFileRecordProcessTime, + String uploadFileRecordFilenameAndPath) { ExternalObjectDirectoryEntity externalObjectDirectory = getExternalObjectDirectoryEntity(externalObjectDirectoryId); if (nonNull(externalObjectDirectory)) { - externalObjectDirectory.setDataIngestionTs(uploadNewFileRecordProcessTime); - externalObjectDirectoryRepository.save(externalObjectDirectory); + OffsetDateTime uploadFileFileRecordProcessTime = externalObjectDirectory.getDataIngestionTs(); + if (isNull(uploadFileFileRecordProcessTime)) { + externalObjectDirectory.setDataIngestionTs(uploadNewFileRecordProcessTime); + externalObjectDirectoryRepository.save(externalObjectDirectory); + } else { + log.warn("EOD {} already has a data ingestion timestamp set to {} - not updating to {}", + externalObjectDirectoryId, uploadFileFileRecordProcessTime, uploadNewFileRecordProcessTime); + if (!uploadFileFileRecordProcessTime.isEqual(externalObjectDirectory.getDataIngestionTs())) { + deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(uploadFileRecordFilenameAndPath)); + } + } } } @@ -599,18 +609,28 @@ private void setProcessTimeForCreateRecordFileRecord(ArmResponseCreateRecord arm String createRecordFilenameAndPath) { try { OffsetDateTime createRecordProcessTime = getCreateRecordProcessTime(armResponseCreateRecord); - setEodCreateRecordProcessTimestamp(externalObjectDirectoryId, createRecordProcessTime); + setEodCreateRecordProcessTimestamp(externalObjectDirectoryId, createRecordProcessTime, createRecordFilenameAndPath); } catch (Exception e) { log.error("Unable to set EOD create record process timestamp for EOD {} - upload file {}", externalObjectDirectoryId, createRecordFilenameAndPath, e); } } - private void setEodCreateRecordProcessTimestamp(Long externalObjectDirectoryId, OffsetDateTime createRecordProcessTime) { + private void setEodCreateRecordProcessTimestamp(Long externalObjectDirectoryId, OffsetDateTime createRecordProcessTime, + String createRecordFilenameAndPath) { ExternalObjectDirectoryEntity externalObjectDirectory = getExternalObjectDirectoryEntity(externalObjectDirectoryId); if (nonNull(externalObjectDirectory)) { - externalObjectDirectory.setCreateRecordProcessedTs(createRecordProcessTime); - externalObjectDirectoryRepository.save(externalObjectDirectory); + OffsetDateTime createRecordProcessedTs = externalObjectDirectory.getCreateRecordProcessedTs(); + if (isNull(createRecordProcessedTs)) { + externalObjectDirectory.setCreateRecordProcessedTs(createRecordProcessTime); + externalObjectDirectoryRepository.save(externalObjectDirectory); + } else { + log.warn("EOD {} already has a create record processed timestamp set to {} - not updating to {}", + externalObjectDirectoryId, createRecordProcessedTs, createRecordProcessTime); + if (!createRecordProcessedTs.isEqual(externalObjectDirectory.getCreateRecordProcessedTs())) { + deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(createRecordFilenameAndPath)); + } + } } } From 0f0b7cfc7b584cd3fe296f64cb7b2ae626ddf2fa Mon Sep 17 00:00:00 2001 From: karen-hedges <133129444+karen-hedges@users.noreply.github.com> Date: Fri, 19 Jun 2026 09:41:25 +0100 Subject: [PATCH 5/9] DMP-5440 ARM - Recreation of response files after deletion Updated the code so that responses that have a different date to what was originally set to be deleted --- .../arm/service/impl/AbstractArmBatchProcessResponseFiles.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java index 7762525337d..6316f807e89 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java @@ -1019,7 +1019,6 @@ protected void onUploadFileChecksumValidationSuccess(BatchInputUploadFileFilenam UserAccountEntity userAccount) { externalObjectDirectory.setExternalFileId(armResponseUploadFileRecord.getA360FileId()); externalObjectDirectory.setExternalRecordId(armResponseUploadFileRecord.getA360RecordId()); - externalObjectDirectory.setDataIngestionTs(OffsetDateTime.now()); updateExternalObjectDirectoryStatus(externalObjectDirectory, EodHelper.armRpoPendingStatus(), userAccount); } From fb83ce615d881fddebc2b4e8604ed1fdc0a21679 Mon Sep 17 00:00:00 2001 From: karen-hedges <133129444+karen-hedges@users.noreply.github.com> Date: Fri, 19 Jun 2026 12:00:58 +0100 Subject: [PATCH 6/9] DMP-5440 ARM - Recreation of response files after deletion Updated the code so that responses that have a different date to what was originally set to be deleted --- .../AbstractArmBatchProcessResponseFiles.java | 4 +- .../ArmBatchProcessResponseFilesImplTest.java | 115 ++++++++++++++++++ 2 files changed, 117 insertions(+), 2 deletions(-) diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java index 6316f807e89..a658abf44d5 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java @@ -598,7 +598,7 @@ private void setEodDataIngestionTimestamp(Long externalObjectDirectoryId, Offset } else { log.warn("EOD {} already has a data ingestion timestamp set to {} - not updating to {}", externalObjectDirectoryId, uploadFileFileRecordProcessTime, uploadNewFileRecordProcessTime); - if (!uploadFileFileRecordProcessTime.isEqual(externalObjectDirectory.getDataIngestionTs())) { + if (!uploadFileFileRecordProcessTime.isEqual(uploadNewFileRecordProcessTime)) { deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(uploadFileRecordFilenameAndPath)); } } @@ -627,7 +627,7 @@ private void setEodCreateRecordProcessTimestamp(Long externalObjectDirectoryId, } else { log.warn("EOD {} already has a create record processed timestamp set to {} - not updating to {}", externalObjectDirectoryId, createRecordProcessedTs, createRecordProcessTime); - if (!createRecordProcessedTs.isEqual(externalObjectDirectory.getCreateRecordProcessedTs())) { + if (!createRecordProcessedTs.isEqual(createRecordProcessTime)) { deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(createRecordFilenameAndPath)); } } diff --git a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmBatchProcessResponseFilesImplTest.java b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmBatchProcessResponseFilesImplTest.java index d0227e525d5..1871d551483 100644 --- a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmBatchProcessResponseFilesImplTest.java +++ b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmBatchProcessResponseFilesImplTest.java @@ -12,6 +12,7 @@ import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; import uk.gov.hmcts.darts.arm.api.ArmDataManagementApi; import uk.gov.hmcts.darts.arm.config.ArmDataManagementConfiguration; import uk.gov.hmcts.darts.arm.config.UnstructuredToArmProcessorConfiguration; @@ -385,6 +386,120 @@ void updateExternalObjectDirectoryStatus_ArmMissingResponse() { verify(externalObjectDirectoryRepository).saveAndFlush(externalObjectDirectoryEntity); } + @Test + void setEodCreateRecordProcessTimestamp_shouldDeleteResponseBlob_WhenTimestampAlreadySetWithDifferentTime() { + // given + Long externalObjectDirectoryId = 123L; + String createRecordFilenameAndPath = "dropzone/DARTS/response/create-record.rsp"; + OffsetDateTime existingCreateRecordProcessedTs = OffsetDateTime.parse("2023-06-10T14:08:28.316382+00:00"); + OffsetDateTime newCreateRecordProcessTime = existingCreateRecordProcessedTs.plusMinutes(1); + + ExternalObjectDirectoryEntity externalObjectDirectoryEntity = new ExternalObjectDirectoryEntity(); + externalObjectDirectoryEntity.setId(externalObjectDirectoryId); + externalObjectDirectoryEntity.setCreateRecordProcessedTs(existingCreateRecordProcessedTs); + + doReturn(externalObjectDirectoryEntity).when(armBatchProcessResponseFiles).getExternalObjectDirectoryEntity(externalObjectDirectoryId); + + // when + ReflectionTestUtils.invokeMethod( + armBatchProcessResponseFiles, + "setEodCreateRecordProcessTimestamp", + externalObjectDirectoryId, + newCreateRecordProcessTime, + createRecordFilenameAndPath + ); + + // then + assertEquals(existingCreateRecordProcessedTs, externalObjectDirectoryEntity.getCreateRecordProcessedTs()); + verify(externalObjectDirectoryRepository, never()).save(externalObjectDirectoryEntity); + verify(deleteArmResponseFilesHelper).deleteResponseBlobs(List.of(createRecordFilenameAndPath)); + } + + @Test + void setEodCreateRecordProcessTimestamp_shouldNotDeleteResponseBlob_WhenTimestampAlreadySetWithSameTime() { + // given + Long externalObjectDirectoryId = 123L; + String createRecordFilenameAndPath = "dropzone/DARTS/response/create-record.rsp"; + OffsetDateTime existingCreateRecordProcessedTs = OffsetDateTime.parse("2023-06-10T14:08:28.316382+00:00"); + + ExternalObjectDirectoryEntity externalObjectDirectoryEntity = new ExternalObjectDirectoryEntity(); + externalObjectDirectoryEntity.setId(externalObjectDirectoryId); + externalObjectDirectoryEntity.setCreateRecordProcessedTs(existingCreateRecordProcessedTs); + + doReturn(externalObjectDirectoryEntity).when(armBatchProcessResponseFiles).getExternalObjectDirectoryEntity(externalObjectDirectoryId); + + // when + ReflectionTestUtils.invokeMethod( + armBatchProcessResponseFiles, + "setEodCreateRecordProcessTimestamp", + externalObjectDirectoryId, + existingCreateRecordProcessedTs, + createRecordFilenameAndPath + ); + + // then + assertEquals(existingCreateRecordProcessedTs, externalObjectDirectoryEntity.getCreateRecordProcessedTs()); + verify(externalObjectDirectoryRepository, never()).save(externalObjectDirectoryEntity); + verify(deleteArmResponseFilesHelper, never()).deleteResponseBlobs(anyList()); + } + + @Test + void setEodDataIngestionTimestamp_shouldDeleteResponseBlob_WhenTimestampAlreadySetWithDifferentTime() { + // given + Long externalObjectDirectoryId = 123L; + String uploadFileRecordFilenameAndPath = "dropzone/DARTS/response/upload-file.rsp"; + OffsetDateTime existingDataIngestionTs = OffsetDateTime.parse("2023-06-10T14:08:28.316382+00:00"); + OffsetDateTime newUploadFileRecordProcessTime = existingDataIngestionTs.plusMinutes(1); + + ExternalObjectDirectoryEntity externalObjectDirectoryEntity = new ExternalObjectDirectoryEntity(); + externalObjectDirectoryEntity.setId(externalObjectDirectoryId); + externalObjectDirectoryEntity.setDataIngestionTs(existingDataIngestionTs); + + doReturn(externalObjectDirectoryEntity).when(armBatchProcessResponseFiles).getExternalObjectDirectoryEntity(externalObjectDirectoryId); + + // when + ReflectionTestUtils.invokeMethod( + armBatchProcessResponseFiles, + "setEodDataIngestionTimestamp", + externalObjectDirectoryId, + newUploadFileRecordProcessTime, + uploadFileRecordFilenameAndPath + ); + + // then + assertEquals(existingDataIngestionTs, externalObjectDirectoryEntity.getDataIngestionTs()); + verify(externalObjectDirectoryRepository, never()).save(externalObjectDirectoryEntity); + verify(deleteArmResponseFilesHelper).deleteResponseBlobs(List.of(uploadFileRecordFilenameAndPath)); + } + + @Test + void setEodDataIngestionTimestamp_shouldNotDeleteResponseBlob_WhenTimestampAlreadySetWithSameTime() { + // given + Long externalObjectDirectoryId = 123L; + String uploadFileRecordFilenameAndPath = "dropzone/DARTS/response/upload-file.rsp"; + OffsetDateTime existingDataIngestionTs = OffsetDateTime.parse("2023-06-10T14:08:28.316382+00:00"); + + ExternalObjectDirectoryEntity externalObjectDirectoryEntity = new ExternalObjectDirectoryEntity(); + externalObjectDirectoryEntity.setId(externalObjectDirectoryId); + externalObjectDirectoryEntity.setDataIngestionTs(existingDataIngestionTs); + + doReturn(externalObjectDirectoryEntity).when(armBatchProcessResponseFiles).getExternalObjectDirectoryEntity(externalObjectDirectoryId); + + // when + ReflectionTestUtils.invokeMethod( + armBatchProcessResponseFiles, + "setEodDataIngestionTimestamp", + externalObjectDirectoryId, + existingDataIngestionTs, + uploadFileRecordFilenameAndPath + ); + + // then + assertEquals(existingDataIngestionTs, externalObjectDirectoryEntity.getDataIngestionTs()); + verify(externalObjectDirectoryRepository, never()).save(externalObjectDirectoryEntity); + verify(deleteArmResponseFilesHelper, never()).deleteResponseBlobs(anyList()); + } + @Test @SuppressWarnings("unchecked") void processBatchResponseFiles_unableToFindResponseFiles_responseProcessingFailedStatus_outsideAllowedTime() { From c5dc032cc3ceec289ebca983575b5807c990f279 Mon Sep 17 00:00:00 2001 From: karen-hedges <133129444+karen-hedges@users.noreply.github.com> Date: Fri, 19 Jun 2026 16:21:33 +0100 Subject: [PATCH 7/9] DMP-5440 ARM - Recreation of response files after deletion Updated code to remove eod from list of responses to process when duplicate response is found --- .../ArmDuplicateResponseException.java | 15 +++++++++ .../AbstractArmBatchProcessResponseFiles.java | 31 +++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 src/main/java/uk/gov/hmcts/darts/arm/exception/ArmDuplicateResponseException.java diff --git a/src/main/java/uk/gov/hmcts/darts/arm/exception/ArmDuplicateResponseException.java b/src/main/java/uk/gov/hmcts/darts/arm/exception/ArmDuplicateResponseException.java new file mode 100644 index 00000000000..000f27b9d0f --- /dev/null +++ b/src/main/java/uk/gov/hmcts/darts/arm/exception/ArmDuplicateResponseException.java @@ -0,0 +1,15 @@ +package uk.gov.hmcts.darts.arm.exception; + +import uk.gov.hmcts.darts.common.exception.DartsException; + +public class ArmDuplicateResponseException extends DartsException { + + public ArmDuplicateResponseException(String message) { + super(message); + } + + public ArmDuplicateResponseException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java index a658abf44d5..0027c2dbd9e 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java @@ -7,6 +7,7 @@ import org.apache.commons.lang3.StringUtils; import uk.gov.hmcts.darts.arm.api.ArmDataManagementApi; import uk.gov.hmcts.darts.arm.config.ArmDataManagementConfiguration; +import uk.gov.hmcts.darts.arm.exception.ArmDuplicateResponseException; import uk.gov.hmcts.darts.arm.model.ResponseFilenames; import uk.gov.hmcts.darts.arm.model.blobs.ArmBatchResponses; import uk.gov.hmcts.darts.arm.model.blobs.ArmResponseBatchData; @@ -549,6 +550,7 @@ private void readCreateRecordFile(BinaryData createRecordBinary, CreateRecordFil ArmBatchResponses armBatchResponses) { String createRecordFilenameAndPath = createRecordFilenameProcessor.getCreateRecordFilenameAndPath(); if (nonNull(createRecordBinary)) { + Long externalObjectDirectoryId = null; try { log.info("Length of ARM CR response file {} is {}", createRecordFilenameAndPath, createRecordBinary.getLength()); log.info("Contents of ARM CR response file: {} - {}", createRecordFilenameAndPath, createRecordBinary); @@ -556,7 +558,7 @@ private void readCreateRecordFile(BinaryData createRecordBinary, CreateRecordFil UploadNewFileRecord uploadNewFileRecord = readInputJson(armResponseCreateRecord.getInput()); if (nonNull(uploadNewFileRecord)) { if (StringUtils.isNotEmpty(uploadNewFileRecord.getRelationId())) { - Long externalObjectDirectoryId = Long.valueOf(uploadNewFileRecord.getRelationId()); + externalObjectDirectoryId = Long.valueOf(uploadNewFileRecord.getRelationId()); armBatchResponses.addResponseBatchData(externalObjectDirectoryId, armResponseCreateRecord, createRecordFilenameProcessor); setProcessTimeForCreateRecordFileRecord(armResponseCreateRecord, externalObjectDirectoryId, createRecordFilenameAndPath); } else { @@ -566,7 +568,12 @@ private void readCreateRecordFile(BinaryData createRecordBinary, CreateRecordFil } else { log.warn("Failed to obtain EOD id (relation id) from create record file {}", createRecordFilenameAndPath); } - + } catch (ArmDuplicateResponseException e) { + log.warn("Duplicate response for create record file {}. Deleted the create record file from blob storage.", + createRecordFilenameAndPath, e); + if (nonNull(externalObjectDirectoryId)) { + armBatchResponses.getArmBatchResponseMap().remove(externalObjectDirectoryId); + } } catch (Exception e) { log.error("Unable to process arm response create record file {}", createRecordFilenameAndPath, e); } @@ -581,6 +588,10 @@ private void setDataIngestionForUploadFileRecord(ArmResponseUploadFileRecord arm try { OffsetDateTime uploadNewFileRecordProcessTime = getUploadFileRecordProcessTime(armResponseUploadFileRecord); setEodDataIngestionTimestamp(externalObjectDirectoryId, uploadNewFileRecordProcessTime, uploadFileRecordFilenameAndPath); + } catch (ArmDuplicateResponseException e) { + log.warn("Duplicate response for EOD {} - upload file {}. Deleted the upload file from blob storage.", + externalObjectDirectoryId, uploadFileRecordFilenameAndPath, e); + throw e; } catch (Exception e) { log.error("Unable to set EOD data ingestion timestamp for EOD {} - upload file {}", externalObjectDirectoryId, uploadFileRecordFilenameAndPath, e); @@ -600,6 +611,8 @@ private void setEodDataIngestionTimestamp(Long externalObjectDirectoryId, Offset externalObjectDirectoryId, uploadFileFileRecordProcessTime, uploadNewFileRecordProcessTime); if (!uploadFileFileRecordProcessTime.isEqual(uploadNewFileRecordProcessTime)) { deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(uploadFileRecordFilenameAndPath)); + throw new ArmDuplicateResponseException( + "Duplicate response for EOD " + externalObjectDirectoryId + " - upload file " + uploadFileRecordFilenameAndPath); } } } @@ -610,6 +623,10 @@ private void setProcessTimeForCreateRecordFileRecord(ArmResponseCreateRecord arm try { OffsetDateTime createRecordProcessTime = getCreateRecordProcessTime(armResponseCreateRecord); setEodCreateRecordProcessTimestamp(externalObjectDirectoryId, createRecordProcessTime, createRecordFilenameAndPath); + } catch (ArmDuplicateResponseException e) { + log.warn("Duplicate response for EOD {} - create record file {}. Deleted the create record file from blob storage.", + externalObjectDirectoryId, createRecordFilenameAndPath, e); + throw e; } catch (Exception e) { log.error("Unable to set EOD create record process timestamp for EOD {} - upload file {}", externalObjectDirectoryId, createRecordFilenameAndPath, e); @@ -629,6 +646,8 @@ private void setEodCreateRecordProcessTimestamp(Long externalObjectDirectoryId, externalObjectDirectoryId, createRecordProcessedTs, createRecordProcessTime); if (!createRecordProcessedTs.isEqual(createRecordProcessTime)) { deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(createRecordFilenameAndPath)); + throw new ArmDuplicateResponseException( + "Duplicate response for EOD " + externalObjectDirectoryId + " - create record file " + createRecordFilenameAndPath); } } } @@ -661,6 +680,7 @@ private void readUploadFile(BinaryData uploadFileBinary, UploadFileFilenameProce String uploadFileFilenameAndPath = uploadFileFilenameProcessor.getUploadFileFilenameAndPath(); if (nonNull(uploadFileBinary)) { + Long externalObjectDirectoryId = null; try { log.info("Length of ARM UF response file {} is {}", uploadFileFilenameAndPath, uploadFileBinary.getLength()); log.info("Contents of ARM UF response file: {} - {}", uploadFileFilenameAndPath, uploadFileBinary); @@ -668,7 +688,7 @@ private void readUploadFile(BinaryData uploadFileBinary, UploadFileFilenameProce UploadNewFileRecord uploadNewFileRecord = readInputJson(armResponseUploadFileRecord.getInput()); if (nonNull(uploadNewFileRecord)) { if (StringUtils.isNotEmpty(uploadNewFileRecord.getRelationId())) { - Long externalObjectDirectoryId = Long.valueOf(uploadNewFileRecord.getRelationId()); + externalObjectDirectoryId = Long.valueOf(uploadNewFileRecord.getRelationId()); armBatchResponses.addResponseBatchData(externalObjectDirectoryId, armResponseUploadFileRecord, uploadFileFilenameProcessor); setDataIngestionForUploadFileRecord(armResponseUploadFileRecord, externalObjectDirectoryId, uploadFileFilenameAndPath); @@ -680,6 +700,11 @@ private void readUploadFile(BinaryData uploadFileBinary, UploadFileFilenameProce } else { log.warn("Failed to obtain EOD id (relation id) from upload file {}", uploadFileFilenameAndPath); } + } catch (ArmDuplicateResponseException e) { + log.warn("Duplicate response for upload file {}. Deleted the upload file from blob storage.", uploadFileFilenameAndPath, e); + if (nonNull(externalObjectDirectoryId)) { + armBatchResponses.getArmBatchResponseMap().remove(externalObjectDirectoryId); + } } catch (Exception e) { log.error("Unable to process arm response upload file {}", uploadFileFilenameAndPath, e); } From 91b3634c5f42de60ae7a60f92abb27d2d2cef730 Mon Sep 17 00:00:00 2001 From: karen-hedges <133129444+karen-hedges@users.noreply.github.com> Date: Fri, 19 Jun 2026 17:00:11 +0100 Subject: [PATCH 8/9] DMP-5440 ARM - Recreation of response files after deletion Updated code to remove eod from list of responses to process when duplicate response is found --- .../ArmBatchProcessResponseFilesImplTest.java | 102 +++++++++++++++++- 1 file changed, 98 insertions(+), 4 deletions(-) diff --git a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmBatchProcessResponseFilesImplTest.java b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmBatchProcessResponseFilesImplTest.java index 1871d551483..116eac9e98c 100644 --- a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmBatchProcessResponseFilesImplTest.java +++ b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/ArmBatchProcessResponseFilesImplTest.java @@ -16,6 +16,7 @@ import uk.gov.hmcts.darts.arm.api.ArmDataManagementApi; import uk.gov.hmcts.darts.arm.config.ArmDataManagementConfiguration; import uk.gov.hmcts.darts.arm.config.UnstructuredToArmProcessorConfiguration; +import uk.gov.hmcts.darts.arm.exception.ArmDuplicateResponseException; import uk.gov.hmcts.darts.arm.model.blobs.ArmBatchResponses; import uk.gov.hmcts.darts.arm.model.blobs.ArmResponseBatchData; import uk.gov.hmcts.darts.arm.model.blobs.ContinuationTokenBlobs; @@ -24,6 +25,8 @@ import uk.gov.hmcts.darts.arm.service.DeleteArmResponseFilesHelper; import uk.gov.hmcts.darts.arm.service.ExternalObjectDirectoryService; import uk.gov.hmcts.darts.arm.util.files.BatchInputUploadFileFilenameProcessor; +import uk.gov.hmcts.darts.arm.util.files.CreateRecordFilenameProcessor; +import uk.gov.hmcts.darts.arm.util.files.UploadFileFilenameProcessor; import uk.gov.hmcts.darts.authorisation.component.UserIdentity; import uk.gov.hmcts.darts.common.config.ObjectMapperConfig; import uk.gov.hmcts.darts.common.entity.ExternalObjectDirectoryEntity; @@ -87,6 +90,9 @@ class ArmBatchProcessResponseFilesImplTest { private static final String DATETIMEKEY = ""; private static final String INPUT_UPLOAD_RESPONSE_DATETIME = "2021-08-01T10:08:28.316382+00:00"; private static final String UPLOAD_RESPONSE_TIMESTAMP_FORAMT = "yyyy-MM-dd'T'HH:mm:ss[.SSSSSS][XXXX][XXXXX]"; + private static final Long EXTERNAL_OBJECT_DIRECTORY_ID = 123L; + private static final String RESPONSE_PROCESS_TIME = "2023-06-10T14:09:28.316382+00:00"; + private static final String UPLOAD_NEW_FILE_INPUT = "{\"operation\":\"upload_new_file\",\"relation_id\":\"123\"}"; @Mock private ExternalObjectDirectoryRepository externalObjectDirectoryRepository; @@ -401,13 +407,13 @@ void setEodCreateRecordProcessTimestamp_shouldDeleteResponseBlob_WhenTimestampAl doReturn(externalObjectDirectoryEntity).when(armBatchProcessResponseFiles).getExternalObjectDirectoryEntity(externalObjectDirectoryId); // when - ReflectionTestUtils.invokeMethod( + assertThrows(ArmDuplicateResponseException.class, () -> ReflectionTestUtils.invokeMethod( armBatchProcessResponseFiles, "setEodCreateRecordProcessTimestamp", externalObjectDirectoryId, newCreateRecordProcessTime, createRecordFilenameAndPath - ); + )); // then assertEquals(existingCreateRecordProcessedTs, externalObjectDirectoryEntity.getCreateRecordProcessedTs()); @@ -458,13 +464,13 @@ void setEodDataIngestionTimestamp_shouldDeleteResponseBlob_WhenTimestampAlreadyS doReturn(externalObjectDirectoryEntity).when(armBatchProcessResponseFiles).getExternalObjectDirectoryEntity(externalObjectDirectoryId); // when - ReflectionTestUtils.invokeMethod( + assertThrows(ArmDuplicateResponseException.class, () -> ReflectionTestUtils.invokeMethod( armBatchProcessResponseFiles, "setEodDataIngestionTimestamp", externalObjectDirectoryId, newUploadFileRecordProcessTime, uploadFileRecordFilenameAndPath - ); + )); // then assertEquals(existingDataIngestionTs, externalObjectDirectoryEntity.getDataIngestionTs()); @@ -500,6 +506,68 @@ void setEodDataIngestionTimestamp_shouldNotDeleteResponseBlob_WhenTimestampAlrea verify(deleteArmResponseFilesHelper, never()).deleteResponseBlobs(anyList()); } + @Test + void readCreateRecordFile_shouldCatchDuplicateResponseExceptionAndRemoveEodFromBatchResponses() { + // given + String createRecordFilenameAndPath = "dropzone/DARTS/response/6a374f19a9ce7dc9cc480ea8d4eca0fb_12374f19a9ce7dc9cc480ea8d4eca0fb_1_cr.rsp"; + OffsetDateTime existingCreateRecordProcessedTs = OffsetDateTime.parse("2023-06-10T14:08:28.316382+00:00"); + + ExternalObjectDirectoryEntity externalObjectDirectoryEntity = new ExternalObjectDirectoryEntity(); + externalObjectDirectoryEntity.setId(EXTERNAL_OBJECT_DIRECTORY_ID); + externalObjectDirectoryEntity.setCreateRecordProcessedTs(existingCreateRecordProcessedTs); + + ArmBatchResponses armBatchResponses = new ArmBatchResponses(); + CreateRecordFilenameProcessor createRecordFilenameProcessor = new CreateRecordFilenameProcessor(createRecordFilenameAndPath); + + doReturn(externalObjectDirectoryEntity).when(armBatchProcessResponseFiles).getExternalObjectDirectoryEntity(EXTERNAL_OBJECT_DIRECTORY_ID); + + // when + ReflectionTestUtils.invokeMethod( + armBatchProcessResponseFiles, + "readCreateRecordFile", + BinaryData.fromString(createRecordResponse()), + createRecordFilenameProcessor, + armBatchResponses + ); + + // then + assertEquals(0, armBatchResponses.getArmBatchResponseMap().size()); + assertEquals(existingCreateRecordProcessedTs, externalObjectDirectoryEntity.getCreateRecordProcessedTs()); + verify(externalObjectDirectoryRepository, never()).save(externalObjectDirectoryEntity); + verify(deleteArmResponseFilesHelper).deleteResponseBlobs(List.of(createRecordFilenameAndPath)); + } + + @Test + void readUploadFile_shouldCatchDuplicateResponseExceptionAndRemoveEodFromBatchResponses() { + // given + String uploadFileFilenameAndPath = "dropzone/DARTS/response/6a374f19a9ce7dc9cc480ea8d4eca0fb_45674f19a9ce7dc9cc480ea8d4eca0fb_1_uf.rsp"; + OffsetDateTime existingDataIngestionTs = OffsetDateTime.parse("2023-06-10T14:08:28.316382+00:00"); + + ExternalObjectDirectoryEntity externalObjectDirectoryEntity = new ExternalObjectDirectoryEntity(); + externalObjectDirectoryEntity.setId(EXTERNAL_OBJECT_DIRECTORY_ID); + externalObjectDirectoryEntity.setDataIngestionTs(existingDataIngestionTs); + + ArmBatchResponses armBatchResponses = new ArmBatchResponses(); + UploadFileFilenameProcessor uploadFileFilenameProcessor = new UploadFileFilenameProcessor(uploadFileFilenameAndPath); + + doReturn(externalObjectDirectoryEntity).when(armBatchProcessResponseFiles).getExternalObjectDirectoryEntity(EXTERNAL_OBJECT_DIRECTORY_ID); + + // when + ReflectionTestUtils.invokeMethod( + armBatchProcessResponseFiles, + "readUploadFile", + BinaryData.fromString(uploadFileResponse()), + uploadFileFilenameProcessor, + armBatchResponses + ); + + // then + assertEquals(0, armBatchResponses.getArmBatchResponseMap().size()); + assertEquals(existingDataIngestionTs, externalObjectDirectoryEntity.getDataIngestionTs()); + verify(externalObjectDirectoryRepository, never()).save(externalObjectDirectoryEntity); + verify(deleteArmResponseFilesHelper).deleteResponseBlobs(List.of(uploadFileFilenameAndPath)); + } + @Test @SuppressWarnings("unchecked") void processBatchResponseFiles_unableToFindResponseFiles_responseProcessingFailedStatus_outsideAllowedTime() { @@ -760,6 +828,32 @@ void getInputUploadFileTimestamp_shouldReturnParsedOffsetDateTime_WithoutMillise ); } + private String createRecordResponse() { + return """ + { + "relation_id": "123", + "a360_record_id": "a360-record-id", + "process_time": "%s", + "status": 1, + "input": "%s" + } + """.formatted(RESPONSE_PROCESS_TIME, UPLOAD_NEW_FILE_INPUT.replace("\"", "\\\"")); + } + + private String uploadFileResponse() { + return """ + { + "relation_id": "123", + "a360_record_id": "a360-record-id", + "process_time": "%s", + "status": 1, + "input": "%s", + "a360_file_id": "a360-file-id", + "s_md5": "checksum" + } + """.formatted(RESPONSE_PROCESS_TIME, UPLOAD_NEW_FILE_INPUT.replace("\"", "\\\"")); + } + class ArmBatchProcessResponseFilesImplProtectedMethodSupport extends ArmBatchProcessResponseFilesImpl { public ArmBatchProcessResponseFilesImplProtectedMethodSupport(ExternalObjectDirectoryRepository externalObjectDirectoryRepository, From 7d0c4fb33c828e031d88000cb30be3b846f4a95f Mon Sep 17 00:00:00 2001 From: karen-hedges <133129444+karen-hedges@users.noreply.github.com> Date: Fri, 19 Jun 2026 17:40:33 +0100 Subject: [PATCH 9/9] DMP-5440 ARM - Recreation of response files after deletion Updated code to remove eod from list of responses to process when duplicate response is found --- .../arm/service/impl/AbstractArmBatchProcessResponseFiles.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java index 0027c2dbd9e..33c0a5b1671 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java @@ -66,7 +66,8 @@ @SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.CouplingBetweenObjects", - "PMD.TooManyMethods" + "PMD.TooManyMethods", + "PMD.CognitiveComplexity" }) public abstract class AbstractArmBatchProcessResponseFiles implements ArmResponseFilesProcessor {