Skip to content

Dmp 5565 rpo use create record process time #3347

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
karen-hedges wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Dmp 5565 rpo use create record process time #3347

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
61fa34f
DMP-5565 RPO - Use create record process_time in CSV reconcilliation
karen-hedges Jun 4, 2026
8d8b3e7
DMP-5565 RPO - Use create record process_time in CSV reconcilliation
karen-hedges Jun 16, 2026
226cd12
DMP-5565 RPO - Use create record process_time in CSV reconcilliation
karen-hedges Jun 18, 2026
9087bd3
User create record processed in RPO reconciliation
stephenokeefe Jun 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions src/integrationTest/java/uk/gov/hmcts/darts/arm/service/ArmRpoServiceIntTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ void reconcileArmRpoCsvData_Success() {
ARM,
UUID.randomUUID().toString()
);
externalObjectDirectoryEntity1.setInputUploadProcessedTs(OffsetDateTime.now().minusHours(26));
externalObjectDirectoryEntity1.setCreateRecordProcessedTs(OffsetDateTime.now().minusHours(26));
dartsDatabase.save(externalObjectDirectoryEntity1);

var externalObjectDirectoryEntity2 = dartsDatabase.getExternalObjectDirectoryStub().createExternalObjectDirectory(
Expand All @@ -152,7 +152,7 @@ void reconcileArmRpoCsvData_Success() {
ARM,
UUID.randomUUID().toString()
);
externalObjectDirectoryEntity2.setInputUploadProcessedTs(OffsetDateTime.now().minusHours(6));
externalObjectDirectoryEntity2.setCreateRecordProcessedTs(OffsetDateTime.now().minusHours(6));
dartsDatabase.save(externalObjectDirectoryEntity2);

ObjectStateRecordEntity objectStateRecordEntity = dartsDatabase.getObjectStateRecordRepository()
Expand All @@ -164,7 +164,7 @@ void reconcileArmRpoCsvData_Success() {
ARM,
UUID.randomUUID().toString()
);
externalObjectDirectoryEntity3.setInputUploadProcessedTs(OffsetDateTime.now().minusHours(30));
externalObjectDirectoryEntity3.setCreateRecordProcessedTs(OffsetDateTime.now().minusHours(30));
externalObjectDirectoryEntity3.setOsrUuid(objectStateRecordEntity.getUuid());
dartsDatabase.getExternalObjectDirectoryRepository().saveAndFlush(externalObjectDirectoryEntity3);
objectStateRecordEntity.setArmEodId(externalObjectDirectoryEntity3.getId());
Expand All @@ -176,7 +176,7 @@ void reconcileArmRpoCsvData_Success() {
ARM,
UUID.randomUUID().toString()
);
externalObjectDirectoryEntity4.setInputUploadProcessedTs(OffsetDateTime.now().minusHours(28));
externalObjectDirectoryEntity4.setCreateRecordProcessedTs(OffsetDateTime.now().minusHours(28));
dartsDatabase.save(externalObjectDirectoryEntity4);

var externalObjectDirectoryEntity5 = dartsDatabase.getExternalObjectDirectoryStub().createExternalObjectDirectory(
Expand All @@ -194,7 +194,7 @@ void reconcileArmRpoCsvData_Success() {
ARM,
UUID.randomUUID().toString()
);
externalObjectDirectoryEntity6.setInputUploadProcessedTs(OffsetDateTime.now().minusHours(30));
externalObjectDirectoryEntity6.setCreateRecordProcessedTs(OffsetDateTime.now().minusHours(30));
dartsDatabase.save(externalObjectDirectoryEntity6);

armRpoExecutionDetailEntity.setCreatedDateTime(OffsetDateTime.now());
Expand Down Expand Up @@ -243,7 +243,7 @@ void reconcileArmRpoCsvData_Success() {
assertEquals(1, foundMediaList6.size());
ExternalObjectDirectoryEntity foundMedia6 = foundMediaList6.getFirst();
assertEquals(ARM_REPLAY.getId(), foundMedia6.getStatus().getId());
assertNotNull(foundMedia6.getInputUploadProcessedTs());
assertNotNull(foundMedia6.getCreateRecordProcessedTs());

}

Expand All @@ -259,7 +259,7 @@ void reconcileArmRpoCsvData_FailsDets() {
ARM,
UUID.randomUUID().toString()
);
externalObjectDirectory.setInputUploadProcessedTs(OffsetDateTime.now().minusHours(30));
externalObjectDirectory.setCreateRecordProcessedTs(OffsetDateTime.now().minusHours(30));
externalObjectDirectory.setOsrUuid(objectStateRecordEntity.getUuid());
dartsDatabase.getExternalObjectDirectoryRepository().saveAndFlush(externalObjectDirectory);
objectStateRecordEntity.setArmEodId(123L);
Expand Down Expand Up @@ -297,7 +297,7 @@ void reconcileArmRpoCsvData_DoesNothing_WhenIngestionDateOutsideOfRange() {
ARM,
UUID.randomUUID().toString()
);
externalObjectDirectoryEntity1.setInputUploadProcessedTs(OffsetDateTime.now().minusHours(1));
externalObjectDirectoryEntity1.setCreateRecordProcessedTs(OffsetDateTime.now().minusHours(1));
dartsDatabase.save(externalObjectDirectoryEntity1);

var externalObjectDirectoryEntity2 = dartsDatabase.getExternalObjectDirectoryStub().createExternalObjectDirectory(
Expand All @@ -306,7 +306,7 @@ void reconcileArmRpoCsvData_DoesNothing_WhenIngestionDateOutsideOfRange() {
ARM,
UUID.randomUUID().toString()
);
externalObjectDirectoryEntity2.setInputUploadProcessedTs(OffsetDateTime.now().minusHours(100));
externalObjectDirectoryEntity2.setCreateRecordProcessedTs(OffsetDateTime.now().minusHours(100));
dartsDatabase.save(externalObjectDirectoryEntity2);

armRpoExecutionDetailEntity.setCreatedDateTime(OffsetDateTime.now());
Expand Down Expand Up @@ -341,7 +341,7 @@ void reconcileArmRpoCsvData_UpdatesStatus_WhenCsvDataMatches() {
ARM,
UUID.randomUUID().toString()
);
externalObjectDirectoryEntity.setInputUploadProcessedTs(OffsetDateTime.now().minusHours(26));
externalObjectDirectoryEntity.setCreateRecordProcessedTs(OffsetDateTime.now().minusHours(26));
dartsDatabase.save(externalObjectDirectoryEntity);

armRpoExecutionDetailEntity.setCreatedDateTime(OffsetDateTime.now());
Expand Down Expand Up @@ -369,7 +369,7 @@ void reconcileArmRpoCsvData_UpdatesEodStatusToReplay_WhenCsvDataContainsEodsThat
ARM,
UUID.randomUUID().toString()
);
externalObjectDirectoryEntity.setInputUploadProcessedTs(OffsetDateTime.now().minusHours(26));
externalObjectDirectoryEntity.setCreateRecordProcessedTs(OffsetDateTime.now().minusHours(26));
dartsDatabase.save(externalObjectDirectoryEntity);

armRpoExecutionDetailEntity.setCreatedDateTime(OffsetDateTime.now());
Expand All @@ -386,7 +386,7 @@ void reconcileArmRpoCsvData_UpdatesEodStatusToReplay_WhenCsvDataContainsEodsThat
assertEquals(1, foundMediaList.size());
ExternalObjectDirectoryEntity foundMedia = foundMediaList.getFirst();
assertEquals(ARM_REPLAY.getId(), foundMedia.getStatus().getId());
assertNotNull(foundMedia.getInputUploadProcessedTs());
assertNotNull(foundMedia.getCreateRecordProcessedTs());
}

@Test
Expand All @@ -398,7 +398,7 @@ void reconcileArmRpoCsvData_DoesNothing_WhenNoEodsInRange() {
ARM,
UUID.randomUUID().toString()
);
externalObjectDirectoryEntity.setInputUploadProcessedTs(OffsetDateTime.now().minusHours(1));
externalObjectDirectoryEntity.setCreateRecordProcessedTs(OffsetDateTime.now().minusHours(1));
dartsDatabase.save(externalObjectDirectoryEntity);

armRpoExecutionDetailEntity.setCreatedDateTime(OffsetDateTime.now());
Expand Down
4 changes: 2 additions & 2 deletions ...tionTest/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceIntTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ void performCatchup_withSuccessfulArmApiCalls_doesNotThrowException() {
.status(dartsDatabase.getObjectRecordStatusEntity(ARM_RPO_PENDING))
.externalLocationType(dartsDatabase.getExternalLocationTypeEntity(ARM))
.externalLocation(UUID.randomUUID().toString()).build();
armEod1.setInputUploadProcessedTs(lastModifiedDateTime);
armEod1.setCreateRecordProcessedTs(lastModifiedDateTime);
armEod1.setVerificationAttempts(1);
dartsPersistence.save(armEod1);

Expand Down Expand Up @@ -154,7 +154,7 @@ void performCatchup_doesNothing_WhenOldestPendingEodIsTooNew() {
.status(dartsDatabase.getObjectRecordStatusEntity(ARM_RPO_PENDING))
.externalLocationType(dartsDatabase.getExternalLocationTypeEntity(ARM))
.externalLocation(UUID.randomUUID().toString()).build();
armEod1.setInputUploadProcessedTs(lastModifiedDateTime);
armEod1.setCreateRecordProcessedTs(lastModifiedDateTime);
armEod1.setVerificationAttempts(1);
dartsPersistence.save(armEod1);

Expand Down
18 changes: 9 additions & 9 deletions ...Test/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepositoryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -431,23 +431,23 @@ void findByStatusAndDataIngestionTsWithPaging_ShouldReturnAsExpected() throws Ex
}

@Test
void findByStatusAndInputUploadProcessedTsWithPaging_ReturnsResults() throws Exception {
void findByStatusAndCreateRecordProcessedTsWithPaging_ReturnsResults() throws Exception {
// Given
OffsetDateTime pastCurrentDateTime1 = OffsetDateTime.now().minusHours(2);
OffsetDateTime pastCurrentDateTime2 = OffsetDateTime.now().minusHours(20);

List<ExternalObjectDirectoryEntity> matchingEods = externalObjectDirectoryStub.generateWithStatusAndMediaLocation(
ExternalLocationTypeEnum.ARM, ARM_RPO_PENDING, 11, Optional.of(pastCurrentDateTime1));
matchingEods.forEach(eod -> {
eod.setInputUploadProcessedTs(pastCurrentDateTime1);
eod.setCreateRecordProcessedTs(pastCurrentDateTime1);
});
dartsPersistence.saveAll(matchingEods);
assertEquals(11, matchingEods.size());

List<ExternalObjectDirectoryEntity> nonMatchingEods = externalObjectDirectoryStub.generateWithStatusAndMediaLocation(
ExternalLocationTypeEnum.ARM, ARM_RPO_PENDING, 4, Optional.of(pastCurrentDateTime2));
nonMatchingEods.forEach(eod -> {
eod.setInputUploadProcessedTs(pastCurrentDateTime2);
eod.setCreateRecordProcessedTs(pastCurrentDateTime2);
});
dartsPersistence.saveAll(nonMatchingEods);
assertEquals(4, nonMatchingEods.size());
Expand All @@ -459,7 +459,7 @@ void findByStatusAndInputUploadProcessedTsWithPaging_ReturnsResults() throws Exc
ObjectRecordStatusEntity status = EodHelper.armRpoPendingStatus();

// When
Page<ExternalObjectDirectoryEntity> result = externalObjectDirectoryRepository.findByStatusAndInputUploadProcessedTsWithPaging(
Page<ExternalObjectDirectoryEntity> result = externalObjectDirectoryRepository.findByStatusAndCreateRecordProcessedTsWithPaging(
status, startDateTime, endDateTime, pageable
);

Expand All @@ -469,7 +469,7 @@ void findByStatusAndInputUploadProcessedTsWithPaging_ReturnsResults() throws Exc
assertThat(result.getTotalElements()).isEqualTo(11);
result.getContent().forEach(entity -> {
assertThat(entity.getStatus().getId()).isEqualTo(status.getId());
assertThat(entity.getInputUploadProcessedTs()).isBetween(startDateTime, endDateTime);
assertThat(entity.getCreateRecordProcessedTs()).isBetween(startDateTime, endDateTime);
});
}

Expand Down Expand Up @@ -1258,29 +1258,29 @@ private void generateDataWithMediaForInbound(int hoursBeforeCurrentTime)
}

@Test
void findOldestByInputUploadProcessedTsAndStatusAndLocation() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException {
void findOldestByCreateRecordProcessedTsAndStatusAndLocation() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException {
// Given
OffsetDateTime pastCurrentDateTime1 = OffsetDateTime.now().minusHours(200);
OffsetDateTime pastCurrentDateTime2 = OffsetDateTime.now().minusHours(2);

List<ExternalObjectDirectoryEntity> matchingEods = externalObjectDirectoryStub.generateWithStatusAndMediaLocation(
ExternalLocationTypeEnum.ARM, ARM_RPO_PENDING, 10, Optional.of(pastCurrentDateTime1));
matchingEods.forEach(eod -> {
eod.setInputUploadProcessedTs(pastCurrentDateTime1);
eod.setCreateRecordProcessedTs(pastCurrentDateTime1);
});
dartsPersistence.saveAll(matchingEods);
assertEquals(10, matchingEods.size());

List<ExternalObjectDirectoryEntity> nonMatchingEods = externalObjectDirectoryStub.generateWithStatusAndMediaLocation(
ExternalLocationTypeEnum.ARM, ARM_RPO_PENDING, 4, Optional.of(pastCurrentDateTime2));
nonMatchingEods.forEach(eod -> {
eod.setInputUploadProcessedTs(pastCurrentDateTime2);
eod.setCreateRecordProcessedTs(pastCurrentDateTime2);
});
dartsPersistence.saveAll(nonMatchingEods);
assertEquals(4, nonMatchingEods.size());

// When
ExternalObjectDirectoryEntity result = externalObjectDirectoryRepository.findOldestByInputUploadProcessedTsAndStatusAndLocation(
ExternalObjectDirectoryEntity result = externalObjectDirectoryRepository.findOldestByCreateRecordProcessedTsAndStatusAndLocation(
EodHelper.armRpoPendingStatus(), EodHelper.armLocation());

// Then
Expand Down
61 changes: 53 additions & 8 deletions src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,16 @@ public OffsetDateTime getUploadFileRecordProcessTime(ArmResponseUploadFileRecord
}
}

public OffsetDateTime getCreateRecordProcessTime(ArmResponseCreateRecord armResponseCreateRecord) {
try {
return OffsetDateTime.parse(armResponseCreateRecord.getProcessTime(), dateTimeFormatter);
} catch (Exception e) {
log.warn("Unable to parse timestamp {} from ARM create record (CR) record {}", armResponseCreateRecord.getProcessTime(),
armResponseCreateRecord.getA360RecordId(), e);
throw new IllegalArgumentException(e);
}
}

private ArmResponseInputUploadFileRecord getResponseInputUploadFileRecordOrDelete(BatchInputUploadFileFilenameProcessor batchUploadFileFilenameProcessor,
String inputUploadFileRecordStr,
List<ExternalObjectDirectoryEntity> externalObjectDirectoryEntities,
Expand Down Expand Up @@ -303,14 +313,16 @@ private void processResponseFileByHashcode(BatchInputUploadFileFilenameProcessor
List<ExternalObjectDirectoryEntity> externalObjectDirectoryEntities = externalObjectDirectoryRepository
.findAllByStatusAndManifestFile(EodHelper.armProcessingResponseFilesStatus(), manifestName);

OffsetDateTime minInputUploadProcessedTime = timeHelper.currentOffsetDateTime().minus(
OffsetDateTime minProcessedTime = timeHelper.currentOffsetDateTime().minus(
armDataManagementConfiguration.getArmMissingResponseDuration());

externalObjectDirectoryEntities.forEach(
externalObjectDirectoryEntity -> {
if (externalObjectDirectoryEntity.getInputUploadProcessedTs() != null
&& externalObjectDirectoryEntity.getInputUploadProcessedTs().isBefore(minInputUploadProcessedTime)) {
markEodAsResponseProcessingFailed(externalObjectDirectoryEntity, userAccount);
// Use the latest date we have a process time for
OffsetDateTime processedTime = getLatestProcessedTs(externalObjectDirectoryEntity);
if (nonNull(processedTime)
&& processedTime.isBefore(minProcessedTime)) {
markEodAsMissingResponseFailure(externalObjectDirectoryEntity, userAccount);
} else {
updateExternalObjectDirectoryStatus(externalObjectDirectoryEntity, EodHelper.armDropZoneStatus(), userAccount);
}
Expand All @@ -322,6 +334,16 @@ private void processResponseFileByHashcode(BatchInputUploadFileFilenameProcessor
}
}

private OffsetDateTime getLatestProcessedTs(ExternalObjectDirectoryEntity externalObjectDirectoryEntity) {
if (nonNull(externalObjectDirectoryEntity.getCreateRecordProcessedTs())) {
return externalObjectDirectoryEntity.getCreateRecordProcessedTs();
} else if (nonNull(externalObjectDirectoryEntity.getInputUploadProcessedTs())) {
return externalObjectDirectoryEntity.getInputUploadProcessedTs();
} else {
return externalObjectDirectoryEntity.getDataIngestionTs();
}
}

protected void processBatchResponseFiles(BatchInputUploadFileFilenameProcessor batchUploadFileFilenameProcessor,
ArmBatchResponses armBatchResponses,
UserAccountEntity userAccount) {
Expand Down Expand Up @@ -372,9 +394,12 @@ private void missingResponsesFiles(UserAccountEntity userAccount, ArmResponseBat
OffsetDateTime minIngestionTime = timeHelper.currentOffsetDateTime().minus(
armDataManagementConfiguration.getArmMissingResponseDuration());

if (externalObjectDirectory.getInputUploadProcessedTs() != null
&& externalObjectDirectory.getInputUploadProcessedTs().isBefore(minIngestionTime)) {
markEodAsResponseProcessingFailed(externalObjectDirectory, userAccount);
// Use the latest date we have a process time for
OffsetDateTime processedTime = getLatestProcessedTs(externalObjectDirectory);

if (nonNull(processedTime)
&& processedTime.isBefore(minIngestionTime)) {
markEodAsMissingResponseFailure(externalObjectDirectory, userAccount);
} else {
updateExternalObjectDirectoryStatus(externalObjectDirectory, EodHelper.armDropZoneStatus(), userAccount);
}
Expand All @@ -383,7 +408,7 @@ private void missingResponsesFiles(UserAccountEntity userAccount, ArmResponseBat
}
}

protected void markEodAsResponseProcessingFailed(ExternalObjectDirectoryEntity externalObjectDirectory, UserAccountEntity userAccount) {
protected void markEodAsMissingResponseFailure(ExternalObjectDirectoryEntity externalObjectDirectory, UserAccountEntity userAccount) {
updateExternalObjectDirectoryStatus(externalObjectDirectory, EodHelper.armMissingResponseStatus(), userAccount);
}

Expand Down Expand Up @@ -533,6 +558,7 @@ private void readCreateRecordFile(BinaryData createRecordBinary, CreateRecordFil
if (StringUtils.isNotEmpty(uploadNewFileRecord.getRelationId())) {
Long externalObjectDirectoryId = Long.valueOf(uploadNewFileRecord.getRelationId());
armBatchResponses.addResponseBatchData(externalObjectDirectoryId, armResponseCreateRecord, createRecordFilenameProcessor);
setProcessTimeForCreateRecordFileRecord(armResponseCreateRecord, externalObjectDirectoryId, createRecordFilenameAndPath);
} else {
log.warn("Unable to get EOD id (relation id) from uploadNewFileRecord {} create record {}",
armResponseCreateRecord.getInput(), createRecordFilenameAndPath);
Expand Down Expand Up @@ -569,6 +595,25 @@ private void setEodDataIngestionTimestamp(Long externalObjectDirectoryId, Offset
}
}

private void setProcessTimeForCreateRecordFileRecord(ArmResponseCreateRecord armResponseCreateRecord, Long externalObjectDirectoryId,
String createRecordFilenameAndPath) {
try {
OffsetDateTime createRecordProcessTime = getCreateRecordProcessTime(armResponseCreateRecord);
setEodCreateRecordProcessTimestamp(externalObjectDirectoryId, createRecordProcessTime);
} catch (Exception e) {
log.error("Unable to set EOD create record process timestamp for EOD {} - upload file {}",
externalObjectDirectoryId, createRecordFilenameAndPath, e);
}
}

private void setEodCreateRecordProcessTimestamp(Long externalObjectDirectoryId, OffsetDateTime createRecordProcessTime) {
ExternalObjectDirectoryEntity externalObjectDirectory = getExternalObjectDirectoryEntity(externalObjectDirectoryId);
if (nonNull(externalObjectDirectory)) {
externalObjectDirectory.setCreateRecordProcessedTs(createRecordProcessTime);
externalObjectDirectoryRepository.save(externalObjectDirectory);
}
}

private ArmResponseCreateRecord getResponseCreateRecordOrDelete(String createRecordFilenameAndPath,
BinaryData createRecordBinary) throws IOException {
try {
Expand Down
Loading
Loading