Dmp 5440 arm recreation response files after deletion

src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceIntTest.java

@@ -100,7 +100,7 @@ void performCatchup_withSuccessfulArmApiCalls_doesNotThrowException() {
             .status(dartsDatabase.getObjectRecordStatusEntity(ARM_RPO_PENDING))
             .externalLocationType(dartsDatabase.getExternalLocationTypeEntity(ARM))
             .externalLocation(UUID.randomUUID().toString()).build();
-        armEod1.setInputUploadProcessedTs(lastModifiedDateTime);
+        armEod1.setCreateRecordProcessedTs(lastModifiedDateTime);
         armEod1.setVerificationAttempts(1);
         dartsPersistence.save(armEod1);
 
@@ -154,7 +154,7 @@ void performCatchup_doesNothing_WhenOldestPendingEodIsTooNew() {
             .status(dartsDatabase.getObjectRecordStatusEntity(ARM_RPO_PENDING))
             .externalLocationType(dartsDatabase.getExternalLocationTypeEntity(ARM))
             .externalLocation(UUID.randomUUID().toString()).build();
-        armEod1.setInputUploadProcessedTs(lastModifiedDateTime);
+        armEod1.setCreateRecordProcessedTs(lastModifiedDateTime);
         armEod1.setVerificationAttempts(1);
         dartsPersistence.save(armEod1);
 

src/integrationTest/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepositoryTest.java

@@ -1258,29 +1258,29 @@ private void generateDataWithMediaForInbound(int hoursBeforeCurrentTime)
     }
 
     @Test
-    void findOldestByInputUploadProcessedTsAndStatusAndLocation() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException {
+    void findOldestByCreateRecordProcessedTsAndStatusAndLocation() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException {
         // Given
         OffsetDateTime pastCurrentDateTime1 = OffsetDateTime.now().minusHours(200);
         OffsetDateTime pastCurrentDateTime2 = OffsetDateTime.now().minusHours(2);
 
         List<ExternalObjectDirectoryEntity> matchingEods = externalObjectDirectoryStub.generateWithStatusAndMediaLocation(
             ExternalLocationTypeEnum.ARM, ARM_RPO_PENDING, 10, Optional.of(pastCurrentDateTime1));
         matchingEods.forEach(eod -> {
-            eod.setInputUploadProcessedTs(pastCurrentDateTime1);
+            eod.setCreateRecordProcessedTs(pastCurrentDateTime1);
         });
         dartsPersistence.saveAll(matchingEods);
         assertEquals(10, matchingEods.size());
 
         List<ExternalObjectDirectoryEntity> nonMatchingEods = externalObjectDirectoryStub.generateWithStatusAndMediaLocation(
             ExternalLocationTypeEnum.ARM, ARM_RPO_PENDING, 4, Optional.of(pastCurrentDateTime2));
         nonMatchingEods.forEach(eod -> {
-            eod.setInputUploadProcessedTs(pastCurrentDateTime2);
+            eod.setCreateRecordProcessedTs(pastCurrentDateTime2);
         });
         dartsPersistence.saveAll(nonMatchingEods);
         assertEquals(4, nonMatchingEods.size());
 
         // When
-        ExternalObjectDirectoryEntity result = externalObjectDirectoryRepository.findOldestByInputUploadProcessedTsAndStatusAndLocation(
+        ExternalObjectDirectoryEntity result = externalObjectDirectoryRepository.findOldestByCreateRecordProcessedTsAndStatusAndLocation(
             EodHelper.armRpoPendingStatus(), EodHelper.armLocation());
 
         // Then

src/main/java/uk/gov/hmcts/darts/arm/exception/ArmDuplicateResponseException.java

@@ -0,0 +1,15 @@
+package uk.gov.hmcts.darts.arm.exception;
+
+import uk.gov.hmcts.darts.common.exception.DartsException;
+
+public class ArmDuplicateResponseException extends DartsException {
+
+    public ArmDuplicateResponseException(String message) {
+        super(message);
+    }
+
+    public ArmDuplicateResponseException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+}

src/main/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoBacklogCatchupServiceImpl.java

@@ -36,8 +36,8 @@ public class ArmRpoBacklogCatchupServiceImpl implements ArmRpoBacklogCatchupServ
     @Override
     public void performCatchup(Integer batchSize, Integer maxHoursEndingPoint, Integer totalCatchupHours, Duration threadSleepDuration) {
         var armRpoExecutionDetailEntity = armRpoService.getLatestArmRpoExecutionDetailEntity();
-        var earliestEodInRpo = externalObjectDirectoryRepository.findOldestByInputUploadProcessedTsAndStatusAndLocation(EodHelper.armRpoPendingStatus(),
-                                                                                                                        EodHelper.armLocation());
+        var earliestEodInRpo = externalObjectDirectoryRepository.findOldestByCreateRecordProcessedTsAndStatusAndLocation(EodHelper.armRpoPendingStatus(),
+                                                                                                                         EodHelper.armLocation());
 
         // Only perform backlog catchup if the last execution is in REMOVE_PRODUCTION state or FAILED status
         if (!validateTaskCanBeRun(maxHoursEndingPoint, totalCatchupHours, armRpoExecutionDetailEntity, earliestEodInRpo)) {
@@ -56,12 +56,15 @@ public void performCatchup(Integer batchSize, Integer maxHoursEndingPoint, Integ
             return;
         }
 
-        OffsetDateTime inputUploadProcessedTs = earliestEodInRpo.getInputUploadProcessedTs();
-        // subtract 10 minutes to account for any potential delay in EOD being picked up for RPO search after the input upload processed timestamp
-        OffsetDateTime adjustedOldestEodDateTime = inputUploadProcessedTs.minus(PRE_AMBLE_MINUTES, ChronoUnit.MINUTES);
+        // Use createRecordProcessedTs if it's been set else use inputUploadProcessedTs
+        OffsetDateTime processedTs = isNull(earliestEodInRpo.getCreateRecordProcessedTs())
+            ? earliestEodInRpo.getInputUploadProcessedTs() : earliestEodInRpo.getCreateRecordProcessedTs();
+        // subtract 10 minutes to account for any potential delay in EOD being picked up for RPO search after
+        // the create record processed timestamp
+        OffsetDateTime adjustedOldestEodDateTime = processedTs.minus(PRE_AMBLE_MINUTES, ChronoUnit.MINUTES);
         int hoursEnd = (int) calculateHoursFromStartToNow(adjustedOldestEodDateTime.toString());
 
-        Integer originalStaartHour = armAutomatedTaskEntity.getRpoCsvStartHour();
+        Integer originalStartHour = armAutomatedTaskEntity.getRpoCsvStartHour();
         Integer originalEndHour = armAutomatedTaskEntity.getRpoCsvEndHour();
         try {
             armAutomatedTaskEntity.setRpoCsvStartHour(hoursEnd - totalCatchupHours);
@@ -70,7 +73,7 @@ public void performCatchup(Integer batchSize, Integer maxHoursEndingPoint, Integ
             triggerArmRpoSearchService.triggerArmRpoSearch(threadSleepDuration);
         } catch (Exception e) {
             // reset the start and end hour to the original value if there are any failures
-            armAutomatedTaskEntity.setRpoCsvStartHour(originalStaartHour);
+            armAutomatedTaskEntity.setRpoCsvStartHour(originalStartHour);
             armAutomatedTaskEntity.setRpoCsvEndHour(originalEndHour);
             armAutomatedTaskRepository.save(armAutomatedTaskEntity);
         }
@@ -98,7 +101,7 @@ private boolean validateTaskCanBeRun(Integer maxHoursEndingPoint, Integer totalC
         OffsetDateTime currentTime = currentTimeHelper.currentOffsetDateTime();
         int amountToSubtract = maxHoursEndingPoint + totalCatchupHours;
         OffsetDateTime lastRunTaskDateTime = currentTime.minus(amountToSubtract, ChronoUnit.HOURS);
-        if (earliestEodInRpo.getInputUploadProcessedTs().isAfter(lastRunTaskDateTime)) {
+        if (isNull(earliestEodInRpo.getCreateRecordProcessedTs()) || earliestEodInRpo.getCreateRecordProcessedTs().isAfter(lastRunTaskDateTime)) {
             log.info("Earliest EODs found in ARM RPO pending status is not suitable for backlog catchup.");
             return false;
         }

src/main/java/uk/gov/hmcts/darts/arm/service/impl/DetsToArmBatchProcessResponseFilesImpl.java

@@ -177,8 +177,8 @@ private void updateOsrIngestStatusToFailure(ObjectStateRecordEntity osr,
     }
 
     @Override
-    protected void markEodAsResponseProcessingFailed(ExternalObjectDirectoryEntity externalObjectDirectory, UserAccountEntity userAccount) {
-        super.markEodAsResponseProcessingFailed(externalObjectDirectory, userAccount);
+    protected void markEodAsMissingResponseFailure(ExternalObjectDirectoryEntity externalObjectDirectory, UserAccountEntity userAccount) {
+        super.markEodAsMissingResponseFailure(externalObjectDirectory, userAccount);
 
         ObjectStateRecordEntity objectStateRecordEntity = externalObjectDirectory.getObjectStateRecordEntity();
 

src/main/java/uk/gov/hmcts/darts/common/entity/ExternalObjectDirectoryEntity.java

@@ -109,6 +109,9 @@ public class ExternalObjectDirectoryEntity extends CreatedModifiedBaseEntity imp
     @Column(name = "is_dets", nullable = false)
     private boolean isDets;
 
+    @Column(name = "create_record_processed_ts")
+    private OffsetDateTime createRecordProcessedTs;
+
     @Override
     public int getStatusId() {
         return getStatus().getId();

src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java

@@ -731,9 +731,9 @@ List<Long> findIdsByStatusAndLastModifiedBetweenAndLocationAndLimit(@Param("stat
         SELECT eod FROM ExternalObjectDirectoryEntity eod
         WHERE eod.status = :status
         AND eod.externalLocationType = :locationType
-        ORDER BY eod.inputUploadProcessedTs ASC
+        ORDER BY eod.createRecordProcessedTs ASC
         LIMIT 1
         """)
-    ExternalObjectDirectoryEntity findOldestByInputUploadProcessedTsAndStatusAndLocation(ObjectRecordStatusEntity status,
-                                                                                         ExternalLocationTypeEntity locationType);
+    ExternalObjectDirectoryEntity findOldestByCreateRecordProcessedTsAndStatusAndLocation(ObjectRecordStatusEntity status,
+                                                                                          ExternalLocationTypeEntity locationType);
 }

src/main/resources/db/migration/common/V1_514__add_column_create_record_process_time_to_eod.sql

@@ -0,0 +1,9 @@
+ALTER TABLE external_object_directory
+    ADD COLUMN IF NOT EXISTS create_record_processed_ts TIMESTAMP WITH TIME ZONE;
+
+UPDATE external_object_directory
+SET create_record_processed_ts = input_upload_processed_ts
+WHERE create_record_processed_ts is null
+  and input_upload_processed_ts is not null;
+
+CREATE INDEX IF NOT EXISTS eod_cr_proc_ts_idx ON external_object_directory (create_record_processed_ts);