DMP-5415 Interruption of deleting CR and UF files

build.gradle

@@ -458,6 +458,7 @@ dependencies {
 
   implementation platform('com.azure:azure-sdk-bom:1.3.7')
   implementation 'com.azure:azure-storage-blob'
+  implementation 'com.azure:azure-storage-blob-batch'
 
   implementation group: 'io.rest-assured', name: 'rest-assured'
   implementation group: 'org.flywaydb', name: 'flyway-core', version: '11.20.3'

src/functionalTest/java/uk/gov/hmcts/darts/arm/ArmServiceFunctionalTest.java

@@ -172,6 +172,31 @@ void listBlobsUsingMarkerTestContinuationToken() {
 
     }
 
+    @Test
+    void saveAndDeleteMultipleBlobsUsingBatching() {
+        log.info("------------------  saveAndDeleteMultipleBlobs test");
+
+        byte[] testStringInBytes = TEST_BINARY_STRING.getBytes(StandardCharsets.UTF_8);
+        BinaryData data = BinaryData.fromBytes(testStringInBytes);
+        String filename1 = String.format("%s_functional_test", UUID.randomUUID());
+        String actualResult1 = armService.saveBlobData(armContainerName, filename1, data);
+        armSubmissionBlobsToBeDeleted.add(actualResult1);
+        assertNotNull(actualResult1);
+        log.info("Blob filename1: {}", actualResult1);
+
+        String filename2 = String.format("%s_functional_test2", UUID.randomUUID());
+        String actualResult2 = armService.saveBlobData(armContainerName, filename2, data);
+        armSubmissionBlobsToBeDeleted.add(actualResult2);
+        assertNotNull(actualResult2);
+        log.info("Blob filename2: {}", actualResult2);
+
+        armService.deleteMultipleBlobsUsingBatching(armContainerName, List.of(
+            actualResult1,
+            actualResult2
+        ));
+        cleanupArmBlobData();
+    }
+
     private void uploadBatchedSubmissionBlobs(BinaryData data) {
         for (int counter = 0; counter < 11; counter++) {
             String filename = String.format("functional_test_%s", UUID.randomUUID());

src/functionalTest/java/uk/gov/hmcts/darts/datamanagement/DataManagementServiceFunctionalTest.java

@@ -12,7 +12,6 @@
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.ActiveProfiles;
 import uk.gov.hmcts.darts.FunctionalTest;
-import uk.gov.hmcts.darts.arm.config.ArmDataManagementConfiguration;
 import uk.gov.hmcts.darts.common.datamanagement.component.impl.DownloadResponseMetaData;
 import uk.gov.hmcts.darts.common.datamanagement.enums.DatastoreContainerType;
 import uk.gov.hmcts.darts.common.exception.AzureDeleteBlobException;
@@ -43,14 +42,12 @@ class DataManagementServiceFunctionalTest extends FunctionalTest {
     private static final String TEST_BLOB_ID = "b0f23c62-8dd3-4e4e-ae6a-321ff6eb61d8";
 
     @Value("${darts.storage.blob.container-name.unstructured}")
-    String unstructuredStorageContainerName;
+    private String unstructuredStorageContainerName;
 
     @Autowired
-    DataManagementService dataManagementService;
+    private DataManagementService dataManagementService;
     @Autowired
-    DataManagementConfiguration dataManagementConfiguration;
-    @Autowired
-    ArmDataManagementConfiguration armDataManagementConfiguration;
+    private DataManagementConfiguration dataManagementConfiguration;
 
     @Test
     void saveBinaryDataToBlobStorage() {

src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/DeleteArmResponseFilesHelperIntTest.java

@@ -6,15 +6,13 @@
 import org.springframework.test.context.bean.override.mockito.MockitoBean;
 import uk.gov.hmcts.darts.arm.api.ArmDataManagementApi;
 import uk.gov.hmcts.darts.arm.model.blobs.ArmResponseBatchData;
-import uk.gov.hmcts.darts.arm.service.ExternalObjectDirectoryService;
 import uk.gov.hmcts.darts.arm.util.files.BatchInputUploadFileFilenameProcessor;
 import uk.gov.hmcts.darts.arm.util.files.CreateRecordFilenameProcessor;
 import uk.gov.hmcts.darts.arm.util.files.UploadFileFilenameProcessor;
 import uk.gov.hmcts.darts.authorisation.component.UserIdentity;
 import uk.gov.hmcts.darts.common.entity.ExternalObjectDirectoryEntity;
 import uk.gov.hmcts.darts.common.entity.MediaEntity;
 import uk.gov.hmcts.darts.common.entity.UserAccountEntity;
-import uk.gov.hmcts.darts.common.repository.ExternalObjectDirectoryRepository;
 import uk.gov.hmcts.darts.testutils.PostgresIntegrationBase;
 
 import java.util.List;
@@ -36,18 +34,12 @@ class DeleteArmResponseFilesHelperIntTest extends PostgresIntegrationBase {
     private static final String MANIFEST_PREFIX = "DARTS_6a374f19a9ce7dc9cc480ea8d4eca0fb";
     private static final String RESPONSE_FILE_PREFIX = "04e6bc3b-952a-79b6-8362-13259aae1895";
 
-    @Autowired
-    private ExternalObjectDirectoryRepository externalObjectDirectoryRepository;
-
     @MockitoBean
     private ArmDataManagementApi armDataManagementApi;
 
     @MockitoBean
     private UserIdentity userIdentity;
 
-    @Autowired
-    private ExternalObjectDirectoryService externalObjectDirectoryService;
-
     @Autowired
     private DeleteArmResponseFilesHelperImpl deleteArmResponseFilesHelper;
 
@@ -66,7 +58,7 @@ void setUp() {
     }
 
     @Test
-    void deleteResponseBlobsByManifestName_shouldDeleteBlobsWhenAllResponsesAreCompletedAndCleaned() {
+    void deleteResponseBlobsByManifestName_shouldDeleteBlobsIndividuallyWhenAllResponsesAreCompletedAndCleaned() {
         // given
         String manifestName = MANIFEST_PREFIX + ".a360";
         eodRpoPending.setManifestFile(manifestName);
@@ -92,33 +84,36 @@ void deleteResponseBlobsByManifestName_shouldDeleteBlobsWhenAllResponsesAreCompl
     void deleteDanglingResponses_shouldDeleteDanglingResponses() {
         // given
         BatchInputUploadFileFilenameProcessor batchInputUploadFileFilenameProcessor = new BatchInputUploadFileFilenameProcessor(DARTS_INPUT_UPLOAD_FILE);
-        String responseFile = "dropzone/DARTS/response/" + RESPONSE_FILE_PREFIX + "_ABC_1_rsp";
-        when(armDataManagementApi.listResponseBlobs(any())).thenReturn(List.of(responseFile));
+        String otherResponseFile = "dropzone/DARTS/response/" + RESPONSE_FILE_PREFIX + "_ABC_1_rsp";
+        String crResponseFile = "dropzone/DARTS/response/" + RESPONSE_FILE_PREFIX + "_b17b9015-e6ad-77c5-8d1e-13259aae1896_0_cr.rsp";
+        String ilResponseFile = "dropzone/DARTS/response/" + RESPONSE_FILE_PREFIX + "_c17b9015-e6ad-77c5-8d1e-13259aae1896_1_il.rsp";
+
+        when(armDataManagementApi.listResponseBlobs(any())).thenReturn(List.of(otherResponseFile, crResponseFile, ilResponseFile));
+        when(armDataManagementApi.deleteMultipleBlobsIndividually(any())).thenReturn(true);
         when(armDataManagementApi.deleteBlobData(anyString())).thenReturn(true);
 
         // when
         deleteArmResponseFilesHelper.deleteDanglingResponses(batchInputUploadFileFilenameProcessor);
 
         // then
-        verify(armDataManagementApi).deleteBlobData(responseFile);
+        verify(armDataManagementApi).deleteMultipleBlobsIndividually(List.of(otherResponseFile, crResponseFile, ilResponseFile));
         verify(armDataManagementApi).deleteBlobData(DARTS_INPUT_UPLOAD_FILE);
         verify(armDataManagementApi).listResponseBlobs(batchInputUploadFileFilenameProcessor.getHashcode());
         verifyNoMoreInteractions(armDataManagementApi);
     }
 
     @Test
-    void deleteResponseBlobs_shouldDeleteAllResponseBlobs() {
+    void deleteResponseBlobsIndividually_shouldDeleteAllResponseBlobIndividually() {
         // given
-        List<String> responseBlobs = List.of("blob1", "blob2");
+        String responseBlob = "blob1";
         when(armDataManagementApi.deleteBlobData(anyString())).thenReturn(true);
 
         // when
-        List<Boolean> result = deleteArmResponseFilesHelper.deleteResponseBlobs(responseBlobs);
+        Boolean result = deleteArmResponseFilesHelper.deleteResponseBlobIndividually(responseBlob);
 
         // then
-        assertTrue(result.stream().allMatch(Boolean::booleanValue));
+        assertTrue(result);
         verify(armDataManagementApi).deleteBlobData("blob1");
-        verify(armDataManagementApi).deleteBlobData("blob2");
         verifyNoMoreInteractions(armDataManagementApi);
     }
 
@@ -134,14 +129,13 @@ void deleteResponseBlobs_shouldUpdateEodWhenResponsesAreDeleted() {
             .uploadFileFilenameProcessor(new UploadFileFilenameProcessor(uploadFileFilename1))
             .build();
 
-        when(armDataManagementApi.deleteBlobData(anyString())).thenReturn(true);
+        when(armDataManagementApi.deleteMultipleBlobsIndividually(any())).thenReturn(true);
 
         // when
         deleteArmResponseFilesHelper.deleteResponseBlobs(batchData);
 
         // then
-        verify(armDataManagementApi).deleteBlobData(createRecordFilename1);
-        verify(armDataManagementApi).deleteBlobData(uploadFileFilename1);
+        verify(armDataManagementApi).deleteMultipleBlobsIndividually(List.of(createRecordFilename1, uploadFileFilename1));
         verifyNoMoreInteractions(armDataManagementApi);
     }
 

src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/ArmServiceStubImpl.java

@@ -76,6 +76,18 @@ public boolean deleteBlobData(String containerName, String blobPathAndName) {
         return true;
     }
 
+    @Override
+    public boolean deleteMultipleBlobsUsingBatching(String containerName, List<String> blobPathAndName) {
+        logStubUsageWarning();
+        return true;
+    }
+
+    @Override
+    public boolean deleteMultipleBlobsIndividually(String containerName, List<String> blobPathAndName) {
+        logStubUsageWarning();
+        return true;
+    }
+
     private void logStubUsageWarning() {
         log.warn("### This implementation is intended only for integration tests. If you see this log message elsewhere"
                      + " you should ask questions! ###");

src/main/java/uk/gov/hmcts/darts/arm/api/ArmDataManagementApi.java

@@ -9,6 +9,7 @@
 import java.time.OffsetDateTime;
 import java.util.List;
 
+@SuppressWarnings({"PMD.TooManyMethods"})
 public interface ArmDataManagementApi extends BlobContainerDownloadable {
 
     String saveBlobDataToArm(String filename, BinaryData binaryData);
@@ -27,6 +28,10 @@ public interface ArmDataManagementApi extends BlobContainerDownloadable {
 
     boolean deleteBlobData(String blobPathAndName);
 
+    boolean deleteMultipleBlobsUsingBatching(List<String> blobPathAndNames);
+
+    boolean deleteMultipleBlobsIndividually(List<String> blobPathAndNames);
+
     UpdateMetadataResponse updateMetadata(String externalRecordId,
                                           OffsetDateTime eventTimestamp,
                                           RetentionConfidenceScoreEnum retConfScore,

src/main/java/uk/gov/hmcts/darts/arm/api/impl/ArmDataManagementApiImpl.java

@@ -26,6 +26,7 @@
 
 @Service
 @RequiredArgsConstructor
+@SuppressWarnings({"PMD.TooManyMethods"})
 public class ArmDataManagementApiImpl implements ArmDataManagementApi {
 
     private final ArmService armService;
@@ -88,6 +89,16 @@ public boolean deleteBlobData(String blobPathAndName) {
         return armService.deleteBlobData(armDataManagementConfiguration.getContainerName(), blobPathAndName);
     }
 
+    @Override
+    public boolean deleteMultipleBlobsUsingBatching(List<String> blobPathAndNames) {
+        return armService.deleteMultipleBlobsUsingBatching(armDataManagementConfiguration.getContainerName(), blobPathAndNames);
+    }
+
+    @Override
+    public boolean deleteMultipleBlobsIndividually(List<String> blobPathAndNames) {
+        return armService.deleteMultipleBlobsIndividually(armDataManagementConfiguration.getContainerName(), blobPathAndNames);
+    }
+
     @Override
     public UpdateMetadataResponse updateMetadata(String externalRecordId,
                                                  OffsetDateTime eventTimestamp,

src/main/java/uk/gov/hmcts/darts/arm/service/ArmService.java

@@ -5,6 +5,7 @@
 
 import java.util.List;
 
+@SuppressWarnings({"PMD.TooManyMethods"})
 public interface ArmService {
 
     String saveBlobData(String containerName, String filename, BinaryData binaryData);
@@ -34,4 +35,8 @@ public interface ArmService {
      */
     boolean deleteBlobData(String containerName, String blobPathAndName);
 
+    boolean deleteMultipleBlobsUsingBatching(String containerName, List<String> blobPathAndName);
+
+    boolean deleteMultipleBlobsIndividually(String containerName, List<String> blobPathAndName);
+
 }

src/main/java/uk/gov/hmcts/darts/arm/service/DeleteArmResponseFilesHelper.java

@@ -11,10 +11,12 @@ public interface DeleteArmResponseFilesHelper {
 
     void deleteDanglingResponses(BatchInputUploadFileFilenameProcessor batchUploadFileFilenameProcessor);
 
-    List<Boolean> deleteResponseBlobs(List<String> responseBlobsToBeDeleted);
+    Boolean deleteResponseBlobIndividually(String responseBlobsToBeDeleted);
+
+    Boolean deleteResponseBlobs(List<String> responseBlobsToBeDeleted);
 
     void deleteResponseBlobs(ArmResponseBatchData armResponseBatchData);
 
     List<String> getResponseBlobsToBeDeleted(ArmResponseBatchData armResponseBatchData);
-    
+
 }

src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java

@@ -620,7 +620,7 @@ private ArmResponseCreateRecord getResponseCreateRecordOrDelete(String createRec
             return objectMapper.readValue(createRecordBinary.toString(), ArmResponseCreateRecord.class);
         } catch (Exception e) {
             log.error("Unable to read ARM response create record file {} - About to delete ", createRecordFilenameAndPath, e);
-            deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(createRecordFilenameAndPath));
+            deleteArmResponseFilesHelper.deleteResponseBlobIndividually(createRecordFilenameAndPath);
             throw e;
         }
     }
@@ -673,7 +673,7 @@ private ArmResponseUploadFileRecord getResponseUploadFileRecordOrDelete(String u
             return objectMapper.readValue(uploadFileBinary.toString(), ArmResponseUploadFileRecord.class);
         } catch (Exception e) {
             log.error("Unable to read ARM response upload file {} - About to delete ", uploadFileFilenameAndPath, e);
-            deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(uploadFileFilenameAndPath));
+            deleteArmResponseFilesHelper.deleteResponseBlobIndividually(uploadFileFilenameAndPath);
             throw e;
         }
     }
@@ -804,7 +804,7 @@ private ResponseFilenames getArmResponseFilenames(List<String> responseFiles, St
             } catch (IllegalArgumentException e) {
                 // This occurs when the filename is not parsable
                 log.error("Invalid ARM response filename: {} for manifest {}", responseFile, manifestName);
-                deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(responseFile));
+                deleteArmResponseFilesHelper.deleteResponseBlobIndividually(responseFile);
             }
         }
         return responseFilenames;
@@ -843,7 +843,7 @@ private void readInvalidLineFile(BinaryData invalidLineFileBinary, InvalidLineFi
                 } else {
                     log.warn("Failed to obtain EOD id (relation id) from invalid line record {} from file {}", input,
                              invalidLineFileFilenameProcessor.getInvalidLineFilename());
-                    deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(invalidLineFileFilenameAndPath));
+                    deleteArmResponseFilesHelper.deleteResponseBlobIndividually(invalidLineFileFilenameAndPath);
                 }
 
             } catch (Exception e) {
@@ -860,7 +860,7 @@ private ArmResponseInvalidLineRecord getResponseInvalidLineRecordOrDelete(String
             return objectMapper.readValue(invalidLineFileBinary.toString(), ArmResponseInvalidLineRecord.class);
         } catch (Exception e) {
             log.error("Unable to read ARM response {} - About to delete ", invalidLineFileFilenameAndPath, e);
-            deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(invalidLineFileFilenameAndPath));
+            deleteArmResponseFilesHelper.deleteResponseBlobIndividually(invalidLineFileFilenameAndPath);
             throw e;
         }
     }