Feature/contextual variables report

pom.xml

@@ -22,7 +22,7 @@
         <commons-lang3.version>3.20.0</commons-lang3.version>
 
         <!-- Dependency versions not managed by Spring -->
-        <bpm.version>1.2.1</bpm.version>
+        <bpm.version>1.2.2</bpm.version>
         <springdoc.version>3.0.3</springdoc.version>
         <mapstruct.version>1.6.3</mapstruct.version>
         <pitest.version>1.25.5</pitest.version>

src/main/java/fr/insee/genesis/Constants.java

@@ -2,6 +2,8 @@
 
 import lombok.experimental.UtilityClass;
 
+import java.util.regex.Pattern;
+
 @UtilityClass
 public class Constants {
 
@@ -11,6 +13,8 @@ public class Constants {
     public static final String METADATA_SEPARATOR = ".";
     public static final String DATE_REGEX = "(^([0-9]{4})[\\-\\/]([0-9]|1[0-2]|0[1-9])[\\-\\/]([0-9]|[0-2][0-9]|3[0-1])$)|" +
             "(^([0-9]|[0-2][0-9]|3[0-1])[\\-\\/]([0-9]|1[0-2]|0[1-9])[\\-\\/]([0-9]{4})$)";
+    public static final Pattern QUESTIONNAIRE_ID_PATTERN =
+            Pattern.compile("^[A-Za-z0-9_-]+$");
     public static final String FILTER_RESULT_PREFIX = "FILTER_RESULT_";
     public static final String MISSING_SUFFIX = "_MISSING";
     public static final String MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME = "lunaticjsondata";
@@ -21,6 +25,8 @@ public class Constants {
     public static final String MONGODB_CONTEXT_COLLECTION_NAME = "dataProcessingContexts";
     public static final String MONGODB_CONTEXTUAL_PREVIOUS_COLLECTION_NAME = "editedPrevious";
     public static final String MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME = "editedExternal";
+    public static final String TYPE_PREVIOUS = "PREVIOUS";
+    public static final String TYPE_EXTERNAL = "EXTERNAL";
     public static final String LOOP_NAME_PREFIX = "BOUCLE";
     public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses";
     public static final String MONGODB_RAW_RESPONSES_COLLECTION_NAME = "rawResponses";

src/main/java/fr/insee/genesis/controller/dto/ContextualVariableFileReportDto.java

@@ -0,0 +1,6 @@
+package fr.insee.genesis.controller.dto;
+
+public record ContextualVariableFileReportDto(
+        String fileName,
+        String type
+) {}

src/main/java/fr/insee/genesis/controller/dto/SaveContextualVariablesReportDto.java

@@ -0,0 +1,9 @@
+package fr.insee.genesis.controller.dto;
+
+import java.util.List;
+
+public record SaveContextualVariablesReportDto(
+        String questionnaireId,
+        int processedFiles,
+        List<ContextualVariableFileReportDto> files
+) {}

src/main/java/fr/insee/genesis/controller/rest/responses/ContextualVariableController.java

@@ -2,6 +2,7 @@
 
 import fr.insee.genesis.Constants;
 import fr.insee.genesis.configuration.Config;
+import fr.insee.genesis.controller.dto.SaveContextualVariablesReportDto;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.ports.api.ContextualExternalVariableApiPort;
 import fr.insee.genesis.domain.ports.api.ContextualPreviousVariableApiPort;
@@ -22,6 +23,9 @@
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.regex.Pattern;
+
+import static fr.insee.genesis.Constants.QUESTIONNAIRE_ID_PATTERN;
 
 @RequestMapping(path = "/contextual-variables")
 @Controller
@@ -34,6 +38,7 @@ public class ContextualVariableController {
     private final ContextualVariableApiPort contextualVariableApiPort;
     private final Config config;
 
+
     @Operation(summary = "Get contextual variables (external and previous)")
     @GetMapping(path = "/")
     @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER')")
@@ -52,16 +57,35 @@ public ResponseEntity<Object> getContextualVariables(
     public ResponseEntity<Object> saveContextualVariables(
             @RequestParam("questionnaireId") String questionnaireId
     ) throws GenesisException{
-            FileUtils fileUtils = new FileUtils(config);
+            validateQuestionnaireId(questionnaireId);
 
-            String contextualFolderPath = fileUtils.getDataFolder(questionnaireId, "WEB", null) + Constants.CONTEXTUAL_FOLDER;
+            FileUtils fileUtils = new FileUtils(config);
 
-            int fileCount = contextualVariableApiPort.saveContextualVariableFiles(questionnaireId, fileUtils,contextualFolderPath);
+            int fileCount = contextualVariableApiPort.saveContextualVariableFiles(questionnaireId, fileUtils);
 
             return ResponseEntity.ok("%d file(s) processed for questionnaire %s !".formatted(fileCount, questionnaireId));
 
     }
 
+    @Operation(summary = "Save all contextual variables json files and return processed file details")
+    @PostMapping(path = "/json/report")
+    @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER')")
+    public ResponseEntity<SaveContextualVariablesReportDto> saveContextualVariablesWithReport(
+            @RequestParam("questionnaireId") String questionnaireId
+    ) throws GenesisException {
+        validateQuestionnaireId(questionnaireId);
+
+        FileUtils fileUtils = new FileUtils(config);
+
+        SaveContextualVariablesReportDto report =
+                contextualVariableApiPort.saveContextualVariableFilesWithReport(
+                        questionnaireId,
+                        fileUtils
+                );
+
+        return ResponseEntity.ok(report);
+    }
+
     @Operation(summary = "Add contextual previous json file")
     @PostMapping(path = "previous/json")
     @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER','USER_BACK_OFFICE')")
@@ -121,4 +145,10 @@ private static void moveFile(String questionnaireId, Mode mode, FileUtils fileUt
             throw new GenesisException(HttpStatus.INTERNAL_SERVER_ERROR, "Error while moving file to done");
         }
     }
+
+    private void validateQuestionnaireId(String questionnaireId) throws GenesisException {
+        if (!QUESTIONNAIRE_ID_PATTERN.matcher(questionnaireId).matches()) {
+            throw new GenesisException(HttpStatus.BAD_REQUEST, "Invalid questionnaireId");
+        }
+    }
 }

src/main/java/fr/insee/genesis/domain/ports/api/ContextualVariableApiPort.java

@@ -1,10 +1,15 @@
 package fr.insee.genesis.domain.ports.api;
 
+import fr.insee.genesis.controller.dto.SaveContextualVariablesReportDto;
 import fr.insee.genesis.domain.model.contextualvariable.ContextualVariableModel;
 import fr.insee.genesis.exceptions.GenesisException;
 import fr.insee.genesis.infrastructure.utils.FileUtils;
 
 public interface ContextualVariableApiPort {
     ContextualVariableModel getContextualVariable(String collectionInstrumentId, String interrogationId);
-    int saveContextualVariableFiles(String collectionInstrumentId, FileUtils fileUtils, String contextualFolderPath) throws GenesisException;
+    int saveContextualVariableFiles(String collectionInstrumentId, FileUtils fileUtils) throws GenesisException;
+    SaveContextualVariablesReportDto saveContextualVariableFilesWithReport(
+            String collectionInstrumentId,
+            FileUtils fileUtils
+    ) throws GenesisException;
 }

src/main/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonService.java

@@ -1,5 +1,7 @@
 package fr.insee.genesis.domain.service.contextualvariable;
 
+import fr.insee.genesis.controller.dto.ContextualVariableFileReportDto;
+import fr.insee.genesis.controller.dto.SaveContextualVariablesReportDto;
 import fr.insee.genesis.controller.dto.VariableQualityToolDto;
 import fr.insee.genesis.controller.dto.VariableStateDto;
 import fr.insee.genesis.domain.model.contextualvariable.ContextualExternalVariableModel;
@@ -26,8 +28,13 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Stream;
 
+import static fr.insee.genesis.Constants.CONTEXTUAL_FOLDER;
+import static fr.insee.genesis.Constants.TYPE_EXTERNAL;
+import static fr.insee.genesis.Constants.TYPE_PREVIOUS;
+
 @Service
 @Slf4j
 public class ContextualVariableJsonService implements ContextualVariableApiPort {
@@ -76,20 +83,47 @@ public ContextualVariableModel getContextualVariable(String collectionInstrument
     }
 
     @Override
-    public int saveContextualVariableFiles(String collectionInstrumentId, FileUtils fileUtils, String contextualFolderPath) throws GenesisException {
-        int fileCount = 0;
+    public int saveContextualVariableFiles(
+            String collectionInstrumentId,
+            FileUtils fileUtils
+    ) throws GenesisException {
+        return saveContextualVariableFilesWithReport(
+                collectionInstrumentId,
+                fileUtils
+        ).processedFiles();
+    }
+
+    /**
+     * Generates a detailed report of processed contextual variable files.
+     * Added to preserve the existing behavior of {@link #saveContextualVariableFiles(String, FileUtils)},
+     * which only returns the number of processed files.
+     */
+    @Override
+    public SaveContextualVariablesReportDto saveContextualVariableFilesWithReport(
+            String collectionInstrumentId,
+            FileUtils fileUtils
+    ) throws GenesisException {
+        List<ContextualVariableFileReportDto> files = new ArrayList<>();
+        String contextualFolderPath =
+                fileUtils.getDataFolder(collectionInstrumentId, "WEB", null)
+                        + CONTEXTUAL_FOLDER;
 
         for (Mode mode : Mode.values()) {
-            try (Stream<Path> filePaths = Files.list(Path.of(contextualFolderPath))) {
+            Path contextualFolder = Path.of(contextualFolderPath);
+            try (Stream<Path> filePaths = Files.list(contextualFolder)) {
                 Iterator<Path> it = filePaths
                         .filter(path -> path.toString().endsWith(".json"))
                         .iterator();
+
                 while (it.hasNext()) {
                     Path jsonFilePath = it.next();
-                    if (processContextualVariableFile(collectionInstrumentId, jsonFilePath)) {
-                        //If the file is indeed a contextual variables file and had been processed
+
+                    Optional<ContextualVariableFileReportDto> report =
+                            processContextualVariableFileForReport(collectionInstrumentId, jsonFilePath);
+
+                    if (report.isPresent()) {
                         moveFile(collectionInstrumentId, mode, fileUtils, jsonFilePath.toString());
-                        fileCount++;
+                        files.add(report.get());
                     }
                 }
             } catch (NoSuchFileException nsfe) {
@@ -98,9 +132,64 @@ public int saveContextualVariableFiles(String collectionInstrumentId, FileUtils
                 log.warn(ioe.toString());
             }
         }
-        return fileCount;
+
+        return new SaveContextualVariablesReportDto(
+                collectionInstrumentId,
+                files.size(),
+                files
+        );
     }
 
+    private Optional<ContextualVariableFileReportDto> processContextualVariableFileForReport(
+            String collectionInstrumentId,
+            Path jsonFilePath
+    ) throws GenesisException {
+        try {
+            Optional<String> type = processContextualVariableFileAndGetType(
+                    collectionInstrumentId,
+                    jsonFilePath
+            );
+
+            return type.map(value -> new ContextualVariableFileReportDto(
+                    jsonFilePath.getFileName().toString(),
+                    value
+            ));
+        } catch (GenesisException e) {
+            throw new GenesisException(
+                    e.getStatus(),
+                    "Error while processing file '%s' : %s"
+                            .formatted(jsonFilePath.getFileName().toString(), e.getMessage())
+            );
+        }
+    }
+
+    private Optional<String> processContextualVariableFileAndGetType(
+            String collectionInstrumentId,
+            Path jsonFilePath
+    ) throws GenesisException {
+        boolean isPrevious = contextualPreviousVariableApiPort.readContextualPreviousFile(
+                collectionInstrumentId.toUpperCase(),
+                null,
+                jsonFilePath.toString()
+        );
+
+        if (isPrevious) {
+            return Optional.of(TYPE_PREVIOUS);
+        }
+
+        boolean isExternal = contextualExternalVariableApiPort.readContextualExternalFile(
+                collectionInstrumentId.toUpperCase(),
+                jsonFilePath.toString()
+        );
+
+        if (isExternal) {
+            return Optional.of(TYPE_EXTERNAL);
+        }
+
+        return Optional.empty();
+    }
+
+
     private static void moveFile(String collectionInstrumentId, Mode mode, FileUtils fileUtils, String filePath) throws GenesisException {
         try {
             fileUtils.moveFiles(Path.of(filePath), fileUtils.getDoneFolder(collectionInstrumentId, mode.getFolder()));
@@ -142,18 +231,4 @@ private VariableQualityToolDto extractValue(Object variable, String variableName
         );
         return variableQualityToolDto;
     }
-
-    /**
-     * @return true if any contextual variable part found in file, false otherwise
-     */
-    private boolean processContextualVariableFile(String collectionInstrumentId, Path jsonFilePath) throws GenesisException {
-        return contextualPreviousVariableApiPort.readContextualPreviousFile(
-                collectionInstrumentId.toUpperCase(),
-                null,
-                jsonFilePath.toString()
-        ) || contextualExternalVariableApiPort.readContextualExternalFile(
-                collectionInstrumentId.toUpperCase(),
-                jsonFilePath.toString()
-        );
-    }
 }

src/test/java/fr/insee/genesis/controller/rest/responses/ContextualVariableControllerTest.java

@@ -2,6 +2,8 @@
 
 import fr.insee.genesis.Constants;
 import fr.insee.genesis.TestConstants;
+import fr.insee.genesis.controller.dto.ContextualVariableFileReportDto;
+import fr.insee.genesis.controller.dto.SaveContextualVariablesReportDto;
 import fr.insee.genesis.domain.model.contextualvariable.ContextualVariableModel;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.ports.api.ContextualExternalVariableApiPort;
@@ -18,10 +20,13 @@
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.mockito.junit.jupiter.MockitoSettings;
 import org.mockito.quality.Strictness;
+import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
+
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doReturn;
@@ -87,11 +92,41 @@ void saveContextualVariables() {
         //THEN
         verify(contextualVariableApiPort, times(1)).saveContextualVariableFiles(
                 eq(TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID),
-                any(),
                 any()
         );
     }
 
+    @Test
+    @SneakyThrows
+    void saveContextualVariablesWithReport() {
+        // GIVEN
+        SaveContextualVariablesReportDto report = new SaveContextualVariablesReportDto(
+                TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID,
+                1,
+                List.of(new ContextualVariableFileReportDto("ok.json", "PREVIOUS"))
+        );
+
+        doReturn(report).when(contextualVariableApiPort).saveContextualVariableFilesWithReport(
+                eq(TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID),
+                any()
+        );
+
+        // WHEN
+        ResponseEntity<SaveContextualVariablesReportDto> response =
+                contextualVariableController.saveContextualVariablesWithReport(
+                        TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID
+                );
+
+        // THEN
+        verify(contextualVariableApiPort, times(1)).saveContextualVariableFilesWithReport(
+                eq(TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID),
+                any()
+        );
+
+        Assertions.assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
+        Assertions.assertThat(response.getBody()).isEqualTo(report);
+    }
+
     @Test
     @SneakyThrows
     void readContextualPreviousJson() {
@@ -134,7 +169,7 @@ void readContextualPreviousJson() {
     }
 
     @Test
-    void readContextualPreviousJson_notJson() throws GenesisException {
+    void readContextualPreviousJson_notJson() {
         Assertions.assertThatThrownBy(() -> contextualVariableController.readContextualPreviousJson(
                         TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID,
                         Mode.WEB,
@@ -180,13 +215,13 @@ void readContextualExternalJson() {
 
         //THEN
         verify(contextualExternalVariableApiPort).readContextualExternalFile(
-                eq(TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID.toUpperCase()),
-                eq(expectedFilePath)
+                TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID.toUpperCase(),
+                expectedFilePath
         );
     }
 
     @Test
-    void readContextualExternalJson_notJson() throws GenesisException {
+    void readContextualExternalJson_notJson() {
 
         Assertions.assertThatThrownBy(() -> contextualVariableController.readContextualExternalJson(
                         TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID,