From 2f534fd061ea1cf9b26d50b4f6d79ec497a5f6b7 Mon Sep 17 00:00:00 2001 From: JeromeMBourgeois Date: Mon, 16 Sep 2024 17:44:07 +0200 Subject: [PATCH 01/25] Add endpoint to transform a dataRelationShip in DDI to Json --- .../controller/TransformationController.java | 22 ++- src/main/resources/dataRelToJson.xsl | 170 ++++++++++++++++++ 2 files changed, 185 insertions(+), 7 deletions(-) create mode 100644 src/main/resources/dataRelToJson.xsl diff --git a/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java b/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java index f1c1532f..2ffd280f 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java @@ -1,5 +1,7 @@ package fr.insee.rmes.transfoxsl.controller; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import fr.insee.rmes.exceptions.VtlTransformationException; import fr.insee.rmes.exceptions.XsltTransformationException; import fr.insee.rmes.transfoxsl.service.XsltTransformationService; @@ -33,6 +35,7 @@ public class TransformationController { public static final String DEREFERENCE_XSL = "dereference.xsl"; public static final String FAILED_TO_PROCESS_THE_INPUT_FILE = "Failed to process the input file"; public static final String TRANSFORMATION_FAILED_DURING_THE_XSLT_PROCESSING = "Transformation failed during the XSLT processing"; + public static final String REL_TO_JSON_XSL = "dataRelToJson.xsl"; private final XsltTransformationService xsltTransformationService; private final MultipartFileUtils multipartFileUtils; @@ -105,24 +108,29 @@ public ResponseEntity ddi2vtl(@RequestParam("file") Multipa } } - @Operation(summary = "Générer les règles VTL à partir d'une physicalInstance et renvoyer sous forme de JSON") + @Operation(summary = "Transformer un objet dataRelationShip en JSON") @PostMapping(value = "/ddi2vtlJson", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public ResponseEntity> ddi2vtlJson(@RequestParam("file") MultipartFile file) { + public ResponseEntity ddiDataRelationShiptoJson(@RequestParam("file") MultipartFile file) { try { // Conversion du MultipartFile en InputStream InputStream inputStream = multipartFileUtils.convertToInputStream(file); // Première transformation - XML en sortie - List intermediateOutput = xsltTransformationService.transform(inputStream, DEREFERENCE_XSL, false); + List outputText = xsltTransformationService.transform(inputStream, REL_TO_JSON_XSL, true); + // Joindre les éléments de la liste en une seule chaîne JSON propre + String jsonOutput = String.join("", outputText); - // Deuxième transformation - Texte en sortie - InputStream intermediateInputStream = new ByteArrayInputStream(String.join("\n", intermediateOutput).getBytes(StandardCharsets.UTF_8)); - List outputText = xsltTransformationService.transform(intermediateInputStream, DDI_2_VTL_XSL, true); + // Utiliser Jackson pour vérifier la validité du JSON + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = mapper.readTree(jsonOutput); // Désérialisation pour vérifier + + // Sérialiser à nouveau le JSON proprement + String cleanJson = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonNode); // Retourner la liste sous forme de JSON return ResponseEntity.ok() .contentType(MediaType.APPLICATION_JSON) - .body(outputText); + .body(cleanJson); } catch (IOException e) { throw new VtlTransformationException(FAILED_TO_PROCESS_THE_INPUT_FILE + ".", e); } catch (XsltTransformationException e) { diff --git a/src/main/resources/dataRelToJson.xsl b/src/main/resources/dataRelToJson.xsl new file mode 100644 index 00000000..c393f7ae --- /dev/null +++ b/src/main/resources/dataRelToJson.xsl @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + [ + + + + + + + + + , + + { + + + + "id": " + + ", "nom": " + + ", "label": + [ + { + "contenu": " + + ", "langue": "fr" + + }, + { + "contenu": "", + "langue": "en" + } + ] + + , + "variables": + [ + + + + + + + + , + + { + + + + "id": " + + ", "nom": " + + ", "label": + + [ + { + "contenu": " + + ", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ] + + , "ordre": " + + " + , "representation": " + + + + + + + codes + numerique + texte + date + + + + " + + + , "controles": + { + "codes": [ + + + + + + , + + + + { "code": " + + " , "label": [ + + + { + "langue": "fr", + "contenu": " + + " + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + + + ] + } + + + + + + + + + + + + + } + + + + ] + } + + ] + + \ No newline at end of file From f40547490efabb89863586b87567ccf20ef7de19 Mon Sep 17 00:00:00 2001 From: JeromeMBourgeois Date: Mon, 16 Sep 2024 18:05:35 +0200 Subject: [PATCH 02/25] Fix test --- .../controller/TransformationController.java | 2 +- .../TransformationControllerTest.java | 20 ++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java b/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java index 2ffd280f..344b891d 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java @@ -109,7 +109,7 @@ public ResponseEntity ddi2vtl(@RequestParam("file") Multipa } @Operation(summary = "Transformer un objet dataRelationShip en JSON") - @PostMapping(value = "/ddi2vtlJson", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @PostMapping(value = "/dataRelationShiptoJson", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) public ResponseEntity ddiDataRelationShiptoJson(@RequestParam("file") MultipartFile file) { try { // Conversion du MultipartFile en InputStream diff --git a/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java b/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java index 00da2ce0..b7909295 100644 --- a/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java +++ b/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java @@ -82,31 +82,33 @@ void ddi2vtl_ShouldReturnServerError_WhenTransformationFails() throws Exception @Test @WithMockUser // Simule un utilisateur authentifié - void ddi2vtlJson_ShouldReturnJson_WhenTransformationIsSuccessful() throws Exception { + void dataRelationShiptoJson_ShouldReturnJson_WhenTransformationIsSuccessful() throws Exception { // Mocking MultipartFile and transformation service MockMultipartFile mockFile = new MockMultipartFile("file", "test.xml", "application/xml", "".getBytes()); // Mocking conversion to InputStream when(multipartFileUtils.convertToInputStream(mockFile)).thenReturn(new ByteArrayInputStream("".getBytes())); - // Mocking transformation steps - List intermediateOutput = List.of("XML"); - List finalOutput = List.of("VTL rule 1", "VTL rule 2"); + // Mocking transformation result + String intermediateOutput = "[{\"VTL\":\"rule 1\", \"VTL2\":\"rule 2\"}]"; when(xsltTransformationService.transform(any(InputStream.class), anyString(), anyBoolean())) - .thenReturn(intermediateOutput); - when(xsltTransformationService.transform(any(InputStream.class), anyString(), anyBoolean())) - .thenReturn(finalOutput); + .thenReturn(List.of(intermediateOutput)); + + // Expected formatted JSON output (as an object) + String expectedJson = "[{\n \"VTL\" : \"rule 1\",\n \"VTL2\" : \"rule 2\"\n}]"; // Perform the request with CSRF token and verify the JSON response - mockMvc.perform(MockMvcRequestBuilders.multipart("/xsl//ddi2vtlJson") + mockMvc.perform(MockMvcRequestBuilders.multipart("/xsl/dataRelationShiptoJson") // Correct URL .file(mockFile) .with(csrf())) // Add CSRF token to the request .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON)) // Verify JSON content type - .andExpect(content().json("[\"VTL rule 1\", \"VTL rule 2\"]")); + .andExpect(content().json(expectedJson)); // Verify formatted JSON response } + + @Test @WithMockUser void ddi2vtlBrut_ShouldReturnPlainText_WhenTransformationIsSuccessful() throws Exception { From 953171b15e0db997c6ee873ba42130150361fdc4 Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Mon, 23 Sep 2024 17:18:56 +0200 Subject: [PATCH 03/25] refactor (mutulualize, separate concerns, remove useless, remove redondant, remove elastic search) WIP --- .../XsltTransformationException.java | 15 +++-- .../service/ColecticaServiceImpl.java | 39 ++---------- .../controller/TransformationController.java | 42 ++++++------- .../service/SerializerConfigurer.java | 28 +++++++++ .../service/XsltTransformationService.java | 59 ++++++++++--------- .../service/internal/DDIDerefencer.java | 21 +++++++ .../service/internal/DDITransformerToVtl.java | 24 ++++++++ 7 files changed, 142 insertions(+), 86 deletions(-) create mode 100644 src/main/java/fr/insee/rmes/transfoxsl/service/SerializerConfigurer.java create mode 100644 src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDIDerefencer.java create mode 100644 src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDITransformerToVtl.java diff --git a/src/main/java/fr/insee/rmes/exceptions/XsltTransformationException.java b/src/main/java/fr/insee/rmes/exceptions/XsltTransformationException.java index 19e4e9f8..8a4fc0f2 100644 --- a/src/main/java/fr/insee/rmes/exceptions/XsltTransformationException.java +++ b/src/main/java/fr/insee/rmes/exceptions/XsltTransformationException.java @@ -1,11 +1,18 @@ package fr.insee.rmes.exceptions; +import net.sf.saxon.s9api.SaxonApiException; + +import java.util.Optional; + public class XsltTransformationException extends RuntimeException { - public XsltTransformationException(String message) { - super(message); - } - public XsltTransformationException(String message, Throwable cause) { + public XsltTransformationException(String message, SaxonApiException cause) { super(message, cause); } + + public Optional getXmlErrorMessage(){ + var cause = getCause(); + return cause != null ? Optional.of(cause.getMessage()) : Optional.empty(); + } + } \ No newline at end of file diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java index e3222055..35401ba6 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java @@ -1,6 +1,5 @@ package fr.insee.rmes.tocolecticaapi.service; -import co.elastic.clients.elasticsearch.ElasticsearchClient; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -21,6 +20,7 @@ import fr.insee.rmes.utils.XMLUtils; import fr.insee.rmes.utils.export.XDocReport; import lombok.NonNull; +import lombok.RequiredArgsConstructor; import net.sf.saxon.TransformerFactoryImpl; import net.sf.saxon.s9api.ExtensionFunction; import net.sf.saxon.s9api.Processor; @@ -81,10 +81,10 @@ import java.nio.file.Files; import java.util.*; -import static fr.insee.rmes.transfoxsl.controller.TransformationController.DEREFERENCE_XSL; import static org.springframework.security.oauth2.core.endpoint.OAuth2ParameterNames.ERROR; @Service +@RequiredArgsConstructor public class ColecticaServiceImpl implements ColecticaService { private static final String CONTENT_TYPE = "Content-Type"; @@ -92,8 +92,6 @@ public class ColecticaServiceImpl implements ColecticaService { private static final String APPLICATION_JSON = "application/json"; private static final String AUTHORIZATION = "Authorization"; private static final String BEARER = "Bearer "; - private static final String HTTP = "http://"; - private static final String HTTPS = "https://"; private static final String VERSION = "version"; private static final String SEARCH = "/_search"; private static final String APIKEYHEADER = "apiKey "; @@ -139,44 +137,19 @@ public class ColecticaServiceImpl implements ColecticaService { @Value("${fr.insee.rmes.elasticsearch.url}") private String elasticUrl; - @Value("${fr.insee.rmes.elasticsearch.apiId}") - private String apiId; - @Value("${fr.insee.rmes.elasticsearch.apikey}") private String apiKey; - @Autowired - public ElasticsearchClient elasticsearchClient; - @Autowired public RestTemplate restTemplate; @Autowired - XDocReport xdr; + private final XDocReport xdr; @Autowired - ExportUtils exportUtils; + private final ExportUtils exportUtils; - public CloseableHttpClient httpClient; + private final CloseableHttpClient httpClient; @Autowired - private XsltTransformationService xsltTransformationService; - public ColecticaServiceImpl(ElasticsearchClient elasticsearchClient, RestTemplate restTemplate) { - this.elasticsearchClient=elasticsearchClient; - this.restTemplate=restTemplate; - } - private CloseableHttpClient mockHttpClient; - public ColecticaServiceImpl(CloseableHttpClient mockHttpClient,ElasticsearchClient elasticsearchClient, RestTemplate restTemplate) { - this.elasticsearchClient=elasticsearchClient; - this.restTemplate=restTemplate; - this.mockHttpClient=mockHttpClient; - } - - public ColecticaServiceImpl(){ - - } - @Autowired - public ColecticaServiceImpl(CloseableHttpClient httpClient) { - this.httpClient = httpClient; - } - + private final XsltTransformationService xsltTransformationService; @Override public ResponseEntity findFragmentByUuid(String uuid) throws ExceptionColecticaUnreachable, IOException { diff --git a/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java b/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java index 344b891d..5e7410c1 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java @@ -5,11 +5,16 @@ import fr.insee.rmes.exceptions.VtlTransformationException; import fr.insee.rmes.exceptions.XsltTransformationException; import fr.insee.rmes.transfoxsl.service.XsltTransformationService; +import fr.insee.rmes.transfoxsl.service.internal.DDIDerefencer; +import fr.insee.rmes.transfoxsl.service.internal.DDITransformerToVtl; import fr.insee.rmes.transfoxsl.utils.MultipartFileUtils; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ByteArrayResource; import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; @@ -29,10 +34,9 @@ @Controller @RequestMapping("/xsl") @Tag(name = "TransformationController", description = "API pour lancer des transformations XSLT") +@Slf4j public class TransformationController { - public static final String DDI_2_VTL_XSL = "ddi2vtl.xsl"; - public static final String DEREFERENCE_XSL = "dereference.xsl"; public static final String FAILED_TO_PROCESS_THE_INPUT_FILE = "Failed to process the input file"; public static final String TRANSFORMATION_FAILED_DURING_THE_XSLT_PROCESSING = "Transformation failed during the XSLT processing"; public static final String REL_TO_JSON_XSL = "dataRelToJson.xsl"; @@ -47,50 +51,46 @@ public TransformationController(XsltTransformationService xsltTransformationServ @Operation(summary = "Déréférencer un objet DDI") @PostMapping(value = "/Derefddi", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public ResponseEntity Derefddi(@RequestParam("file") MultipartFile file) { + public ResponseEntity dereferenceDDI(@RequestParam("file") MultipartFile file) { + DDIDerefencer ddiDerefencer = new DDIDerefencer(xsltTransformationService); + try { - // Conversion du MultipartFile en InputStream InputStream inputStream = multipartFileUtils.convertToInputStream(file); - - // Première transformation - XML en sortie - List outputText = xsltTransformationService.transform(inputStream, DEREFERENCE_XSL, false); - String finalOutput = String.join("\n", outputText); + String finalOutput = ddiDerefencer.dereferenceToString(inputStream); // Retourner la liste sous forme de JSON return ResponseEntity.ok() .contentType(MediaType.TEXT_PLAIN) .body(finalOutput); } catch (IOException e) { - throw new VtlTransformationException(FAILED_TO_PROCESS_THE_INPUT_FILE + ".", e); - } catch (XsltTransformationException e) { - throw new VtlTransformationException(TRANSFORMATION_FAILED_DURING_THE_XSLT_PROCESSING + ".", e); - } catch (VtlTransformationException e) { + log.error("Error while processing dereferenceDDI", e); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(null); + } catch (XsltTransformationException e) { + log.info("Exception with user input", e.getCause()); + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(TRANSFORMATION_FAILED_DURING_THE_XSLT_PROCESSING+" : "+e.getXmlErrorMessage()); } } @Operation(summary = "Générer un fichier texte contenant les règles VTL à partir d'une physicalInstance") @PostMapping(value = "/ddi2vtl", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public ResponseEntity ddi2vtl(@RequestParam("file") MultipartFile file) { + public ResponseEntity ddi2vtl(@RequestParam("file") MultipartFile file) { + + DDITransformerToVtl ddiTransformerToVtl = new DDITransformerToVtl(xsltTransformationService); + try { - // Conversion du MultipartFile en InputStream InputStream inputStream = multipartFileUtils.convertToInputStream(file); // Première transformation - XML en sortie (on récupère la sortie en tant que chaîne) - List intermediateOutput = xsltTransformationService.transform(inputStream, DEREFERENCE_XSL, false); + List intermediateOutput = xsltTransformationService.transformToXml(inputStream, DEREFERENCE_XSL, false); // Conversion de la sortie intermédiaire en InputStream pour la deuxième transformation InputStream intermediateInputStream = new ByteArrayInputStream(String.join("\n", intermediateOutput).getBytes(StandardCharsets.UTF_8)); - // Deuxième transformation - Texte en sortie (on récupère directement une liste de lignes de texte) - List outputText = xsltTransformationService.transform(intermediateInputStream, DDI_2_VTL_XSL, true); - // Préparation du contenu à retourner sous forme d'InputStream - String finalOutput = String.join("\n", outputText); - ByteArrayInputStream finalInputStream = new ByteArrayInputStream(finalOutput.getBytes(StandardCharsets.UTF_8)); + // Préparation de la réponse avec le fichier texte - InputStreamResource resource = new InputStreamResource(finalInputStream); + Resource resource = new ByteArrayResource(ddiTransformerToVtl.transform(inputStream)); HttpHeaders headers = new HttpHeaders(); headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=vtl.txt"); diff --git a/src/main/java/fr/insee/rmes/transfoxsl/service/SerializerConfigurer.java b/src/main/java/fr/insee/rmes/transfoxsl/service/SerializerConfigurer.java new file mode 100644 index 00000000..b8fe3577 --- /dev/null +++ b/src/main/java/fr/insee/rmes/transfoxsl/service/SerializerConfigurer.java @@ -0,0 +1,28 @@ +package fr.insee.rmes.transfoxsl.service; + +import net.sf.saxon.s9api.Processor; +import net.sf.saxon.s9api.Serializer; + +import java.io.OutputStream; + +public interface SerializerConfigurer { + + static SerializerConfigurer forXmlString(Processor processor) { + return outputStream -> { + var serializer = processor.newSerializer(outputStream); + serializer.setOutputProperty(Serializer.Property.METHOD, "xml"); + return serializer; + }; + } + + static SerializerConfigurer forRawText(Processor processor) { + return outputStream -> { + var serializer = processor.newSerializer(outputStream); + serializer.setOutputProperty(Serializer.Property.METHOD, "text"); // Configurer pour un texte simple + serializer.setOutputProperty(Serializer.Property.OMIT_XML_DECLARATION, "yes"); // Pas de déclaration XML pour le texte + return serializer; + }; + } + + Serializer configure(OutputStream outputStream); +} diff --git a/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java b/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java index c65c6cfb..235bac67 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java @@ -1,34 +1,29 @@ package fr.insee.rmes.transfoxsl.service; import fr.insee.rmes.exceptions.XsltTransformationException; +import lombok.extern.slf4j.Slf4j; import net.sf.saxon.s9api.*; import org.springframework.core.io.ClassPathResource; import org.springframework.stereotype.Service; import javax.xml.transform.stream.StreamSource; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; @Service +@Slf4j public class XsltTransformationService { - private static final Logger logger = Logger.getLogger(XsltTransformationService.class.getName()); private final Processor processor; public XsltTransformationService() { this.processor = new Processor(false); } - public List transform(InputStream inputStream, String xslFileName, boolean isTextOutput) throws XsltTransformationException, IOException { + byte[] transform(InputStream inputStream, String xslFileName, SerializerConfigurer serializerConfigurer) throws XsltTransformationException, IOException { try { - logger.log(Level.INFO, "Starting transformation with XSLT file: {0}", xslFileName); + log.atDebug().log(()->"Starting transformation with XSLT file: "+ xslFileName); XsltCompiler compiler = processor.newXsltCompiler(); @@ -42,30 +37,38 @@ public List transform(InputStream inputStream, String xslFileName, boole // Utilisation d'un OutputStream pour capturer la sortie en mémoire ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - Serializer out = processor.newSerializer(outputStream); - if (isTextOutput) { - out.setOutputProperty(Serializer.Property.METHOD, "text"); // Configurer pour un texte simple - out.setOutputProperty(Serializer.Property.OMIT_XML_DECLARATION, "yes"); // Pas de déclaration XML pour le texte - } else { - out.setOutputProperty(Serializer.Property.METHOD, "xml"); // Configurer pour un XML si nécessaire - } - transformer.setDestination(out); + Serializer serializer = serializerConfigurer.configure(outputStream); + + transformer.setDestination(serializer); transformer.transform(); - // Capture de la sortie sous forme de chaîne - String result = outputStream.toString(); + return outputStream.toByteArray(); - // Si sortie texte, on peut découper par ligne et retourner une liste de String - if (isTextOutput) { - return Arrays.asList(result.split("\n")); - } else { - // Pour XML, on retourne une seule chaîne sous forme de liste (vous pouvez adapter selon vos besoins) - return Collections.singletonList(result); - } +// // Si sortie texte, on peut découper par ligne et retourner une liste de String +// if (isTextOutput) { +// return Arrays.asList(result.split("\n")); +// } else { +// // Pour XML, on retourne une seule chaîne sous forme de liste (vous pouvez adapter selon vos besoins) +// return Collections.singletonList(result); +// } } catch (SaxonApiException e) { throw new XsltTransformationException("Error during XSLT transformation", e); - } catch (IOException e) { - throw new IOException("I/O error during XSLT transformation", e); } } + + public String transformToXmlString(InputStream inputStream, String xslFileName)throws XsltTransformationException, IOException { + return toXmlString(transformToXml(inputStream, xslFileName)); + } + + public byte[] transformToXml(InputStream inputStream, String xslFileName) throws IOException { + return transform(inputStream, xslFileName, SerializerConfigurer.forXmlString(processor)); + } + + private String toXmlString(byte[] result) { + return new String(result); + } + + public byte[] transformToRawText(InputStream stream, String ddi2VtlXsl) throws IOException { + return transform(stream, ddi2VtlXsl, SerializerConfigurer.forRawText(processor)); + } } diff --git a/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDIDerefencer.java b/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDIDerefencer.java new file mode 100644 index 00000000..345049e5 --- /dev/null +++ b/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDIDerefencer.java @@ -0,0 +1,21 @@ +package fr.insee.rmes.transfoxsl.service.internal; + +import fr.insee.rmes.transfoxsl.service.XsltTransformationService; + +import java.io.IOException; +import java.io.InputStream; + +public record DDIDerefencer(XsltTransformationService xsltTransformationService) { + + public static final String DEREFERENCE_XSL = "dereference.xsl"; + + public String dereferenceToString(InputStream inputStream) throws IOException { + // Première transformation - XML en sortie + return xsltTransformationService.transformToXmlString(inputStream, DEREFERENCE_XSL); + } + + public byte[] intermediateDereference(InputStream inputStream) throws IOException { + return xsltTransformationService.transformToXml(inputStream, DEREFERENCE_XSL); + } + +} diff --git a/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDITransformerToVtl.java b/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDITransformerToVtl.java new file mode 100644 index 00000000..457844d4 --- /dev/null +++ b/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDITransformerToVtl.java @@ -0,0 +1,24 @@ +package fr.insee.rmes.transfoxsl.service.internal; + +import fr.insee.rmes.transfoxsl.service.XsltTransformationService; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +public record DDITransformerToVtl(XsltTransformationService xsltTransformationService, DDIDerefencer ddiDerefencer) { + + public static final String DDI_2_VTL_XSL = "ddi2vtl.xsl"; + + public DDITransformerToVtl(XsltTransformationService xsltTransformationService){ + this(xsltTransformationService, new DDIDerefencer(xsltTransformationService)); + } + + public byte[] transform(InputStream inputStream) throws IOException { + + + // Deuxième transformation - Texte en sortie (on récupère directement une liste de lignes de texte) + return xsltTransformationService.transformToRawText(ddiDerefencer.intermediateDereference(inputStream), DDI_2_VTL_XSL); + + } +} From 9cb31a91c9a68b097ea06f94e23be1f6397d8a73 Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Tue, 24 Sep 2024 18:19:03 +0200 Subject: [PATCH 04/25] refactor (mutulualize, separate concerns, remove useless, remove redondant, remove elastic search) WIP - ColecticaServiceImpl, line 177 --- pom.xml | 35 +- .../java/fr/insee/rmes/RmesApplication.java | 3 - .../insee/rmes/config/ApplicationContext.java | 90 ---- .../rmes/config/OpenApiConfiguration.java | 33 ++ .../exceptions/RmesBadRequestException.java | 16 +- .../RmesNotAcceptableException.java | 10 +- .../exceptions/RmesNotFoundException.java | 9 +- .../exceptions/RmesUnauthorizedException.java | 18 +- .../tocolecticaapi/controller/DeleteItem.java | 5 +- .../service/ColecticaServiceImpl.java | 28 +- .../service/VarBookExportBuilder.java | 8 +- .../rmes/utils/DocumentBuilderUtils.java | 5 - .../java/fr/insee/rmes/utils/ExportUtils.java | 81 ++-- .../fr/insee/rmes/utils/FileExtension.java | 42 ++ .../java/fr/insee/rmes/utils/FilesUtils.java | 46 +- .../java/fr/insee/rmes/utils/HttpUtils.java | 6 +- .../java/fr/insee/rmes/utils/XsltUtils.java | 15 +- .../rmes/utils/ddi/DDIDocumentBuilder.java | 456 ------------------ .../insee/rmes/utils/export/XDocReport.java | 4 +- 19 files changed, 169 insertions(+), 741 deletions(-) delete mode 100644 src/main/java/fr/insee/rmes/config/ApplicationContext.java create mode 100644 src/main/java/fr/insee/rmes/config/OpenApiConfiguration.java create mode 100644 src/main/java/fr/insee/rmes/utils/FileExtension.java delete mode 100644 src/main/java/fr/insee/rmes/utils/ddi/DDIDocumentBuilder.java diff --git a/pom.xml b/pom.xml index bb5047bb..8790694e 100644 --- a/pom.xml +++ b/pom.xml @@ -17,11 +17,8 @@ 21 UTF-8 - 21 - 2.4.0 - 6.2.2 + 2.6.0 2.0.6 - 1.17 12.4 1.1.1 2.17.0 @@ -35,26 +32,6 @@ java-jwt 4.4.0 - - org.zeroturnaround - zt-zip - ${zt.version} - jar - - - - com.google.guava - guava - 32.0.0-android - - - org.json - json - - - - - org.apache.commons commons-lang3 @@ -146,16 +123,6 @@ - - org.apache.commons - commons-fileupload2-jakarta-servlet6 - 2.0.0-M2 - - - co.elastic.clients - elasticsearch-java - 8.14.1 - jakarta.json jakarta.json-api diff --git a/src/main/java/fr/insee/rmes/RmesApplication.java b/src/main/java/fr/insee/rmes/RmesApplication.java index 571abda7..0a830597 100644 --- a/src/main/java/fr/insee/rmes/RmesApplication.java +++ b/src/main/java/fr/insee/rmes/RmesApplication.java @@ -1,12 +1,9 @@ package fr.insee.rmes; import fr.insee.rmes.config.PropertiesLogger; -import io.swagger.v3.oas.annotations.OpenAPIDefinition; -import io.swagger.v3.oas.annotations.info.Info; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; -@OpenAPIDefinition(info = @Info(title = "DDI-AS", version = "${fr.insee.rmes.ddias.version}")) @SpringBootApplication public class RmesApplication { diff --git a/src/main/java/fr/insee/rmes/config/ApplicationContext.java b/src/main/java/fr/insee/rmes/config/ApplicationContext.java deleted file mode 100644 index 66275ec0..00000000 --- a/src/main/java/fr/insee/rmes/config/ApplicationContext.java +++ /dev/null @@ -1,90 +0,0 @@ -package fr.insee.rmes.config; - -import io.swagger.v3.oas.models.OpenAPI; -import io.swagger.v3.oas.models.security.SecurityRequirement; -import io.swagger.v3.oas.models.security.SecurityScheme; -import io.swagger.v3.oas.models.servers.Server; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.ssl.SSLContexts; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; -import org.springframework.web.client.RestTemplate; - -import javax.net.ssl.SSLContext; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.X509Certificate; -import java.util.List; - -@Configuration -public class ApplicationContext { - - @Value("${fr.insee.rmes.api.host}") - private String serverHost; - - @Autowired - private final Environment environment; - - public ApplicationContext(Environment environment) { - this.environment = environment; - } - - @Bean - public RestTemplate restTemplate(RestTemplateBuilder builder) { - - return builder.build(); - } - - @Bean - public OpenAPI customOpenAPI() { - String serverUrl = "http://"; // Par défaut, utilise HTTP - if (isProductionProfileActive()) { - serverUrl = "https://"; // Si le profil de production est actif, utilise HTTPS - } - - return new OpenAPI().servers(List.of( - new Server().url(serverUrl + serverHost).description("Server"))) - .components(new io.swagger.v3.oas.models.Components() - .addSecuritySchemes("bearerAuth", - new SecurityScheme() - .type(SecurityScheme.Type.HTTP) - .scheme("bearer") - .bearerFormat("JWT"))) - .addSecurityItem(new SecurityRequirement().addList("bearerAuth")); - } - - private boolean isProductionProfileActive() { - String[] activeProfiles = environment.getActiveProfiles(); - for (String profile : activeProfiles) { - if (profile.equalsIgnoreCase("prod") || profile.equalsIgnoreCase("production")) { - return true; - } - } - return false; - } - - @Bean - public CloseableHttpClient httpClient() throws Exception { - return createAcceptSelfSignedCertificateClient(); - } - public CloseableHttpClient createAcceptSelfSignedCertificateClient() - throws NoSuchAlgorithmException, KeyManagementException, KeyStoreException { - TrustStrategy acceptingTrustStrategy = (X509Certificate[] chain, String authType) -> true; - SSLContext sslContext = SSLContexts.custom() - .loadTrustMaterial(null, acceptingTrustStrategy) - .build(); - SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext); - - return HttpClients.custom() - .setSSLSocketFactory(csf) - .build(); - } -} diff --git a/src/main/java/fr/insee/rmes/config/OpenApiConfiguration.java b/src/main/java/fr/insee/rmes/config/OpenApiConfiguration.java new file mode 100644 index 00000000..5839531d --- /dev/null +++ b/src/main/java/fr/insee/rmes/config/OpenApiConfiguration.java @@ -0,0 +1,33 @@ +package fr.insee.rmes.config; + +import io.swagger.v3.oas.annotations.enums.SecuritySchemeType; +import io.swagger.v3.oas.annotations.security.SecurityScheme; +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.info.Info; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +@SecurityScheme(//add Authorize button in swagger + name = "bearerAuth", + type = SecuritySchemeType.HTTP, + bearerFormat = "JWT", + scheme = "bearer" +) +public class OpenApiConfiguration { + + public static final String DESCRIPTION="Back office de Bauhaus (rmesgncs)"; + public static final String TITLE="DDI-AS"; + + @Bean + public OpenAPI openAPI(@Value("${fr.insee.rmes.ddias.version}") String appVersion) { + return new OpenAPI() + .info(new Info() + .title(TITLE) + .version(appVersion) + .description(DESCRIPTION) + ); + } + +} diff --git a/src/main/java/fr/insee/rmes/exceptions/RmesBadRequestException.java b/src/main/java/fr/insee/rmes/exceptions/RmesBadRequestException.java index 80e6dd3c..894c3be8 100644 --- a/src/main/java/fr/insee/rmes/exceptions/RmesBadRequestException.java +++ b/src/main/java/fr/insee/rmes/exceptions/RmesBadRequestException.java @@ -1,36 +1,36 @@ package fr.insee.rmes.exceptions; -import org.apache.http.HttpStatus; import org.json.JSONArray; import org.json.JSONObject; +import org.springframework.http.HttpStatus; public class RmesBadRequestException extends RmesException { private static final long serialVersionUID = 400L; public RmesBadRequestException(String message) { - super(HttpStatus.SC_BAD_REQUEST, message, ""); + super(HttpStatus.BAD_REQUEST, message, ""); } public RmesBadRequestException(String message, String details) { - super(HttpStatus.SC_BAD_REQUEST, message, details); + super(HttpStatus.BAD_REQUEST, message, details); } public RmesBadRequestException(String message, JSONArray details) { - super(HttpStatus.SC_BAD_REQUEST, message, details); + super(HttpStatus.BAD_REQUEST.value(), message, details); } public RmesBadRequestException(int errorCode, String message, String details) { - super(HttpStatus.SC_BAD_REQUEST, errorCode, message, details); + super(HttpStatus.BAD_REQUEST.value(), errorCode, message, details); } public RmesBadRequestException(int errorCode, String message, JSONArray details) { - super(HttpStatus.SC_BAD_REQUEST, errorCode, message, details); + super(HttpStatus.BAD_REQUEST.value(), errorCode, message, details); } public RmesBadRequestException(int errorCode, String message) { - super(HttpStatus.SC_BAD_REQUEST, errorCode, message, ""); + super(HttpStatus.BAD_REQUEST.value(), errorCode, message, ""); } public RmesBadRequestException(int errorCode, String message, JSONObject details) { - super(HttpStatus.SC_FORBIDDEN, errorCode, message, details); } + super(HttpStatus.FORBIDDEN.value(), errorCode, message, details); } } diff --git a/src/main/java/fr/insee/rmes/exceptions/RmesNotAcceptableException.java b/src/main/java/fr/insee/rmes/exceptions/RmesNotAcceptableException.java index 083775fd..adb06e08 100644 --- a/src/main/java/fr/insee/rmes/exceptions/RmesNotAcceptableException.java +++ b/src/main/java/fr/insee/rmes/exceptions/RmesNotAcceptableException.java @@ -1,26 +1,26 @@ package fr.insee.rmes.exceptions; -import org.apache.http.HttpStatus; import org.json.JSONArray; +import org.springframework.http.HttpStatus; public class RmesNotAcceptableException extends RmesException { private static final long serialVersionUID = 2L; public RmesNotAcceptableException(String message, String details) { - super(HttpStatus.SC_NOT_ACCEPTABLE, message, details); + super(HttpStatus.NOT_ACCEPTABLE, message, details); } public RmesNotAcceptableException(String message, JSONArray details) { - super(HttpStatus.SC_NOT_ACCEPTABLE, message, details); + super(HttpStatus.NOT_ACCEPTABLE.value(), message, details); } public RmesNotAcceptableException(int errorCode, String message, String details) { - super(HttpStatus.SC_NOT_ACCEPTABLE, errorCode, message, details); + super(HttpStatus.NOT_ACCEPTABLE.value(), errorCode, message, details); } public RmesNotAcceptableException(int errorCode, String message, JSONArray details) { - super(HttpStatus.SC_NOT_ACCEPTABLE, errorCode, message, details); + super(HttpStatus.NOT_ACCEPTABLE.value(), errorCode, message, details); } } diff --git a/src/main/java/fr/insee/rmes/exceptions/RmesNotFoundException.java b/src/main/java/fr/insee/rmes/exceptions/RmesNotFoundException.java index e9e1d5c2..8bca692f 100644 --- a/src/main/java/fr/insee/rmes/exceptions/RmesNotFoundException.java +++ b/src/main/java/fr/insee/rmes/exceptions/RmesNotFoundException.java @@ -1,20 +1,21 @@ package fr.insee.rmes.exceptions; -import org.apache.http.HttpStatus; + +import org.springframework.http.HttpStatus; public class RmesNotFoundException extends RmesException { private static final long serialVersionUID = 1L; public RmesNotFoundException(String message, String details) { - super(HttpStatus.SC_NOT_FOUND, message, details); + super(HttpStatus.NOT_FOUND, message, details); } public RmesNotFoundException(int errorCode, String message, String details) { - super(HttpStatus.SC_NOT_FOUND, errorCode + " : " + message, details); + super(HttpStatus.NOT_FOUND, errorCode + " : " + message, details); } public RmesNotFoundException(String message) { - super(HttpStatus.SC_NOT_FOUND, message, "Not found"); + super(HttpStatus.NOT_FOUND, message, "Not found"); } } diff --git a/src/main/java/fr/insee/rmes/exceptions/RmesUnauthorizedException.java b/src/main/java/fr/insee/rmes/exceptions/RmesUnauthorizedException.java index cc74e946..7fa299ae 100644 --- a/src/main/java/fr/insee/rmes/exceptions/RmesUnauthorizedException.java +++ b/src/main/java/fr/insee/rmes/exceptions/RmesUnauthorizedException.java @@ -1,41 +1,41 @@ package fr.insee.rmes.exceptions; -import org.apache.http.HttpStatus; import org.json.JSONArray; import org.json.JSONObject; +import org.springframework.http.HttpStatus; public class RmesUnauthorizedException extends RmesException { private static final long serialVersionUID = 5611172589954490294L; public RmesUnauthorizedException() { - super(HttpStatus.SC_FORBIDDEN, "Unauthorized", ""); + super(HttpStatus.FORBIDDEN, "Unauthorized", ""); } public RmesUnauthorizedException(String message, String details) { - super(HttpStatus.SC_FORBIDDEN, message, details); + super(HttpStatus.FORBIDDEN, message, details); } public RmesUnauthorizedException(String message, JSONArray details) { - super(HttpStatus.SC_FORBIDDEN, message, details); + super(HttpStatus.FORBIDDEN.value(), message, details); } public RmesUnauthorizedException(int errorCode, String details) { - super(HttpStatus.SC_FORBIDDEN, errorCode, details); + super(HttpStatus.FORBIDDEN.value(), errorCode, details); } public RmesUnauthorizedException(int errorCode, JSONArray details) { - super(HttpStatus.SC_FORBIDDEN, errorCode, details); + super(HttpStatus.FORBIDDEN.value(), errorCode, details); } public RmesUnauthorizedException(int errorCode, String message, JSONArray details) { - super(HttpStatus.SC_FORBIDDEN, errorCode, message, details); + super(HttpStatus.FORBIDDEN.value(), errorCode, message, details); } public RmesUnauthorizedException(int errorCode, String message, String details) { - super(HttpStatus.SC_FORBIDDEN, errorCode, message, details); + super(HttpStatus.FORBIDDEN.value(), errorCode, message, details); } public RmesUnauthorizedException(int errorCode, String message, JSONObject details) { - super(HttpStatus.SC_FORBIDDEN, errorCode, message, details); } + super(HttpStatus.FORBIDDEN.value(), errorCode, message, details); } } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/DeleteItem.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/DeleteItem.java index 78702396..b06c54da 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/DeleteItem.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/DeleteItem.java @@ -42,12 +42,9 @@ public class DeleteItem { private final ColecticaService colecticaService; - private final RestTemplate restTemplate; - @Autowired - public DeleteItem(ColecticaService colecticaService, RestTemplate restTemplate) { + public DeleteItem(ColecticaService colecticaService) { this.colecticaService = colecticaService; - this.restTemplate = restTemplate; } @DeleteMapping(value = "/deleteCodeList") diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java index 35401ba6..c1115045 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java @@ -24,16 +24,6 @@ import net.sf.saxon.TransformerFactoryImpl; import net.sf.saxon.s9api.ExtensionFunction; import net.sf.saxon.s9api.Processor; -import org.apache.http.client.config.CookieSpecs; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -45,7 +35,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.ByteArrayResource; import org.springframework.core.io.Resource; -import org.springframework.http.*; +import org.springframework.http.ResponseEntity; import org.springframework.http.client.SimpleClientHttpRequestFactory; import org.springframework.stereotype.Service; import org.springframework.web.client.HttpClientErrorException; @@ -77,8 +67,11 @@ import javax.xml.xpath.XPathFactory; import java.io.*; import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.util.*; import static org.springframework.security.oauth2.core.endpoint.OAuth2ParameterNames.ERROR; @@ -146,8 +139,6 @@ public class ColecticaServiceImpl implements ColecticaService { @Autowired private final ExportUtils exportUtils; - private final CloseableHttpClient httpClient; - @Autowired private final XsltTransformationService xsltTransformationService; @@ -164,8 +155,8 @@ public ResponseEntity findFragmentByUuid(String uuid) throws ExceptionCo private ResponseEntity searchColecticaFragmentByUuid(String uuid) throws ExceptionColecticaUnreachable, IOException { - try (CloseableHttpClient httpClient = HttpClients.createDefault()) { - HttpGet httpGet = getGetSearchColecticaFragmentByUuid(uuid); + try (HttpClient httpClient = HttpClient.newHttpClient()) { + HttpRequest getRequest = getGetSearchColecticaFragmentByUuid(uuid); try (CloseableHttpResponse response = httpClient.execute(httpGet)) { return getResponseEntitySearchColecticaFragmentByUuid(response); @@ -180,9 +171,10 @@ private ResponseEntity searchColecticaFragmentByUuid(String uuid) throws } } - private HttpGet getGetSearchColecticaFragmentByUuid(String uuid) throws ExceptionColecticaUnreachable, JsonProcessingException, RmesExceptionIO, ParseException { + private HttpRequest getGetSearchColecticaFragmentByUuid(String uuid) throws ExceptionColecticaUnreachable, JsonProcessingException, RmesExceptionIO, ParseException { String url = String.format("%s/api/v1/item/%s/%s", serviceUrl, agency, uuid); - HttpGet httpGet = new HttpGet(url); + HttpRequest httpGet = HttpRequest.newBuilder().build(); + httpGet.headers() httpGet.addHeader(CONTENT_TYPE, APPLICATION_XML); httpGet.addHeader("Accept", APPLICATION_JSON); @@ -1465,7 +1457,7 @@ public ResponseEntity getCodeBookExportV2(String ddi, String xslPatter HashMap contentXML = new HashMap<>(); contentXML.put("ddi-file", Files.readString(ddiRemoveNameSpaces.toPath())); - return exportUtils.exportAsODT("export.odt", contentXML, dicoCode, xslPatternFile, zipRmes, "dicoVariable"); + return exportUtils.exportAsODT(Path.of("export.odt"), contentXML, dicoCode, xslPatternFile, zipRmes, "dicoVariable"); } public static void transformerStringWithXsl(String ddi,InputStream xslRemoveNameSpaces, File output) throws Exception{ diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/VarBookExportBuilder.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/VarBookExportBuilder.java index 59739d2a..8fa14951 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/VarBookExportBuilder.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/VarBookExportBuilder.java @@ -5,9 +5,9 @@ import fr.insee.rmes.utils.DocumentBuilders; import fr.insee.rmes.utils.StringUtils; import fr.insee.rmes.utils.XMLUtils; -import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -42,7 +42,7 @@ public String getData(String xml) throws RmesException { try { return XMLUtils.toString(xmlReadyToExport); } catch (TransformerFactoryConfigurationError | TransformerException e) { - throw new RmesException(HttpStatus.SC_BAD_REQUEST, e.getMessage(), + throw new RmesException(HttpStatus.BAD_REQUEST, e.getMessage(), "IOException - Can't convert xml to text"); } } @@ -68,7 +68,7 @@ private Document dereferenceAll(Document xmlInput) throws RmesException { xmlOutput = builder.newDocument(); } catch (ParserConfigurationException e) { - throw new RmesException(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage(), + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage(), "ParserConfigurationException"); } @@ -195,7 +195,7 @@ private static Document getDocument(String xml) throws RmesException { db = DocumentBuilders.createSaferDocumentBuilder(DocumentBuilderFactory::isIgnoringElementContentWhitespace); xmlInitial = db.parse(stream); } catch (ParserConfigurationException | SAXException | IOException e) { - throw new RmesException(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage(), + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage(), e.getClass() + " Can't parse xml"); } return xmlInitial; diff --git a/src/main/java/fr/insee/rmes/utils/DocumentBuilderUtils.java b/src/main/java/fr/insee/rmes/utils/DocumentBuilderUtils.java index 15a08a9a..11715943 100644 --- a/src/main/java/fr/insee/rmes/utils/DocumentBuilderUtils.java +++ b/src/main/java/fr/insee/rmes/utils/DocumentBuilderUtils.java @@ -1,6 +1,5 @@ package fr.insee.rmes.utils; -import fr.insee.rmes.utils.ddi.DDIDocumentBuilder; import org.springframework.stereotype.Service; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -43,8 +42,4 @@ public static Node getNode(String fragment, Document doc) throws Exception { doc.adoptNode(newNode); return newNode; } - - public static Node getNode(String fragment, DDIDocumentBuilder doc) throws Exception { - return getNode(fragment, doc.getDocument()); - } } diff --git a/src/main/java/fr/insee/rmes/utils/ExportUtils.java b/src/main/java/fr/insee/rmes/utils/ExportUtils.java index b7d641a6..5a9b73d3 100644 --- a/src/main/java/fr/insee/rmes/utils/ExportUtils.java +++ b/src/main/java/fr/insee/rmes/utils/ExportUtils.java @@ -2,7 +2,6 @@ import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.tocolecticaapi.service.Constants; -import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,7 +15,10 @@ import org.springframework.stereotype.Component; import javax.xml.transform.TransformerException; -import java.io.*; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -34,23 +36,23 @@ public ExportUtils(@Value("${fr.insee.rmes.ddias.filenames.maxlength}") int maxL this.maxLength = maxLength; } - public ResponseEntity exportAsODT(String fileName, Map xmlContent, String xslFile, String xmlPattern, String zip, String objectType) throws RmesException { - return exportAsFileByExtension(fileName, xmlContent, xslFile, xmlPattern, zip, objectType, FilesUtils.ODT_EXTENSION); + public ResponseEntity exportAsODT(Path fileName, Map xmlContent, String xslFile, String xmlPattern, String zip, String objectType) throws RmesException { + return exportAsFileByExtension(fileName, xmlContent, xslFile, xmlPattern, zip, objectType, FileExtension.ODT_EXTENSION); } - public ResponseEntity exportAsODS(String fileName, Map xmlContent, String xslFile, String xmlPattern, String zip, String objectType) throws RmesException { - return exportAsFileByExtension(fileName, xmlContent, xslFile, xmlPattern, zip, objectType, FilesUtils.ODS_EXTENSION); + public ResponseEntity exportAsODS(Path fileName, Map xmlContent, String xslFile, String xmlPattern, String zip, String objectType) throws RmesException { + return exportAsFileByExtension(fileName, xmlContent, xslFile, xmlPattern, zip, objectType, FileExtension.ODS_EXTENSION); } - private ResponseEntity exportAsFileByExtension(String fileName, Map xmlContent, String xslFile, String xmlPattern, String zip, String objectType, String extension) throws RmesException { + private ResponseEntity exportAsFileByExtension(Path fileName, Map xmlContent, String xslFile, String xmlPattern, String zip, String objectType, FileExtension extension) throws RmesException { logger.debug("Begin To export {} as Response", objectType); - fileName = fileName.replace(extension, ""); - InputStream input = exportAsInputStream(fileName, xmlContent, xslFile, xmlPattern, zip, objectType, extension); + + InputStream input = exportAsInputStream(fileName, xmlContent, xslFile, xmlPattern, zip, objectType); if (input == null) throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, CAN_T_GENERATE_CODEBOOK, NULL_STREAM); - ByteArrayResource resource = null; + ByteArrayResource resource; try { resource = new ByteArrayResource(IOUtils.toByteArray(input)); input.close(); @@ -60,7 +62,7 @@ private ResponseEntity exportAsFileByExtension(String fileName, Map exportAsFileByExtension(String fileName, Map xmlContent, String xslFile, String xmlPattern, String zip, String objectType, String extension) throws RmesException { + public InputStream exportAsInputStream(Path fileName, Map xmlContent, String xslFile, String xmlPattern, String zip, String objectType) throws RmesException { logger.debug("Begin To export {} as InputStream", objectType); - File output = null; - InputStream odtFileIS = null; - InputStream xslFileIS = null; - InputStream zipToCompleteIS = null; - fileName = fileName.replace(extension, ""); //Remove extension if exists - try { - xslFileIS = getClass().getResourceAsStream(xslFile); - odtFileIS = getClass().getResourceAsStream(xmlPattern); - zipToCompleteIS = getClass().getResourceAsStream(zip); + Path outputPath = Files.createTempFile(Constants.OUTPUT, FileExtension.XML_EXTENSION.extension()); - // prepare output - output = File.createTempFile(Constants.OUTPUT, FilesUtils.getExtension(Constants.XML)); - output.deleteOnExit(); + try ( + InputStream xslFileIS = getClass().getResourceAsStream(xslFile); + InputStream odtFileIS = getClass().getResourceAsStream(xmlPattern); + InputStream zipToCompleteIS = getClass().getResourceAsStream(zip); + OutputStream osOutputFile = Files.newOutputStream(outputPath); + PrintStream printStream = new PrintStream(osOutputFile)) { - } catch (IOException ioe) { - logger.error(ioe.getMessage()); - } + outputPath.toFile().deleteOnExit(); - try (OutputStream osOutputFile = FileUtils.openOutputStream(output); - PrintStream printStream = new PrintStream(osOutputFile);) { - Path tempDir = Files.createTempDirectory("forExport"); - Path finalPath = Paths.get(tempDir.toString(), fileName + extension); + Path tempDir = Files.createTempDirectory("forExport"); + Path finalPath = Paths.get(tempDir.toString()).resolve(fileName); - // transform - XsltUtils.xsltTransform(xmlContent, odtFileIS, xslFileIS, printStream, tempDir); - // create odt - XsltUtils.createOdtFromXml(output, finalPath, zipToCompleteIS, tempDir); + // transform + XsltUtils.xsltTransform(xmlContent, odtFileIS, xslFileIS, printStream, tempDir); + // create odt + XsltUtils.createOdtFromXml(outputPath, finalPath, zipToCompleteIS, tempDir); - logger.debug("End To export {} as InputStream", objectType); + logger.debug("End To export {} as InputStream", objectType); - return Files.newInputStream(finalPath); + return Files.newInputStream(finalPath); + } } catch (IOException | TransformerException e) { throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage(), e.getClass().getSimpleName()); - } finally { - try { - if (odtFileIS != null) - odtFileIS.close(); - if (xslFileIS != null) - xslFileIS.close(); - if (zipToCompleteIS != null) - zipToCompleteIS.close(); - } catch (IOException ioe) { - logger.error(ioe.getMessage()); - } } } - - } diff --git a/src/main/java/fr/insee/rmes/utils/FileExtension.java b/src/main/java/fr/insee/rmes/utils/FileExtension.java new file mode 100644 index 00000000..d88d7612 --- /dev/null +++ b/src/main/java/fr/insee/rmes/utils/FileExtension.java @@ -0,0 +1,42 @@ +package fr.insee.rmes.utils; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import org.springframework.http.MediaType; + +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public enum FileExtension { + + ODT_EXTENSION(".odt", new MediaType("application", "vnd.oasis.opendocument.text")), + ODS_EXTENSION(".ods", new MediaType("application", "vnd.oasis.opendocument.spreadsheet")), + ZIP_EXTENSION(".zip", new MediaType("application", "zip")), + PDF_EXTENSION(".pdf", null), + XML_EXTENSION(".xml", null), + FODT_EXTENSION(".fodt", null); + + private final String extension; + private final MediaType mediaType; + + public MediaType getMediaTypeFromExtension() { + if (mediaType == null) { + throw new IllegalStateException("Unexpected value: " + extension); + } + return mediaType; + } + + public static FileExtension get(String acceptHeader) { + return switch (acceptHeader) { + case "application/octet-stream" -> PDF_EXTENSION; + case "flatODT" -> FODT_EXTENSION; + case "XML" -> XML_EXTENSION; + // also "application/vnd.oasis.opendocument.text" -> ODT_EXTENSION + case null, default -> ODT_EXTENSION; + }; + } + + public String extension() { + return extension; + } + +} + diff --git a/src/main/java/fr/insee/rmes/utils/FilesUtils.java b/src/main/java/fr/insee/rmes/utils/FilesUtils.java index 4655ab9b..6aaaa10c 100644 --- a/src/main/java/fr/insee/rmes/utils/FilesUtils.java +++ b/src/main/java/fr/insee/rmes/utils/FilesUtils.java @@ -4,12 +4,12 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.http.MediaType; -import org.zeroturnaround.zip.FileSource; -import org.zeroturnaround.zip.ZipEntrySource; -import org.zeroturnaround.zip.ZipUtil; import java.io.*; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; import java.text.Normalizer; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -17,38 +17,7 @@ public class FilesUtils { private static final Logger log = LoggerFactory.getLogger(FilesUtils.class); - public static final String ODT_EXTENSION = ".odt"; - public static final String ODS_EXTENSION = ".ods"; - public static final String ZIP_EXTENSION = ".zip"; - public static final String PDF_EXTENSION = ".pdf"; - public static final String XML_EXTENSION = ".xml"; - public static final String FODT_EXTENSION = ".fodt"; - public static MediaType getMediaTypeFromExtension(String extension) { - MediaType contentType = switch (extension){ - case FilesUtils.ODT_EXTENSION -> new MediaType("application", "vnd.oasis.opendocument.text"); - case FilesUtils.ODS_EXTENSION -> new MediaType("application", "vnd.oasis.opendocument.spreadsheet"); - case FilesUtils.ZIP_EXTENSION -> new MediaType("application", "zip"); - default -> throw new IllegalStateException("Unexpected value: " + extension); - }; - return contentType; - } - - public static String getExtension(String acceptHeader) { - if (acceptHeader == null) { - return FilesUtils.ODT_EXTENSION; - } else if (acceptHeader.equals("application/octet-stream")) { - return PDF_EXTENSION; - } else if (acceptHeader.equals("flatODT")) { - return FODT_EXTENSION; - } else if (acceptHeader.equals("XML")) { - return XML_EXTENSION; - } else if (acceptHeader.equals("application/vnd.oasis.opendocument.text")) { - return ODT_EXTENSION; - } else { - return ODT_EXTENSION; - } - } public static String reduceFileNameSize(String fileName, int maxLength) { return fileName.substring(0, Math.min(fileName.length(), maxLength)); @@ -77,9 +46,10 @@ public static String cleanFileNameAndAddExtension(String fileName, String extens return fileName; } - public static void addFileToZipFolder(File fileToAdd, File zipArchive) { - ZipEntrySource entry = new FileSource(fileToAdd.getName(), fileToAdd); - ZipUtil.addEntry(zipArchive, entry); + public static void addFileToZipFolder(Path fileToAdd, Path zipArchive) throws IOException { + try(FileSystem fs = FileSystems.newFileSystem(zipArchive)){ + Files.write(fs.getPath(fileToAdd.getFileName().toString()), Files.readAllBytes(fileToAdd)); + } } private FilesUtils() { diff --git a/src/main/java/fr/insee/rmes/utils/HttpUtils.java b/src/main/java/fr/insee/rmes/utils/HttpUtils.java index 71c6bdb4..72b49d85 100644 --- a/src/main/java/fr/insee/rmes/utils/HttpUtils.java +++ b/src/main/java/fr/insee/rmes/utils/HttpUtils.java @@ -13,12 +13,12 @@ public final class HttpUtils { private HttpUtils() { } - public static HttpHeaders generateHttpHeaders(String fileName, String extension, int maxLength) { - MediaType contentType = FilesUtils.getMediaTypeFromExtension(extension); + public static HttpHeaders generateHttpHeaders(String fileName, FileExtension extension, int maxLength) { + MediaType contentType = extension.getMediaTypeFromExtension(); ContentDisposition content = ContentDisposition.builder(HttpUtils.ATTACHMENT).filename( FilesUtils.reduceFileNameSize( - FilesUtils.removeAsciiCharacters(fileName), maxLength) + extension + FilesUtils.removeAsciiCharacters(fileName), maxLength) ).build(); List allowHeaders = List.of(CONTENT_DISPOSITION, diff --git a/src/main/java/fr/insee/rmes/utils/XsltUtils.java b/src/main/java/fr/insee/rmes/utils/XsltUtils.java index 4b581399..d02badd5 100644 --- a/src/main/java/fr/insee/rmes/utils/XsltUtils.java +++ b/src/main/java/fr/insee/rmes/utils/XsltUtils.java @@ -1,17 +1,16 @@ package fr.insee.rmes.utils; -import fr.insee.rmes.tocolecticaapi.service.Constants; import fr.insee.rmes.exceptions.RmesException; +import fr.insee.rmes.tocolecticaapi.service.Constants; import org.apache.commons.io.IOUtils; -import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; @@ -51,7 +50,7 @@ public static void xsltTransform(Map xmlContent, InputStream odt private static void addParameter(Transformer xsltTransformer, String paramName, String paramData, Path tempDir) throws RmesException { CopyOption[] options = { StandardCopyOption.REPLACE_EXISTING }; try { - Path tempFile = Files.createTempFile(tempDir, paramName, FilesUtils.XML_EXTENSION); + Path tempFile = Files.createTempFile(tempDir, paramName, FileExtension.XML_EXTENSION.extension()); String absolutePath = tempFile.toFile().getAbsolutePath(); InputStream is = IOUtils.toInputStream(paramData, StandardCharsets.UTF_8); Files.copy(is, tempFile, options); @@ -61,17 +60,17 @@ private static void addParameter(Transformer xsltTransformer, String paramName, logger.debug("Setting XSLT parameter '{}' to '{}'", paramName, fileUri); xsltTransformer.setParameter(paramName, fileUri); } catch (IOException e) { - throw new RmesException(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage(), "IOException - Can't create temp files for XSLT Transformer"); + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage(), "IOException - Can't create temp files for XSLT Transformer"); } } - public static void createOdtFromXml(File output, Path finalPath, InputStream zipToCompleteIS, Path tempDir) + public static void createOdtFromXml(Path outputPath, Path finalPath, InputStream zipToCompleteIS, Path tempDir) throws IOException { Path contentPath = Paths.get(tempDir.toString() + "/content.xml"); - Files.copy(Paths.get(output.getAbsolutePath()), contentPath, StandardCopyOption.REPLACE_EXISTING); + Files.copy(outputPath, contentPath, StandardCopyOption.REPLACE_EXISTING); Path zipPath = Paths.get(tempDir.toString() + "/export.zip"); Files.copy(zipToCompleteIS, zipPath, StandardCopyOption.REPLACE_EXISTING); - FilesUtils.addFileToZipFolder(contentPath.toFile(), zipPath.toFile()); + FilesUtils.addFileToZipFolder(contentPath, zipPath); Files.copy(zipPath, finalPath, StandardCopyOption.REPLACE_EXISTING); } diff --git a/src/main/java/fr/insee/rmes/utils/ddi/DDIDocumentBuilder.java b/src/main/java/fr/insee/rmes/utils/ddi/DDIDocumentBuilder.java deleted file mode 100644 index 675b1187..00000000 --- a/src/main/java/fr/insee/rmes/utils/ddi/DDIDocumentBuilder.java +++ /dev/null @@ -1,456 +0,0 @@ -package fr.insee.rmes.utils.ddi; - -import com.google.common.io.Resources; -import fr.insee.rmes.utils.DocumentBuilderUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import javax.xml.xpath.XPathExpressionException; -import java.io.File; -import java.io.StringWriter; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -@Slf4j -public class DDIDocumentBuilder { - - private final static String RESOURCE_PACKAGE = "g:ResourcePackage"; - private Boolean envelope; - private String nameEnvelope = Envelope.DEFAULT.toString(); - private Node itemNode; - private Node resourcePackageNode; - private Document packagedDocument; - - public DDIDocumentBuilder() { - this.envelope = true; - try { - packagedDocument = buildEnvelope(); - } catch (Exception e) { - log.error(e.getMessage()); - } - } - - public void setEnvelope(Boolean envelope) { - this.envelope = envelope; - } - - public DDIDocumentBuilder(Boolean envelope, Enum envelopeName) { - this.nameEnvelope = envelopeName.toString(); - this.envelope = envelope; - if (envelope) { - try { - packagedDocument = buildEnvelope(); - } catch (Exception e) { - log.error(e.getMessage()); - } - } else { - try { - packagedDocument = buildWithoutEnvelope(); - } catch (Exception e) { - log.error(e.getMessage()); - } - } - } - - public DDIDocumentBuilder build() { - if (envelope) { - if (null != itemNode) { - if (this.nameEnvelope.equals(Envelope.DEFAULT.toString())) { - packagedDocument.getDocumentElement().appendChild(itemNode); - } else { - appendChildByParent(RESOURCE_PACKAGE, itemNode); - refactor(itemNode, packagedDocument); - } - } - if (null != resourcePackageNode) { - packagedDocument.getDocumentElement().appendChild(resourcePackageNode); - } - } else { - if (null != itemNode) { - packagedDocument.appendChild(itemNode); - } - if (null != resourcePackageNode) { - packagedDocument.appendChild(resourcePackageNode); - } - } - return this; - } - - /** - * Build a DDIDocument with a specific node in addition to the main node. - * - * @param node - * : node to insert - * @param nameParent - * : name of the parent ex:CodeListScheme to insert CodeLists - * @return DDIDocumentBuilder ddiDocument - */ - public DDIDocumentBuilder buildWithCustomNode(Node node, String nameParent) { - if (envelope) { - if (null != itemNode) { - // packagedDocument.getDocumentElement().appendChild(itemNode); - appendChildByParent(RESOURCE_PACKAGE, itemNode); - importChildByParent(nameParent, node); - refactor(itemNode, packagedDocument); - } - if (null != resourcePackageNode) { - packagedDocument.getDocumentElement().appendChild(resourcePackageNode); - refactor(resourcePackageNode, packagedDocument); - } - } else { - if (null != itemNode) { - packagedDocument.appendChild(itemNode); - } - if (null != resourcePackageNode) { - packagedDocument.appendChild(resourcePackageNode); - } - } - return this; - } - - /** - * Build a DDIDocument with specific nodes in addition to the main node. - * - * @param Map - * nodesWithParentNames - * @return DDIDocumentBuilder ddiDocument - */ - public DDIDocumentBuilder buildWithCustomNodes(TreeMap> nodesWithParentNames) { - if (envelope) { - if (null != itemNode) { - // packagedDocument.getDocumentElement().appendChild(itemNode); - appendChildByParent(RESOURCE_PACKAGE, itemNode); - refactor(itemNode, packagedDocument); - } - for (Integer key : nodesWithParentNames.keySet()) { - Map map = nodesWithParentNames.get(key); - for (Node node : map.keySet()) { - importChildByParent(map.get(node), node); - } - } - } - - if (null != resourcePackageNode) { - packagedDocument.getDocumentElement().appendChild(resourcePackageNode); - } else { - if (null != itemNode) { - // packagedDocument.appendChild(itemNode); - } - } - return this; - } - - /** - * Build a DDIDocument with specific nodes in addition to the - * ressourcePackageNode. - * - * @param Map - * nodesWithParentNames - * @return DDIDocumentBuilder ddiDocument - */ - public DDIDocumentBuilder buildRessourcePackageWithCustomNodes( - TreeMap> nodesWithParentNames) { - if (envelope) { - if (null != itemNode) { - if (this.nameEnvelope.equals(Envelope.DEFAULT.toString())) { - packagedDocument.getDocumentElement().appendChild(itemNode); - } - for (Integer key : nodesWithParentNames.keySet()) { - Map map = nodesWithParentNames.get(key); - for (Node node : map.keySet()) { - if (key.equals(1)) { - importChild(node); - } else { - importChildByParent(map.get(node), node); - } - } - } - - } - refactor(itemNode, packagedDocument); - } - - if (null != resourcePackageNode) { - packagedDocument.getDocumentElement().appendChild(resourcePackageNode); - } else { - if (null != itemNode) { - // packagedDocument.appendChild(itemNode); - } - } - return this; - } - - /** - * Method of refactoring to fix prefix of nodes - * - * @param node - * Node to fix (name and namespace) - * @param document - * DDIDocument - */ - public void refactor(Node node, Document document) { - - switch (node.getNodeName()) { - case "CodeList": - changeTagName(document, "CodeList", "l:CodeList", ""); - break; - case "Code": - changeTagName(document, "Code", "l:Code", ""); - break; - case "Category": - changeTagName(document, "Category", "l:Category", ""); - break; - case "CategoryScheme": - changeTagName(document, "CategoryScheme", "l:CategoryScheme", ""); - break; - default: - break; - } - } - - /** - * Method of refactoring to fix prefix of nodes - * - * @param List - * Nodes to fix (name and namespace) - * @param document - * DDIDocument - */ - public void refactor(List nodes, Document document) { - - for (Node node : nodes) { - refactor(node, document); - } - } - - /** - * Method which change the name of a node for a specific document - * - * @param doc - * : document concerned - * @param fromTag - * : initial Tag ---> target - * @param toTag - * : New tag - * @param namespace - * : New nameSpace - */ - public void changeTagName(Document doc, String fromTag, String toTag, String namespace) { - NodeList nodes = doc.getElementsByTagName(fromTag); - for (int i = 0; i < nodes.getLength(); i++) { - if (nodes.item(i) instanceof Element) { - Element elem = (Element) nodes.item(i); - doc.renameNode(elem, namespace, toTag); - } - } - } - - /** - * Method of adding the itemNode to the DDIDocument (appendChild) - * - * @param childNode - * : node to append - */ - public void appendChild(Node childNode) { - packagedDocument.getDocumentElement().appendChild(childNode); - } - - /** - * Method of adding the itemNode to the DDIDocument (appendChild) - * - * @param parentName - * : Name of the XML Parent - * @param childNode - * : node to append - */ - public void appendChildByParent(String parentName, Node childNode) { - NodeList nodeList = packagedDocument.getElementsByTagName(parentName); - - for (int i = 0; i < nodeList.getLength(); i++) { - Node node = nodeList.item(i); - if (node.getNodeName().equals(parentName)) { - try { - node.appendChild(childNode); - } catch (Exception e) { - node.appendChild(childNode); - } - - } - - } - - } - - public void importChildByParent(String parentName, Node childNode) { - NodeList nodeList = packagedDocument.getElementsByTagName(parentName); - Node node = nodeList.item(0); - if (node.getNodeName().equals(parentName)) { - Node clonedNode = childNode.cloneNode(true); - node.appendChild(packagedDocument.adoptNode(clonedNode)); - - refactor(clonedNode, packagedDocument); - } - } - - public NodeList getElementByTagName(String name) { - return packagedDocument.getElementsByTagName(name); - } - - public void importChild(Node childNode) { - Node node; - Node clonedNode ; - node = packagedDocument.getLastChild(); - clonedNode = childNode.cloneNode(true); - node.appendChild(packagedDocument.adoptNode(clonedNode)); - refactor(clonedNode, packagedDocument); - } - - /** - * Method of adding a list of itemNode to the DDIDocument (appendChild) - * - * @param parentName - * : Name of the XML Parent - * @param childNode - * : node to append - */ - public void appendChildrenByParent(String parentName, List childNodes) { - NodeList nodeList = packagedDocument.getDocumentElement().getChildNodes(); - - for (int i = 0; i < nodeList.getLength(); i++) { - Node node = nodeList.item(i); - if (node.getNodeName().equals(parentName)) { - Node nodeListChild = node.getLastChild(); - Node finalNode = nodeListChild.getPreviousSibling(); - for (Node nodeChild : childNodes) { - finalNode.appendChild(nodeChild); - } - } - } - } - - public DDIDocumentBuilder buildItemDocument(String rootId, Map references) throws Exception { - itemNode = buildNode(packagedDocument, rootId, references); - return this; - } - - public DDIDocumentBuilder buildResourcePackageDocument(String rootId, Map references) - throws Exception { - resourcePackageNode = buildNode(packagedDocument, rootId, references); - return this; - } - - public Document getDocument() { - return packagedDocument; - } - - - - @Override - public String toString() { - StringWriter stringWriter = new StringWriter(); - try { - encode(); - Transformer transformer = TransformerFactory.newInstance().newTransformer(); - transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); - transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "4"); - StreamResult streamResult = new StreamResult(stringWriter); - transformer.transform(new DOMSource(packagedDocument), streamResult); - } catch (Exception e) { - log.error(e.getMessage()); - } - return stringWriter.toString(); - } - - private Node buildNode(Document document, String rootId, Map references) throws Exception { - Node node = DocumentBuilderUtils.getNode(references.get(rootId), document); - walk(node, document, references); - return node; - } - - private Document buildEnvelope() throws Exception { - StringBuilder strBuilder = new StringBuilder(); - strBuilder.append("transforms/templates/"); - strBuilder.append(this.nameEnvelope); - URL url = Resources.getResource(strBuilder.toString()); - String fragment = FileUtils.readFileToString(new File(url.toURI()), StandardCharsets.UTF_8.name()); - return DocumentBuilderUtils.getDocument(fragment); - } - - private Document buildWithoutEnvelope() throws Exception { - return DocumentBuilderUtils.getDocument(null); - - } - - private void walk(Node root, Document document, Map references) throws Exception { - NodeList rootNodes = root.getChildNodes(); - for (int i = 0; i < rootNodes.getLength(); i++) { - Node node = rootNodes.item(i); - if (node.getNodeName().contains("Reference")) { - String fragment = references.get(getId(node)); - if (null != fragment) { - Node child = DocumentBuilderUtils.getNode(fragment, document); - root.appendChild(child); - root.removeChild(node); - walk(child, document, references); - } - } else { - walk(node, document, references); - } - } - } - - private static String getId(Node refNode) throws Exception { - NodeList refChildren = refNode.getChildNodes(); - for (int i = 0; i < refChildren.getLength(); i++) { - if (refChildren.item(i).getNodeName().equals("r:ID")) { - log.info(refNode.getNodeName() + " -> " + refChildren.item(i).getTextContent()); - return refChildren.item(i).getTextContent(); - } - } - throw new Exception("No reference found in node"); - } - - /* - * Read all document, remove unused breakline, normalize spaces and encode XML characters - */ - private void encode() throws XPathExpressionException { - Node node = packagedDocument.getFirstChild(); - encode(node); - } - - private void encode(NodeList nodeListToEncode) { - for (int i = 0; i < nodeListToEncode.getLength(); ++i) { - encode(nodeListToEncode.item(i)); - } - } - - private void encode(Node nodeToEncode) { - if (nodeToEncode == null) { - return; - } - if (nodeToEncode.hasChildNodes()) { - encode(nodeToEncode.getChildNodes()); - } - if (nodeToEncode.getNodeType() == Node.TEXT_NODE) { - nodeToEncode.setTextContent(StringUtils.removeStart(nodeToEncode.getTextContent(), "\n")); - nodeToEncode.setTextContent(StringUtils.removeEnd(nodeToEncode.getTextContent(), "\n")); - nodeToEncode.setTextContent(StringUtils.normalizeSpace(nodeToEncode.getTextContent())); - } - - } -} diff --git a/src/main/java/fr/insee/rmes/utils/export/XDocReport.java b/src/main/java/fr/insee/rmes/utils/export/XDocReport.java index 51b4c236..6746df65 100644 --- a/src/main/java/fr/insee/rmes/utils/export/XDocReport.java +++ b/src/main/java/fr/insee/rmes/utils/export/XDocReport.java @@ -8,9 +8,9 @@ import fr.opensagres.xdocreport.template.IContext; import fr.opensagres.xdocreport.template.TemplateEngineKind; import freemarker.ext.dom.NodeModel; -import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; import org.xml.sax.InputSource; import org.xml.sax.SAXException; @@ -57,7 +57,7 @@ private IContext getXmlData(IXDocReport report, String xmlInput) xml = NodeModel.parse( projectInputSource ); context = report.createContext(); } catch (SAXException | IOException | ParserConfigurationException | XDocReportException e) { - throw new RmesException(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage(), e.getClass()+" - Can't put xml in XdocReport context"); + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage(), e.getClass()+" - Can't put xml in XdocReport context"); } context.put("racine", xml); return context; From 767b47edc561753f1a94aa45b2244f10437031c5 Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Mon, 25 Nov 2024 18:08:47 +0100 Subject: [PATCH 05/25] refactor (mutulualize, separate concerns, remove useless, remove redondant, remove elastic search) WIP --- pom.xml | 2 +- .../config/keycloak/KeycloakServices.java | 14 + .../tocolecticaapi/controller/GetItem.java | 42 +- .../tocolecticaapi/fragments/DdiFragment.java | 11 + .../service/ColecticaServiceImpl.java | 132 +-- .../java/fr/insee/rmes/utils/XsltUtils.java | 29 +- .../controller/GetItemTest.java | 63 ++ .../TransformationControllerTest.java | 6 +- ...-f0bb-47df-b3d2-42ff7f8f5874_expected.json | 810 ++++++++++++++++++ 9 files changed, 981 insertions(+), 128 deletions(-) create mode 100644 src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragment.java create mode 100644 src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java create mode 100644 src/test/resources/getItemTest/34abf2d5-f0bb-47df-b3d2-42ff7f8f5874_expected.json diff --git a/pom.xml b/pom.xml index e3a8ab48..04c5d7e0 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ 21 UTF-8 - 2.6.0 + 2.7.0 2.0.6 12.4 1.1.1 diff --git a/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java b/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java index 0693497b..85b7fc70 100644 --- a/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java +++ b/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java @@ -54,6 +54,8 @@ public class KeycloakServices { @Value("${auth.password}") private String password; private static final Logger logger = LoggerFactory.getLogger(KeycloakServices.class); + private String token; + /** * Permet de récuperer un jeton keycloak * @return jeton @@ -148,4 +150,16 @@ public static String extractAccessToken(String token) { } return null; } + + public String getFreshToken() throws ExceptionColecticaUnreachable, JsonProcessingException { + if ( ! this.isTokenValid(this.token)) { + token = getToken(); + } + return token; + } + + private String getToken() throws ExceptionColecticaUnreachable, JsonProcessingException { + return this.getKeycloakAccessToken(); + + } } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java index 36af43a2..77f97c96 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java @@ -2,20 +2,19 @@ import com.fasterxml.jackson.core.JsonProcessingException; import fr.insee.rmes.exceptions.ExceptionColecticaUnreachable; +import fr.insee.rmes.exceptions.RmesExceptionIO; import fr.insee.rmes.model.DDIItemType; +import fr.insee.rmes.tocolecticaapi.fragments.DdiFragment; import fr.insee.rmes.tocolecticaapi.service.ColecticaService; -import fr.insee.rmes.exceptions.RmesExceptionIO; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.MediaType; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.json.simple.parser.ParseException; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; @@ -29,18 +28,15 @@ @RestController @RequestMapping("/Item") @Tag(name= "DEMO-Colectica",description = "Services for upgrade Colectica-API") +@Slf4j +@RequiredArgsConstructor public class GetItem { - static final Logger logger = LogManager.getLogger(GetItem.class); - @Autowired - private final ColecticaService colecticaService; - public GetItem(ColecticaService colecticaService) { - this.colecticaService = colecticaService; - } + private final ColecticaService colecticaService; + private final DdiFragment ddiFragment; - @GetMapping("fragmentInstance/uuid") + @GetMapping(value = "fragmentInstance/uuid", produces = MediaType.APPLICATION_XML_VALUE) @Operation(summary = "Get FragmentInstance by uuid", description = "Get an XML document for a Fragment:Instance from Colectica repository.") - @Produces(MediaType.APPLICATION_XML) public ResponseEntity findInstanceByUuidColectica( @Parameter( description = "id de l'objet colectica sous la forme uuid/version", @@ -48,12 +44,20 @@ public ResponseEntity findInstanceByUuidColectica( schema = @Schema( type = "string", example="d6c08ec1-c4d2-4b9a-b358-b23aa4e0af93")) String uuid) throws RmesExceptionIO, ParseException { return colecticaService.findInstanceByUuid(uuid); + } + @GetMapping(value = "ddiFragment/{uuid}/dataRelationship", produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Fournir une représentationn JSON de l'objet dataRelationShip du DDI Fragment dont l'uuid est en paramètre") + public ResponseEntity extractDataRelationshipFromFragment(@Parameter( + description = "id du fragment DDI sous la forme uuid", + required = true, + schema = @Schema(type = "string", example="d6c08ec1-c4d2-4b9a-b358-b23aa4e0af93")) + @PathVariable String uuid){ + return ResponseEntity.ok(this.ddiFragment.extractDataRelationship(uuid)); } - @GetMapping("ddiFragment/uuid") + @GetMapping(value = "ddiFragment/uuid", produces = MediaType.APPLICATION_XML_VALUE) @Operation(summary = "Get Fragment by uuid", description = "Get an XML document for a ddi:Fragment from Colectica repository.") - @Produces(MediaType.APPLICATION_XML) public ResponseEntity findFragmentByUuidColectica( @Parameter( description = "id de l'objet colectica sous la forme uuid/version", @@ -64,9 +68,8 @@ public ResponseEntity findFragmentByUuidColectica( } - @GetMapping("FragmentInstance/uuid/withChildren") + @GetMapping(value = "FragmentInstance/uuid/withChildren", produces = MediaType.APPLICATION_XML_VALUE) @Operation(summary = "Get Fragments by uuid", description = "Get an XML document for a Fragment:Instance from Colectica repository.") - @Produces(MediaType.APPLICATION_XML) public String findFragmentByUuidWithChildrenColectica( @Parameter( description = "id de l'objet colectica sous la forme uuid/version", @@ -146,8 +149,7 @@ public ResponseEntity searchByType( - @GetMapping("suggesters/jsonWithChild") - @Produces(MediaType.APPLICATION_JSON) + @GetMapping(value = "suggesters/jsonWithChild", produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Get JSON for Suggester/codelist simple (id,label)", description = "Get a JSON document for suggester or codelist from Colectica repository including an item with childs.") public Object getJsonWithChild( @Parameter( diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragment.java b/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragment.java new file mode 100644 index 00000000..85d34070 --- /dev/null +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragment.java @@ -0,0 +1,11 @@ +package fr.insee.rmes.tocolecticaapi.fragments; + +public interface DdiFragment { + /** + * Retourne une projection json de l'objet datarelationship du fragment DDI dont l'id est passé en paramètre. + * Le DDI fragment est fourni en interrogeant colectica + * @param uuid : l'id du DDI fragment dont on doit extraire le datarelationship + * @return une chaine de caractères représentant le datarelationship + */ + String extractDataRelationship(String uuid); +} diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java index c1115045..106d2249 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java @@ -17,16 +17,14 @@ import fr.insee.rmes.transfoxsl.service.XsltTransformationService; import fr.insee.rmes.utils.ExportUtils; import fr.insee.rmes.utils.FilesUtils; -import fr.insee.rmes.utils.XMLUtils; +import fr.insee.rmes.utils.XsltUtils; import fr.insee.rmes.utils.export.XDocReport; import lombok.NonNull; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import net.sf.saxon.TransformerFactoryImpl; import net.sf.saxon.s9api.ExtensionFunction; import net.sf.saxon.s9api.Processor; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; import org.json.simple.parser.JSONParser; @@ -78,6 +76,7 @@ @Service @RequiredArgsConstructor +@Slf4j public class ColecticaServiceImpl implements ColecticaService { private static final String CONTENT_TYPE = "Content-Type"; @@ -94,7 +93,6 @@ public class ColecticaServiceImpl implements ColecticaService { private static final String ATTACHMENT = "attachment"; - static final Logger logger = LogManager.getLogger(ColecticaServiceImpl.class); public static final String DISALLOW_DOCTYPE_DECL = "http://javax.xml.transform.TransformerFactory/feature/disallow-doctype-decl"; public static final String DISALLOW_DOCTYPE_DECL1 = "http://apache.org/xml/features/disallow-doctype-decl"; public static final String EXTERNAL_GENERAL_ENTITIES = "http://xml.org/sax/features/external-general-entities"; @@ -102,10 +100,8 @@ public class ColecticaServiceImpl implements ColecticaService { @NonNull - @Autowired - private KeycloakServices kc; + private final KeycloakServices kc; - private String token; @Value("${auth.api.url}") private String authApiUrl; @@ -161,7 +157,6 @@ private ResponseEntity searchColecticaFragmentByUuid(String uuid) throws try (CloseableHttpResponse response = httpClient.execute(httpGet)) { return getResponseEntitySearchColecticaFragmentByUuid(response); } catch (IOException e) { - throw new IOException("Erreur IO lors de la récupération des données pour UUID: " + uuid, e); } } catch (ExceptionColecticaUnreachable e) { @@ -178,17 +173,17 @@ private HttpRequest getGetSearchColecticaFragmentByUuid(String uuid) throws Exce httpGet.addHeader(CONTENT_TYPE, APPLICATION_XML); httpGet.addHeader("Accept", APPLICATION_JSON); - String authToken; - if (!serviceUrl.contains("kube")) { - authToken = getFreshToken(); - } else { - String token2 = getAuthToken(); - authToken = extractAccessToken(token2); - } - httpGet.setHeader(AUTHORIZATION, BEARER + authToken); + httpGet.setHeader(AUTHORIZATION, BEARER + resolveBearerToken()); return httpGet; } + private String resolveBearerToken() throws ExceptionColecticaUnreachable, JsonProcessingException, RmesExceptionIO, ParseException { + if (serviceUrl.contains("kube")) { + return extractAccessToken(getAuthToken()); + } + return kc.getFreshToken(); + } + protected static ResponseEntity getResponseEntitySearchColecticaFragmentByUuid(CloseableHttpResponse response) throws IOException { if (response.getStatusLine().getStatusCode() != HttpStatus.OK.value()) { return ResponseEntity.status(response.getStatusLine().getStatusCode()) @@ -358,7 +353,7 @@ public JSONArray processUuidsAndFetchData(Set uuids) throws ExceptionCol } } else { String message = "Erreur ou élément non trouvé pour UUID:" + uuid; - logger.log( Level.ERROR,message); + log.log( Level.ERROR,message); } } @@ -401,7 +396,7 @@ private JSONObject extractDataFromFragment(String fragmentXml) { return fragmentData; } catch (ParserConfigurationException | SAXException | IOException | XPathExpressionException e) { // Log the exception - logger.error("Error extracting data from fragment XML: {}", e.getMessage()); + log.error("Error extracting data from fragment XML: {}", e.getMessage()); // Return an appropriate response to the client return new JSONObject(); @@ -470,7 +465,7 @@ public ResponseEntity filteredSearchText(String index, String texte) { if (responseEntity.getStatusCode().is2xxSuccessful()) { String responseBody = responseEntity.getBody(); String filteredResponse = filterAndTransformResponse(responseBody); - logger.info("Réponse filtrée avec succès pour searchTexte"); + log.info("Réponse filtrée avec succès pour searchTexte"); return ResponseEntity.ok(filteredResponse); } else { return responseEntity; @@ -483,7 +478,7 @@ public ResponseEntity searchByType(String index, DDIItemType type) { if (responseEntity.getStatusCode().is2xxSuccessful()) { String responseBody = responseEntity.getBody(); String filteredResponse = filterAndTransformResponse(responseBody); - logger.info("Réponse filtrée avec succès pour searchByType"); + log.info("Réponse filtrée avec succès pour searchByType"); return ResponseEntity.ok(filteredResponse); } else { return responseEntity; @@ -496,7 +491,7 @@ public ResponseEntity searchTexteByType(String index, String texte, DDII if (responseEntity.getStatusCode().is2xxSuccessful()) { String responseBody = responseEntity.getBody(); String filteredResponse = filterAndTransformResponse(responseBody); - logger.info("Réponse filtrée avec succès pour searchTexteByType"); + log.info("Réponse filtrée avec succès pour searchTexteByType"); return ResponseEntity.ok(filteredResponse); } else { return responseEntity; @@ -642,14 +637,8 @@ private String filterAndTransformResponse(String responseBody) { public ResponseEntity>> getJsonWithChild(String identifier, String outputField, String fieldLabelName) throws Exception { String apiUrl = serviceUrl+"/api/v1/jsonset/fr.insee/" + identifier; - if (!serviceUrl.contains("kube")){ - token = getFreshToken(); - } else { - String token2 = getAuthToken(); - token = extractAccessToken(token2); - } HttpHeaders headers = new HttpHeaders(); - headers.set(AUTHORIZATION, BEARER + token); + headers.set(AUTHORIZATION, BEARER + resolveBearerToken()); HttpEntity requestEntity = new HttpEntity<>(headers); try { @@ -687,12 +676,6 @@ public String convertXmlToJson(String uuid) throws ExceptionColecticaUnreachable String apiUrl = serviceUrl + "/api/v1/jsonset/fr.insee/" + uuid; - if (!serviceUrl.contains("kube")) { - token = getFreshToken(); - } else { - String token2 = getAuthToken(); - token = extractAccessToken(token2); - } HttpHeaders headers = new HttpHeaders(); headers.set(AUTHORIZATION, BEARER + token); @@ -715,7 +698,7 @@ public String convertXmlToJson(String uuid) throws ExceptionColecticaUnreachable result = mapperFinal.writeValueAsString(countriesWithCodesAndLabels); } catch (RestClientException e) { // Log the exception - logger.error("Error while calling the REST API: {}", e.getMessage()); + log.error("Error while calling the REST API: {}", e.getMessage()); // Rethrow the exception or handle it appropriately throw new ExceptionColecticaUnreachable("Error while calling the REST API", e); } @@ -827,12 +810,6 @@ private String transformToJson(Resource resultResource, InputStream xsltFileJson @Override public ResponseEntity getByType(DDIItemType type) throws IOException, ExceptionColecticaUnreachable, ParseException { - if (!serviceUrl.contains("kube")){ - token = getFreshToken(); - } else { - String token2 = getAuthToken(); - token = extractAccessToken(token2); - } HttpHeaders headers = new HttpHeaders(); headers.set(AUTHORIZATION, BEARER + token); @@ -860,13 +837,7 @@ public ResponseEntity sendUpdateColectica(String ddiUpdatingInJson, Tran try { RestTemplate restTemplate=new RestTemplate(); String initTransactionUrl = serviceUrl + "/api/v1/transaction"; - String authentToken; - if (serviceUrl.contains("kube")) { - String token2 = getAuthToken(); - authentToken = extractAccessToken(token2); - } else { - authentToken = getFreshToken(); - } + String authentToken=resolveBearerToken(); HttpHeaders initTransactionHeaders = new HttpHeaders(); initTransactionHeaders.setContentType(MediaType.APPLICATION_JSON); @@ -991,7 +962,7 @@ public ResponseEntity transformFile( String resultFileName2 = UUID.randomUUID().toString() + ".json"; MultipartFile outputFile = processFile(file); - logger.log( Level.ERROR,"Le fichier a été modifié avec succès (ajout balise data) !"); + log.log( Level.ERROR,"Le fichier a été modifié avec succès (ajout balise data) !"); InputStream xsltStream1 = getClass().getResourceAsStream("/jsontoDDIXML.xsl"); String xmlContent = transformToXml(outputFile, xsltStream1, idValue, nomenclatureName, suggesterDescription, timbre,version); @@ -1029,7 +1000,7 @@ public ResponseEntity transformFileForComplexList( String resultFileName2 = UUID.randomUUID() + ".json"; MultipartFile outputFile = processFile(file); - logger.log(Level.getLevel(ERROR),"Le fichier a été modifié avec succès (ajout balise data) !"); + log.log(Level.getLevel(ERROR),"Le fichier a été modifié avec succès (ajout balise data) !"); InputStream xsltStream1 = getClass().getResourceAsStream("/jsonToDdiXmlForComplexList.xsl"); String xmlContent = transformToXmlForComplexList(outputFile, xsltStream1, idValue, nomenclatureName, suggesterDescription, timbre,version,principale,secondaire,labelSecondaire); @@ -1245,19 +1216,20 @@ private String getAuthToken() throws JsonProcessingException, RmesExceptionIO { throw new RmesExceptionIO(401, "Impossible d'obtenir le token d'authentification.", "toto"); } } - public static String extractAccessToken(String token) throws ParseException { + + private static String extractAccessToken(String token) throws ParseException { try { JSONParser parser = new JSONParser(); JSONObject json = (JSONObject) parser.parse(token); return (String) json.get("access_token"); } catch (ParseException e) { // Log the exception - logger.error("Error parsing JSON token: {}", e.getMessage()); + log.error("Error parsing JSON token: {}", e.getMessage()); // Rethrow the exception throw e; } catch (ClassCastException e) { // Log the exception - logger.error("Error casting JSON token: {}", e.getMessage()); + log.error("Error casting JSON token: {}", e.getMessage()); // Convert the exception to a ParseException throw new ParseException(ParseException.ERROR_UNEXPECTED_TOKEN, e); } @@ -1267,14 +1239,7 @@ public static String extractAccessToken(String token) throws ParseException { public String sendDeleteColectica(String uuid, TransactionType transactionType) throws JsonProcessingException, ExceptionColecticaUnreachable, RmesExceptionIO, ParseException { RestTemplate restTemplate = new RestTemplate(); String initTransactionUrl = serviceUrl + "/api/v1/transaction"; - String authentToken; - - if (serviceUrl.contains("kube")) { - String token2 = getAuthToken(); - authentToken = extractAccessToken(token2); - } else { - authentToken = getFreshToken(); - } + String authentToken = resolveBearerToken(); HttpHeaders initTransactionHeaders = new HttpHeaders(); initTransactionHeaders.setContentType(MediaType.APPLICATION_JSON); @@ -1418,11 +1383,11 @@ public ResponseEntity getCodeBookExportV2(String ddi, String xslPatter File ddiRemoveNameSpaces = File.createTempFile("ddiRemoveNameSpaces", ".xml"); ddiRemoveNameSpaces.deleteOnExit(); - transformerStringWithXsl(ddi, xslRemoveNameSpaces, ddiRemoveNameSpaces); + XsltUtils.transformerStringWithXsl(ddi, xslRemoveNameSpaces, ddiRemoveNameSpaces); File control = File.createTempFile("control", ".xml"); control.deleteOnExit(); - transformerFileWithXsl(ddiRemoveNameSpaces, xslCheckReference, control); + XsltUtils.transformerFileWithXsl(ddiRemoveNameSpaces, xslCheckReference, control); // Créer un DocumentBuilderFactory sécurisé DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); @@ -1460,22 +1425,6 @@ public ResponseEntity getCodeBookExportV2(String ddi, String xslPatter return exportUtils.exportAsODT(Path.of("export.odt"), contentXML, dicoCode, xslPatternFile, zipRmes, "dicoVariable"); } - public static void transformerStringWithXsl(String ddi,InputStream xslRemoveNameSpaces, File output) throws Exception{ - Source stylesheetSource = new StreamSource(xslRemoveNameSpaces); - Transformer transformer = XMLUtils.getTransformerFactory().newTransformer(stylesheetSource); - Source inputSource = new StreamSource(new StringReader(ddi)); - Result outputResult = new StreamResult(output); - transformer.transform(inputSource, outputResult); - } - - public static void transformerFileWithXsl(File input,InputStream xslCheckReference, File output) throws Exception { - Source stylesheetSource = new StreamSource(xslCheckReference); - Transformer transformer = XMLUtils.getTransformerFactory().newTransformer(stylesheetSource); - Source inputSource = new StreamSource(input); - Result outputResult = new StreamResult(output); - transformer.transform(inputSource, outputResult); - } - @Override public ResponseEntity getCodeBookCheck(MultipartFile isCodeBook) throws Exception { @@ -1484,12 +1433,12 @@ public ResponseEntity getCodeBookCheck(MultipartFile isCodeBook) throws Excep InputStream xslRemoveNameSpaces = getClass().getResourceAsStream("/xslTransformerFiles/remove-namespaces.xsl"); File ddiRemoveNameSpaces = File.createTempFile("ddiRemoveNameSpaces", ".xml"); ddiRemoveNameSpaces.deleteOnExit(); - transformerInputStreamWithXsl(codeBook,xslRemoveNameSpaces,ddiRemoveNameSpaces); + XsltUtils.transformerInputStreamWithXsl(codeBook,xslRemoveNameSpaces,ddiRemoveNameSpaces); InputStream xslCodeBookCheck = getClass().getResourceAsStream("/xslTransformerFiles/dico-codes-test-ddi-content.xsl"); File codeBookCheck = File.createTempFile("codeBookCheck", ".xml"); codeBookCheck.deleteOnExit(); - transformerFileWithXsl(ddiRemoveNameSpaces,xslCodeBookCheck,codeBookCheck); + XsltUtils.transformerFileWithXsl(ddiRemoveNameSpaces,xslCodeBookCheck,codeBookCheck); HttpHeaders headers = new HttpHeaders(); headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + codeBookCheck.getName()); @@ -1510,23 +1459,4 @@ public ResponseEntity getCodeBookCheck(MultipartFile isCodeBook) throws Excep } - public static void transformerInputStreamWithXsl(InputStream input,InputStream xslCheckReference, File output) throws Exception { - Source stylesheetSource = new StreamSource(xslCheckReference); - Transformer transformer = XMLUtils.getTransformerFactory().newTransformer(stylesheetSource); - Source inputSource = new StreamSource(input); - Result outputResult = new StreamResult(output); - transformer.transform(inputSource, outputResult); - } - public String getFreshToken() throws ExceptionColecticaUnreachable, JsonProcessingException { - if ( ! kc.isTokenValid(token)) { - token = getToken(); - } - return token; - } - - public String getToken() throws ExceptionColecticaUnreachable, JsonProcessingException { - return kc.getKeycloakAccessToken(); - - } - } diff --git a/src/main/java/fr/insee/rmes/utils/XsltUtils.java b/src/main/java/fr/insee/rmes/utils/XsltUtils.java index d02badd5..4cf7ee29 100644 --- a/src/main/java/fr/insee/rmes/utils/XsltUtils.java +++ b/src/main/java/fr/insee/rmes/utils/XsltUtils.java @@ -7,13 +7,13 @@ import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; +import javax.xml.transform.Result; +import javax.xml.transform.Source; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintStream; +import java.io.*; import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.util.Map; @@ -99,4 +99,27 @@ public static String buildParams(Boolean lg1, Boolean lg2, Boolean includeEmptyF return XMLUtils.encodeXml(parametersXML); } + public static void transformerInputStreamWithXsl(InputStream input,InputStream xslCheckReference, File output) throws Exception { + Source stylesheetSource = new StreamSource(xslCheckReference); + Transformer transformer = XMLUtils.getTransformerFactory().newTransformer(stylesheetSource); + Source inputSource = new StreamSource(input); + Result outputResult = new StreamResult(output); + transformer.transform(inputSource, outputResult); + } + + public static void transformerStringWithXsl(String ddi,InputStream xslRemoveNameSpaces, File output) throws Exception{ + Source stylesheetSource = new StreamSource(xslRemoveNameSpaces); + Transformer transformer = XMLUtils.getTransformerFactory().newTransformer(stylesheetSource); + Source inputSource = new StreamSource(new StringReader(ddi)); + Result outputResult = new StreamResult(output); + transformer.transform(inputSource, outputResult); + } + + public static void transformerFileWithXsl(File input, InputStream xslCheckReference, File output) throws Exception { + Source stylesheetSource = new StreamSource(xslCheckReference); + Transformer transformer = XMLUtils.getTransformerFactory().newTransformer(stylesheetSource); + Source inputSource = new StreamSource(input); + Result outputResult = new StreamResult(output); + transformer.transform(inputSource, outputResult); + } } diff --git a/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java b/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java new file mode 100644 index 00000000..bfc64503 --- /dev/null +++ b/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java @@ -0,0 +1,63 @@ +package fr.insee.rmes.tocolecticaapi.controller; + +import fr.insee.rmes.tocolecticaapi.fragments.DdiFragment; +import fr.insee.rmes.tocolecticaapi.service.ColecticaService; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.test.web.servlet.MockMvc; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@WebMvcTest(GetItem.class) +@ActiveProfiles("dev") +class GetItemTest { + + @Autowired + private MockMvc mockMvc; + + @MockitoBean + private ColecticaService colecticaService; + + @Test + void whenGetDataRelationship_shouldReturnRightJson() throws Exception { + String uuid="34abf2d5-f0bb-47df-b3d2-42ff7f8f5874"; + var dataRelationShipEndpoint="ddiFragment/"+uuid+"/dataRelationship"; + mockMvc.perform(get(dataRelationShipEndpoint).accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().json(read("34abf2d5-f0bb-47df-b3d2-42ff7f8f5874_expected.json"))); + } + + private String read(String fileName) throws Exception { + try (Stream lines = Files.lines(Path.of(fileName))){ + return lines.collect(Collectors.joining()); + } + } + + @TestConfiguration + static class ConfigurationForTest{ + @Bean + DdiFragment ddiFragment(){ + return new DdiFragment() { + @Override + public String extractDataRelationship(String uuid) { + return ""; + } + }; + } + } + +} \ No newline at end of file diff --git a/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java b/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java index b7909295..e9cf262a 100644 --- a/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java +++ b/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java @@ -6,11 +6,11 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; -import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.http.MediaType; import org.springframework.mock.web.MockMultipartFile; import org.springframework.security.test.context.support.WithMockUser; import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.bean.override.mockito.MockitoBean; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; @@ -30,10 +30,10 @@ class TransformationControllerTest { @Autowired private MockMvc mockMvc; - @MockBean + @MockitoBean private XsltTransformationService xsltTransformationService; - @MockBean + @MockitoBean private MultipartFileUtils multipartFileUtils; @Test diff --git a/src/test/resources/getItemTest/34abf2d5-f0bb-47df-b3d2-42ff7f8f5874_expected.json b/src/test/resources/getItemTest/34abf2d5-f0bb-47df-b3d2-42ff7f8f5874_expected.json new file mode 100644 index 00000000..a8c39682 --- /dev/null +++ b/src/test/resources/getItemTest/34abf2d5-f0bb-47df-b3d2-42ff7f8f5874_expected.json @@ -0,0 +1,810 @@ +[{"id": "urn:ddi:fr.insee:fcc416b8-2ea0-4df4-8420-4f7f3d761168:1", "nom": "indivi17", "label": +[ + { + "contenu": "", "langue": "fr"}, + { + "contenu": "", + "langue": "en" + } +], + "variables": + [{"id": "urn:ddi:fr.insee:8ec8f1bc-ff67-45ee-a78b-c31951a78cb3:1", "nom": "noindiv", "label": [ + { + "contenu": "Identifiant individu", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "1", "representation": "texte"},{"id": "urn:ddi:fr.insee:6758fc71-dc86-4edb-8c14-0f14362b3dee:1", "nom": "ident17", "label": [ + { + "contenu": "Identifiant du ménage", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "2", "representation": "texte"},{"id": "urn:ddi:fr.insee:4d63151e-3bb5-444b-b3f1-9771507e0ea0:1", "nom": "noi", "label": [ + { + "contenu": "Numéro individuel Enquête Emploi", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "3", "representation": "texte"},{"id": "urn:ddi:fr.insee:ab13bec9-0495-426d-a582-4b087a7b6ea5:1", "nom": "cs8cor", "label": [ + { + "contenu": "CS en 8 postes avec correction ERFS", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "4", "representation": "codes", "controles": + { + "codes": [ + { "code": "1 " , "label": [{ + "langue": "fr", + "contenu": "Agriculteurs exploitants" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "2 " , "label": [{ + "langue": "fr", + "contenu": "Artisans, commerçants et chefs d'entreprise" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "3 " , "label": [{ + "langue": "fr", + "contenu": "Cadres et professions intellectuelles supérieures" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "4 " , "label": [{ + "langue": "fr", + "contenu": "Professions intermédiaires" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "5 " , "label": [{ + "langue": "fr", + "contenu": "Employés" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "6 " , "label": [{ + "langue": "fr", + "contenu": "Ouvriers" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "7 " , "label": [{ + "langue": "fr", + "contenu": "Retraités" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "8 " , "label": [{ + "langue": "fr", + "contenu": "Autres inactifs" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ]}},{"id": "urn:ddi:fr.insee:0369bc65-682c-4b63-9d09-9b339ec49dd2:1", "nom": "quelfic", "label": [ + { + "contenu": "Origine de l'observation", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "5", "representation": "codes", "controles": + { + "codes": [ + { "code": "EE&FIP " , "label": [{ + "langue": "fr", + "contenu": "L’individu a été enquêté à l’enquête Emploi, et au moins une déclaration fiscale le concernant a été retrouvée" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "EE_CAF " , "label": [{ + "langue": "fr", + "contenu": "L’individu d’un ménage considéré comme répondant ERFS ne se retrouve dans aucune déclaration fiscale, mais ses revenus ont pu être estimés à partir d’un appariement réussi avec le fichier CNAF" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "EE " , "label": [{ + "langue": "fr", + "contenu": "L'individu d’un ménage considéré comme répondant ERFS ne se retrouve dans aucune des déclarations fiscales associées au ménage, ses revenus individuels seront donc entièrement imputés" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "EE_NRT " , "label": [{ + "langue": "fr", + "contenu": "Etudiants de moins de 26 ans formant un ménage à part dans l’Enquête Emploi et dont on n'a retrouvé aucune trace dans les fichiers fiscaux (non-réponse totale), vraisemblablement parce qu’ils étaient rattachés au foyer fiscal de leur(s) parent(s)" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ]}},{"id": "urn:ddi:fr.insee:5d269749-dd06-4af9-a91e-8972a2b2ba82:1", "nom": "persfip", "label": [ + { + "contenu": "Identifiant individuel (2042) : indique si l'individu est le déclarant ou quel est son lien avec le déclarant", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "6", "representation": "codes", "controles": + { + "codes": [ + { "code": "vous " , "label": [{ + "langue": "fr", + "contenu": "L'individu est le déclarant" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "conj " , "label": [{ + "langue": "fr", + "contenu": "Conjoint" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "pac " , "label": [{ + "langue": "fr", + "contenu": "Personne à charge" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ]}},{"id": "urn:ddi:fr.insee:8abdcea6-ce62-46a4-b0d8-31a7d9bde324:1", "nom": "persfipd", "label": [ + { + "contenu": "Identifiant individuel détaillé (2042) : indique où rechercher les revenus de l’individu sur la déclaration fiscale", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "7", "representation": "codes", "controles": + { + "codes": [ + { "code": "mad " , "label": [{ + "langue": "fr", + "contenu": "Madame" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "mon " , "label": [{ + "langue": "fr", + "contenu": "Monsieur" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "vopa " , "label": [{ + "langue": "fr", + "contenu": "Individus atteignant l'âge de 18 ans l’année de référence, pouvant avoir une double déclaration (seule et sur celle de leurs parents)" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ]}},{"id": "urn:ddi:fr.insee:f76704e8-1df8-45fb-9c0c-3dac8b29b372:1", "nom": "zsali", "label": [ + { + "contenu": "Salaires au sens strict (individu)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "8", "representation": "numerique"},{"id": "urn:ddi:fr.insee:63056208-ea5d-40b4-afba-4e4439f13b9f:1", "nom": "zchoi", "label": [ + { + "contenu": "Allocation chômage et préretraite (individu)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "9", "representation": "numerique"},{"id": "urn:ddi:fr.insee:19e06933-0e02-44e8-9a9b-875c1b7581b2:1", "nom": "ztsai", "label": [ + { + "contenu": "Traitements et salaires (individu)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "10", "representation": "numerique"},{"id": "urn:ddi:fr.insee:3783b748-2f6e-4143-88a9-5d454f432d9a:1", "nom": "zrsti", "label": [ + { + "contenu": "Retraites au sens strict (individu)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "11", "representation": "numerique"},{"id": "urn:ddi:fr.insee:87e85c5d-6380-4423-be51-b4c9f3736e55:1", "nom": "zpii", "label": [ + { + "contenu": "Pensions d'invalidité (individu)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "12", "representation": "numerique"},{"id": "urn:ddi:fr.insee:00b3f378-535f-4770-961d-4822aebf36b0:1", "nom": "zalri", "label": [ + { + "contenu": "Pensions alimentaires reçues (individu)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "13", "representation": "numerique"},{"id": "urn:ddi:fr.insee:42ae6bd5-804a-4182-b68e-70ff83193ba7:1", "nom": "zreti", "label": [ + { + "contenu": "Retraites et pensions (individu)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "14", "representation": "numerique"},{"id": "urn:ddi:fr.insee:96d279f5-cef3-40e6-b519-b9dc7bfd42f7:1", "nom": "zrtoi", "label": [ + { + "contenu": "Rentes viagères à titre onéreux (individu)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "15", "representation": "numerique"},{"id": "urn:ddi:fr.insee:b399dde0-790b-4a57-b530-354664ba7ba1:1", "nom": "zperi", "label": [ + { + "contenu": "Pensions, retraites, rentes (individu)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "16", "representation": "numerique"},{"id": "urn:ddi:fr.insee:f08de465-fb1b-42c3-be4f-34e0390e216e:1", "nom": "zragi", "label": [ + { + "contenu": "Revenus agricoles (individu)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "17", "representation": "numerique"},{"id": "urn:ddi:fr.insee:dd2b743d-82a6-450f-8274-e5dbccc02946:1", "nom": "zrici", "label": [ + { + "contenu": "Revenus industriels et commerciaux (individu)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "18", "representation": "numerique"},{"id": "urn:ddi:fr.insee:d1fd0fe7-458d-43f0-b336-3fd24e12cb70:1", "nom": "zrnci", "label": [ + { + "contenu": "Revenus non commerciaux (individu)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "19", "representation": "numerique"},{"id": "urn:ddi:fr.insee:22d6b5f9-ded9-4da1-a4bf-38c4e6ea6f3f:1", "nom": "zsalo", "label": [ + { + "contenu": "Salaires au sens strict (origine)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "20", "representation": "numerique"},{"id": "urn:ddi:fr.insee:397b2d4d-19e7-4e79-bad8-7c229988e887:1", "nom": "zchoo", "label": [ + { + "contenu": "Allocation chômage et préretraite (origine)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "21", "representation": "numerique"},{"id": "urn:ddi:fr.insee:a36675ec-ba11-4425-b74e-172fed36be6b:1", "nom": "ztsao", "label": [ + { + "contenu": "Traitements et salaires (origine)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "22", "representation": "numerique"},{"id": "urn:ddi:fr.insee:d2ce4d15-6e37-4ea6-a180-01db98beb530:1", "nom": "zrsto", "label": [ + { + "contenu": "Retraites au sens strict (origine)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "23", "representation": "numerique"},{"id": "urn:ddi:fr.insee:4f2e87c9-c9b5-4637-95d3-94641b141e48:1", "nom": "zpio", "label": [ + { + "contenu": "Pensions d'invalidité (origine)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "24", "representation": "numerique"},{"id": "urn:ddi:fr.insee:7991a4e1-bfe1-4c83-a6d9-57dd6e455ab3:1", "nom": "zalro", "label": [ + { + "contenu": "Pensions alimentaires reçues (origine)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "25", "representation": "numerique"},{"id": "urn:ddi:fr.insee:d8f3410e-e828-4454-9cbd-084b0f2b30e6:1", "nom": "zreto", "label": [ + { + "contenu": "Retraites et pensions (origine)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "26", "representation": "numerique"},{"id": "urn:ddi:fr.insee:a5d806bc-a8ee-4d06-a38e-db1ffb603d2c:1", "nom": "zrtoo", "label": [ + { + "contenu": "Rentes viagères à titre onéreux (origine)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "27", "representation": "numerique"},{"id": "urn:ddi:fr.insee:f5d573dd-00c0-4d05-a7ed-45d6e4beb41e:1", "nom": "zpero", "label": [ + { + "contenu": "Pensions, retraites, rentes (origine)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "28", "representation": "numerique"},{"id": "urn:ddi:fr.insee:22d270e6-eb42-4ddf-b0de-55e2652a7de6:1", "nom": "zrago", "label": [ + { + "contenu": "Revenus agricoles (origine)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "29", "representation": "numerique"},{"id": "urn:ddi:fr.insee:63274f17-4fce-4b3d-b065-64e521831b14:1", "nom": "zrico", "label": [ + { + "contenu": "Revenus industriels et commerciaux (origine)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "30", "representation": "numerique"},{"id": "urn:ddi:fr.insee:649d1170-d501-4e8a-a95c-c37af565bb1f:1", "nom": "zrnco", "label": [ + { + "contenu": "Revenus non commerciaux (origine)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "31", "representation": "numerique"},{"id": "urn:ddi:fr.insee:2b168ae2-3391-4688-b1f6-510bb2ce1a38:1", "nom": "declar1", "label": [ + { + "contenu": "Description résumée : déclaration n°1", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "32", "representation": "texte"},{"id": "urn:ddi:fr.insee:f2952d87-a888-434e-adb1-0ca8fe145b6b:1", "nom": "declar2", "label": [ + { + "contenu": "Description résumée : déclaration n°2", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "33", "representation": "texte"},{"id": "urn:ddi:fr.insee:44b36f21-f401-4b35-a537-cadc1b526ffe:1", "nom": "csgsald", "label": [ + { + "contenu": "CSG déductible sur salaires", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "34", "representation": "numerique"},{"id": "urn:ddi:fr.insee:630c67cc-4f16-4148-bb26-94b87d84fa48:1", "nom": "csgsali", "label": [ + { + "contenu": "CSG imposable sur salaires", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "35", "representation": "numerique"},{"id": "urn:ddi:fr.insee:feba9ab5-8938-4b8b-aba8-ca689e78949f:1", "nom": "crdssal", "label": [ + { + "contenu": "CRDS sur salaires", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "36", "representation": "numerique"},{"id": "urn:ddi:fr.insee:8b357a51-6c7d-4d22-badc-8dd7198f0bbc:1", "nom": "csgchod", "label": [ + { + "contenu": "CSG déductible sur allocations chômage", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "37", "representation": "numerique"},{"id": "urn:ddi:fr.insee:b966cdc3-d1b8-40ab-8e76-3b8d335f0d73:1", "nom": "csgchoi", "label": [ + { + "contenu": "CSG imposable sur allocations chômage", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "38", "representation": "numerique"},{"id": "urn:ddi:fr.insee:4bc65274-25e0-4a9f-824f-c86762788614:1", "nom": "crdscho", "label": [ + { + "contenu": "CRDS sur allocations chômage", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "39", "representation": "numerique"},{"id": "urn:ddi:fr.insee:59ccfb28-6023-4384-8e1f-056c56922c47:1", "nom": "csgrstd", "label": [ + { + "contenu": "CSG déductible sur retraites", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "40", "representation": "numerique"},{"id": "urn:ddi:fr.insee:b8f75a18-0856-424e-a6c1-6bafc9ddbf34:1", "nom": "csgrsti", "label": [ + { + "contenu": "CSG imposable sur retraites", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "41", "representation": "numerique"},{"id": "urn:ddi:fr.insee:efe9b759-5926-423a-823d-4a5b9bf84736:1", "nom": "crdsrst", "label": [ + { + "contenu": "CRDS sur retraites", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "42", "representation": "numerique"},{"id": "urn:ddi:fr.insee:c4993982-f996-4e3e-8565-3e1f3a79fad8:1", "nom": "csgpid", "label": [ + { + "contenu": "CSG imposable sur pensions d'invalidité", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "43", "representation": "numerique"},{"id": "urn:ddi:fr.insee:4a08bed8-1dde-4957-8014-788c1a6436bb:1", "nom": "csgpii", "label": [ + { + "contenu": "CSG déductible sur pensions d'invalidité", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "44", "representation": "numerique"},{"id": "urn:ddi:fr.insee:b6d2773c-67b0-435e-9ffe-13b3c083548a:1", "nom": "crdspi", "label": [ + { + "contenu": "CRDS sur pensions d'invalidité", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "45", "representation": "numerique"},{"id": "urn:ddi:fr.insee:f49137c2-8cee-45c2-b3bd-8d3da380dd47:1", "nom": "csgragd", "label": [ + { + "contenu": "CSG déductible sur RAG", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "46", "representation": "numerique"},{"id": "urn:ddi:fr.insee:509c63a6-02ad-49f6-bf27-a1b011662d08:1", "nom": "csgragi", "label": [ + { + "contenu": "CSG imposable sur RAG", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "47", "representation": "numerique"},{"id": "urn:ddi:fr.insee:6eef0c5d-5ad6-4ac6-a8b9-ddd8c7b082de:1", "nom": "crdsrag", "label": [ + { + "contenu": "CRDS sur RAG", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "48", "representation": "numerique"},{"id": "urn:ddi:fr.insee:a49541ee-eea0-4f3f-af3d-36d24e90b81f:1", "nom": "csgricd", "label": [ + { + "contenu": "CSG déductible sur RIC", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "49", "representation": "numerique"},{"id": "urn:ddi:fr.insee:c2e92b35-4372-46c6-9931-453c240aeada:1", "nom": "csgrici", "label": [ + { + "contenu": "CSG imposable sur RIC", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "50", "representation": "numerique"},{"id": "urn:ddi:fr.insee:1d930e75-9aac-4938-8f13-0adbe490f2a3:1", "nom": "crdsric", "label": [ + { + "contenu": "CRDS sur RIC", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "51", "representation": "numerique"},{"id": "urn:ddi:fr.insee:12e924d4-4d9b-40a1-89af-d1b0520540de:1", "nom": "csgrncd", "label": [ + { + "contenu": "CSG déductible sur RNC", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "52", "representation": "numerique"},{"id": "urn:ddi:fr.insee:2ad09d76-ee1c-4280-9d69-059f3a0324b5:1", "nom": "csgrnci", "label": [ + { + "contenu": "CSG imposable sur RNC", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "53", "representation": "numerique"},{"id": "urn:ddi:fr.insee:d2a61c5b-6e39-471e-816c-33759dd71ae7:1", "nom": "crdsrnc", "label": [ + { + "contenu": "CRDS sur RNC", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "54", "representation": "numerique"},{"id": "urn:ddi:fr.insee:f68b0a26-6a19-4bc1-b50f-8abad2c5e81b:1", "nom": "wprm", "label": [ + { + "contenu": "Poids final ERFS", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "55", "representation": "numerique"},{"id": "urn:ddi:fr.insee:9343c28a-3382-4a68-b3a5-78bb9b460062:1", "nom": "logt", "label": [ + { + "contenu": "Statut d'occupation corrigé", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "56", "representation": "texte"},{"id": "urn:ddi:fr.insee:af48a15d-baf1-4ff2-aee5-ae0602b355ce:1", "nom": "nb_UCI", "label": [ + { + "contenu": "Nombre d'Unités de Consommation du ménage", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "57", "representation": "numerique"},{"id": "urn:ddi:fr.insee:b261c008-ac55-4dd4-8be9-2c3e96c9e02f:1", "nom": "m_minvm", "label": [ + { + "contenu": "Minimum vieillesse du ménage", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "58", "representation": "numerique"},{"id": "urn:ddi:fr.insee:f6895b5d-febe-4a7f-b745-03c2b00ac593:1", "nom": "acteu5", "label": [ + { + "contenu": "Activité au sens BIT recalculée (5 postes)", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], "ordre": "59", "representation": "codes", "controles": + { + "codes": [ + { "code": "1 " , "label": [{ + "langue": "fr", + "contenu": "Salarié" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "2 " , "label": [{ + "langue": "fr", + "contenu": "Indépendant" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "3 " , "label": [{ + "langue": "fr", + "contenu": "Chômeur" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "4 " , "label": [{ + "langue": "fr", + "contenu": "Retraité" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ,{ "code": "5 " , "label": [{ + "langue": "fr", + "contenu": "Autre inactif" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + + ]}}]}] \ No newline at end of file From 6ce33207f643bb3f64e636884632d7dc4bd6fa73 Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Fri, 20 Dec 2024 09:48:04 +0100 Subject: [PATCH 06/25] refactor (mutulualize, separate concerns, remove useless, remove redondant, remove elastic search) WIP --- .../config/keycloak/KeycloakServices.java | 51 +- .../insee/rmes/exceptions/RmesException.java | 3 +- .../rmes/exceptions/RmesExceptionIO.java | 8 +- ....java => RandomUUIDExtensionFunction.java} | 2 +- .../controller/ControllerUtils.java | 26 + .../tocolecticaapi/controller/GetItem.java | 27 +- .../tocolecticaapi/controller/PostItem.java | 124 +- .../controller/RmesExceptionHandler.java | 26 + .../service/ColecticaService.java | 47 +- .../service/ColecticaServiceImpl.java | 1284 +++-------------- .../service/ElasticService.java | 84 ++ .../controller/TransformationController.java | 71 +- .../service/XsltTransformationService.java | 291 +++- .../service/internal/DDIDerefencer.java | 12 +- .../transfoxsl/utils/RestClientUtils.java | 22 + .../fr/insee/rmes/utils/FileExtension.java | 2 +- .../java/fr/insee/rmes/utils/FilesUtils.java | 71 +- src/main/resources/application.properties | 1 + .../service/ColecticaServiceImplTest.java | 30 +- 19 files changed, 814 insertions(+), 1368 deletions(-) rename src/main/java/fr/insee/rmes/tocolecticaapi/{randomUUID.java => RandomUUIDExtensionFunction.java} (90%) create mode 100644 src/main/java/fr/insee/rmes/tocolecticaapi/controller/ControllerUtils.java create mode 100644 src/main/java/fr/insee/rmes/tocolecticaapi/controller/RmesExceptionHandler.java create mode 100644 src/main/java/fr/insee/rmes/tocolecticaapi/service/ElasticService.java create mode 100644 src/main/java/fr/insee/rmes/transfoxsl/utils/RestClientUtils.java diff --git a/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java b/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java index 85b7fc70..b2bc9032 100644 --- a/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java +++ b/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java @@ -17,6 +17,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; +import org.springframework.web.client.RestClient; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; @@ -27,16 +28,16 @@ public class KeycloakServices { - @Value("${fr.insee.rmes.metadata.keycloak.secret}") + @Value("${fr.insee.rmes.metadata.keycloak.secret}") String secret; - - @Value("${fr.insee.rmes.metadata.keycloak.resource}") + + @Value("${fr.insee.rmes.metadata.keycloak.resource}") String resource; - - @Value("${fr.insee.rmes.metadata.keycloak.server}") + + @Value("${fr.insee.rmes.metadata.keycloak.server}") String server; - - @Value("${fr.insee.rmes.metadata.keycloak.realm}") + + @Value("${fr.insee.rmes.metadata.keycloak.realm}") String realm; @Value("${fr.insee.rmes.api.remote.metadata.url}") @@ -58,9 +59,10 @@ public class KeycloakServices { /** * Permet de récuperer un jeton keycloak + * * @return jeton */ - public String getKeycloakAccessToken() throws ExceptionColecticaUnreachable, JsonProcessingException { + public String getKeycloakAccessToken() { if (!testKube.contains(".dev.kube.insee.fr")) { RestTemplate keycloakClient = new RestTemplate(); @@ -74,25 +76,22 @@ public String getKeycloakAccessToken() throws ExceptionColecticaUnreachable, Jso body.add("client_id", resource); body.add("client_secret", secret); HttpEntity entity = new HttpEntity<>(body, headers); - try { - Token accessToken = keycloakClient.postForObject(keycloakUrl, entity, Token.class); + Token accessToken = keycloakClient.postForObject(keycloakUrl, entity, Token.class); + + log.trace("Keycloak token provided"); + return accessToken.getAccessToken(); - log.trace("Keycloak token provided"); - return accessToken.getAccessToken(); - } catch (RestClientException e) { - throw new ExceptionColecticaUnreachable("Le serveur Keycloak est injoignable"); - } } else { - String token = getAuthToken(); - return extractAccessToken(token); + return extractAccessToken(getAuthToken()); } } /** * Verifie si le jeton keycloak a expiré + * * @param token * @return boolean */ @@ -108,17 +107,19 @@ public boolean isTokenValid(String token) { log.info("Token is valid"); isValid = true; } - } - catch (JWTDecodeException exception) { + } catch (JWTDecodeException exception) { System.out.println("erreur" + exception.toString()); } return isValid; } - private String getAuthToken() throws JsonProcessingException { + private String getAuthToken() { RestTemplate restTemplate = new RestTemplate(); + // TODO utiliser + //RestClient.create().post().contentType(MediaType.APPLICATION_JSON).body(new AuthRequest("","")).retrieve().body(String.class); + HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); @@ -151,15 +152,11 @@ public static String extractAccessToken(String token) { return null; } - public String getFreshToken() throws ExceptionColecticaUnreachable, JsonProcessingException { - if ( ! this.isTokenValid(this.token)) { - token = getToken(); + public String getFreshToken() { + if (!this.isTokenValid(this.token)) { + token = getKeycloakAccessToken(); } return token; } - private String getToken() throws ExceptionColecticaUnreachable, JsonProcessingException { - return this.getKeycloakAccessToken(); - - } } diff --git a/src/main/java/fr/insee/rmes/exceptions/RmesException.java b/src/main/java/fr/insee/rmes/exceptions/RmesException.java index bd8e3fb8..b8fe93b8 100644 --- a/src/main/java/fr/insee/rmes/exceptions/RmesException.java +++ b/src/main/java/fr/insee/rmes/exceptions/RmesException.java @@ -3,6 +3,7 @@ import org.json.JSONArray; import org.json.JSONObject; import org.springframework.http.HttpStatus; +import org.springframework.http.HttpStatusCode; import org.springframework.util.StringUtils; public class RmesException extends Exception { @@ -58,7 +59,7 @@ public RmesException(int status, int errorCode, String message, JSONObject detai this.details=det.toString(); } - public RmesException(HttpStatus status, String message, String details) { + public RmesException(HttpStatusCode status, String message, String details) { super(); this.status = status.value(); this.details = createDetails(null, message, details); diff --git a/src/main/java/fr/insee/rmes/exceptions/RmesExceptionIO.java b/src/main/java/fr/insee/rmes/exceptions/RmesExceptionIO.java index 1414795f..a9566cb1 100644 --- a/src/main/java/fr/insee/rmes/exceptions/RmesExceptionIO.java +++ b/src/main/java/fr/insee/rmes/exceptions/RmesExceptionIO.java @@ -4,12 +4,8 @@ public class RmesExceptionIO extends IOException { - /** - * - */ - private static final long serialVersionUID = -5786486664894878082L; - private int status; - private String details; + private final int status; + private final String details; /** * diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/randomUUID.java b/src/main/java/fr/insee/rmes/tocolecticaapi/RandomUUIDExtensionFunction.java similarity index 90% rename from src/main/java/fr/insee/rmes/tocolecticaapi/randomUUID.java rename to src/main/java/fr/insee/rmes/tocolecticaapi/RandomUUIDExtensionFunction.java index 8a14c36c..1d08528e 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/randomUUID.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/RandomUUIDExtensionFunction.java @@ -5,7 +5,7 @@ import java.util.UUID; -public class randomUUID implements ExtensionFunction{ +public class RandomUUIDExtensionFunction implements ExtensionFunction{ @Override public QName getName() { diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/ControllerUtils.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/ControllerUtils.java new file mode 100644 index 00000000..3c11fe83 --- /dev/null +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/ControllerUtils.java @@ -0,0 +1,26 @@ +package fr.insee.rmes.tocolecticaapi.controller; + +import org.springframework.core.io.Resource; +import org.springframework.http.ContentDisposition; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; + +import java.io.IOException; + +public class ControllerUtils { + private ControllerUtils() {} + + static ResponseEntity xmltoResponseEntity(Resource response, String fileNameWithExtension, MediaType contentType) throws IOException { + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.attachment().filename(fileNameWithExtension).build().toString()) + .contentLength(response.contentLength()) + .contentType(contentType) + .body(response); + } + + static ResponseEntity xmltoResponseEntity(String xmlContent) { + return ResponseEntity.ok(xmlContent.replace("\\\"", "\"").replace("", "")); + } + +} diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java index 77f97c96..115eab28 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import fr.insee.rmes.exceptions.ExceptionColecticaUnreachable; +import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.exceptions.RmesExceptionIO; import fr.insee.rmes.model.DDIItemType; import fr.insee.rmes.tocolecticaapi.fragments.DdiFragment; @@ -16,10 +17,8 @@ import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; -import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; -import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -63,11 +62,13 @@ public ResponseEntity findFragmentByUuidColectica( description = "id de l'objet colectica sous la forme uuid/version", required = true, schema = @Schema( - type = "string", example="d6c08ec1-c4d2-4b9a-b358-b23aa4e0af93")) String uuid) throws ExceptionColecticaUnreachable, IOException { - return colecticaService.findFragmentByUuid(uuid); + type = "string", example="d6c08ec1-c4d2-4b9a-b358-b23aa4e0af93")) String uuid) throws RmesException { + return getResponseEntitySearchColecticaFragmentByUuid(colecticaService.findFragmentByUuid(uuid)); } + + @GetMapping(value = "FragmentInstance/uuid/withChildren", produces = MediaType.APPLICATION_XML_VALUE) @Operation(summary = "Get Fragments by uuid", description = "Get an XML document for a Fragment:Instance from Colectica repository.") public String findFragmentByUuidWithChildrenColectica( @@ -162,22 +163,10 @@ public Object getJsonWithChild( return colecticaService.getJsonWithChild(identifier, outputField, fieldLabelName); } - @GetMapping("/RessourcePackageToJson") - public String convertXmlToJson( + @GetMapping(value = "/RessourcePackageToJson", produces = MediaType.APPLICATION_JSON_VALUE) + public String getRessourcePackage( @RequestParam(name = "uuid", required = true) String uuid) throws ExceptionColecticaUnreachable, JsonProcessingException, RmesExceptionIO, ParseException { - return colecticaService.convertXmlToJson(uuid); - } - - @PutMapping ("/replace-xml-parameters") - @PreAuthorize("hasRole('Administrateur_RMESGOPS')") - @Operation(summary = "Modify a fragment DDI", description = "Modify a fragment DDI. All field need to be filled with the same data if there are no changes, except for the version number, which takes a plus 1.") - public String replaceXmlParameters(@RequestBody String inputXml, - @RequestParam ("Type") DDIItemType type, - @RequestParam ("Label") String label, - @RequestParam ("Version") int version, - @RequestParam ("Name") String name, - @RequestParam ("VersionResponsibility") String idepUtilisateur) { - return colecticaService.replaceXmlParameters(inputXml, type, label, version, name, idepUtilisateur); + return colecticaService.getRessourcePackage(uuid); } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java index 2e0b45ef..9c91a2ae 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java @@ -6,6 +6,7 @@ import fr.insee.rmes.model.DDIItemType; import fr.insee.rmes.tocolecticaapi.models.TransactionType; import fr.insee.rmes.tocolecticaapi.service.ColecticaService; +import fr.insee.rmes.utils.FileExtension; import fr.insee.rmes.utils.FilesUtils; import io.swagger.v3.oas.annotations.Hidden; import io.swagger.v3.oas.annotations.Operation; @@ -20,15 +21,14 @@ import org.json.simple.parser.ParseException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.Resource; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; +import org.springframework.http.*; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.security.oauth2.core.oidc.user.OidcUser; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.util.UriUtils; import java.io.BufferedInputStream; import java.io.File; @@ -36,10 +36,13 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.UUID; + +import static fr.insee.rmes.tocolecticaapi.controller.ControllerUtils.xmltoResponseEntity; @Controller @RequestMapping("/postItem") -@Tag(name= "DEMO-Colectica",description = "Services for upgrade Colectica-API") +@Tag(name = "DEMO-Colectica", description = "Services for upgrade Colectica-API") @SecurityRequirement(name = "bearerAuth") @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "Success"), @@ -54,66 +57,27 @@ public class PostItem { static final Logger logger = LogManager.getLogger(PostItem.class); + public static final MediaType APPLICATION_ODT = MediaType.valueOf("application/vnd.oasis.opendocument.text"); private final ColecticaService colecticaService; + @Autowired - public PostItem( ColecticaService colecticaService) { + public PostItem(ColecticaService colecticaService) { this.colecticaService = colecticaService; } - @PostMapping(value = "/transformJsonToJsonForAPi", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - @Operation(summary = "transform an JSON Codelist (Id,Label) to an another json for Colectica API ", - description = "tranform a codeList in json to another json with DDI item inside") - @PreAuthorize("hasRole('Administrateur_RMESGOPS')") - public ResponseEntity transformFile(@RequestParam("file") MultipartFile file, - @RequestParam("nom metier") String idValue, - @RequestParam("label") String nomenclatureName, - @RequestParam("description") String suggesterDescription, - @RequestParam(value = "version",defaultValue = "1") String version, - @RequestParam("idepUtilisateur") String idepUtilisateur, - // peut-être lire le jeton pour recup le timbre directement - @RequestParam("timbre") String timbre, @AuthenticationPrincipal - OidcUser principal) { - - return colecticaService.transformFile(file, idValue, nomenclatureName, suggesterDescription, version, idepUtilisateur, timbre); - } - - @PostMapping(value = "/transformJsonToJsonForApiForComplexCodeList", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - @Operation(summary = "transform an JSON Codelist with multiple field for one id to an another json for Colectica API ", - description = "tranform a codeList in json to another json with DDI item inside") - @PreAuthorize("hasRole('Administrateur_RMESGOPS')") - public ResponseEntity transformFileForComplexCodeList(@RequestParam("file") MultipartFile file, - @RequestParam("nom metier") String idValue, - @RequestParam("label") String nomenclatureName, - @RequestParam("description") String suggesterDescription, - @RequestParam(value = "version",defaultValue = "1") String version, - @RequestParam("idepUtilisateur") String idepUtilisateur, - // peut-être lire le jeton pour recup le timbre directement et l'idep - @RequestParam("timbre") String timbre, - @RequestParam("principale") String principale, - @RequestParam("secondaire") List secondaire, - @RequestParam("labelSecondaire") List labelSecondaire, @AuthenticationPrincipal - OidcUser principal) { - - return colecticaService.transformFileForComplexList(file, idValue, nomenclatureName, suggesterDescription, version, idepUtilisateur, timbre,principale,secondaire,labelSecondaire); - } - - @PostMapping("/UpdateToColecticaRepository/{transactionType}") @Operation(summary = "Send an update to Colectica Repository via Colectica API ", description = "Send a json make with /replace-xml-parameters/{Type}/{Label}/{Version}/{Name}/{VersionResponsibility} COPYCOMMIT is for Upload a new object not for update") @PreAuthorize("hasRole('Administrateur_RMESGOPS')") public ResponseEntity sendUpdateColectica( @RequestBody String ddiUpdatingInJson, - @RequestParam("transactionType") TransactionType transactionType - , @AuthenticationPrincipal - OidcUser principal) throws IOException { - return colecticaService.sendUpdateColectica(ddiUpdatingInJson, transactionType); - + @RequestParam("transactionType") TransactionType transactionType) throws RmesException { + colecticaService.sendUpdateColectica(ddiUpdatingInJson, transactionType); + return ResponseEntity.ok("Transaction success"); } - @PostMapping(value = "/upload", consumes = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_JSON_VALUE}) @Operation(summary = "Send suggester JSON to Colectica Repository via Colectica API", description = "Send a suggester JSON to /api/v1/item. This suggester must be simple, a list of Id,Label transform with transformJsonToJsonForAPi") @@ -123,7 +87,7 @@ public ResponseEntity uploadItem(@RequestParam("file") MultipartFile fil throws IOException, ExceptionColecticaUnreachable { String fileContent = new String(file.getBytes()); return colecticaService.sendUpdateColectica(fileContent, TransactionType.COPYCOMMIT); - } + } @Hidden @@ -132,27 +96,27 @@ public ResponseEntity uploadItem(@RequestParam("file") MultipartFile fil @PreAuthorize("hasRole('Administrateur_RMESGOPS')") public ResponseEntity byType( @PathVariable("type") DDIItemType type, @AuthenticationPrincipal - OidcUser principal) + OidcUser principal) throws IOException, ExceptionColecticaUnreachable, ParseException { return colecticaService.getByType(type); } - @PostMapping(value="/operation/codebook", - consumes = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE,"application/vnd.oasis.opendocument.text" }, - produces = {MediaType.MULTIPART_FORM_DATA_VALUE,MediaType.APPLICATION_OCTET_STREAM_VALUE,"application/vnd.oasis.opendocument.text" } + @PostMapping(value = "/operation/codebook", + consumes = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"}, + produces = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"} ) @PreAuthorize("hasRole('Administrateur_RMESGOPS')") @Operation(operationId = "getCodeBook", summary = "Produce a codebook from a DDI") - public ResponseEntity getCodeBook( + public ResponseEntity getCodeBook( @Parameter(schema = @Schema(type = "string", format = "String", description = "Accept")) - @RequestHeader(required=false) String accept, + @RequestHeader(required = false) String accept, @Parameter(schema = @Schema(type = "string", format = "binary", description = "file in DDI")) - @RequestParam(value="file") MultipartFile isDDI, // InputStream isDDI, + @RequestParam(value = "file") MultipartFile isDDI, // InputStream isDDI, @Parameter(schema = @Schema(type = "string", format = "binary", description = "file for structure")) @RequestParam(value = "dicoVar") MultipartFile isCodeBook //InputStream isCodeBook @@ -164,43 +128,36 @@ public ResponseEntity getCodeBook( File codeBookFile; try { ddi = new String(isDDI.getBytes(), StandardCharsets.UTF_8); - codeBookFile = FilesUtils.streamToFile(isCodeBook.getInputStream(), "dicoVar",".odt"); - } catch (IOException e1) { - throw new RmesException(HttpStatus.BAD_REQUEST, e1.getMessage(), "Files can't be read"); - } - ResponseEntity response; - - try { - response=colecticaService.getCodeBookExport(ddi,codeBookFile,accept); + codeBookFile = FilesUtils.streamToFile(isCodeBook.getInputStream(), "dicoVar", ".odt"); + Resource response = colecticaService.exportCodeBookAsOdt(ddi, codeBookFile); logger.debug("Codebook is generated"); + return xmltoResponseEntity(response, "Codebook" + FileExtension.forHeader(accept).extension(), APPLICATION_ODT); + } catch (IOException e) { + throw new RmesException(HttpStatus.BAD_REQUEST, e.getMessage(), "Files can't be read"); } - catch (RmesException e) { - logger.error("Failed to generate codebook {}", e.getMessageAndDetails()); - return returnRmesException(e); - } - return response; } - @PostMapping(value="/operation/codebook/V2", + @PostMapping(value = "/operation/codebook/V2", consumes = {MediaType.MULTIPART_FORM_DATA_VALUE}, - produces = {MediaType.MULTIPART_FORM_DATA_VALUE,MediaType.APPLICATION_OCTET_STREAM_VALUE,"application/vnd.oasis.opendocument.text" } + produces = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"} ) @PreAuthorize("hasRole('Administrateur_RMESGOPS')") @Operation(operationId = "getCodeBookV2", summary = "Produce a codebook from a DDI") - public ResponseEntity getCodeBookV2( + public ResponseEntity getCodeBookV2( @Parameter(schema = @Schema(type = "string", format = "String", description = "Accept")) - @RequestHeader(required=false) String accept, + @RequestHeader(required = false) String accept, - @Parameter(schema = @Schema(type = "string", allowableValues = { "concis" , "concis avec expression" , "scindable" , "non scindable" })) + @Parameter(schema = @Schema(type = "string", allowableValues = {"concis", "concis avec expression", "scindable", "non scindable"})) @RequestParam(value = "dicoVar") String isCodeBook, //InputStream isCodeBook, - @RequestParam(value="file") MultipartFile isDDI // InputStream isDDI, + @RequestParam(value = "file") MultipartFile isDDI // InputStream isDDI, ) throws Exception { - if (isDDI == null) throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Can't generate codebook","Stream is null"); - InputStream ddiInputStream = new BufferedInputStream(isDDI.getInputStream()); + if (isDDI == null) + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Can't generate codebook", "Stream is null"); + InputStream ddiInputStream = new BufferedInputStream(isDDI.getInputStream()); String ddi = new String(ddiInputStream.readAllBytes(), StandardCharsets.UTF_8); String xslPatternFile = null; switch (isCodeBook) { @@ -228,22 +185,23 @@ public ResponseEntity getCodeBookV2( return colecticaService.getCodeBookExportV2(ddi, xslPatternFile); } - @PostMapping(value="/operation/codebook/checkCodeBookContent", - consumes = {MediaType.MULTIPART_FORM_DATA_VALUE,MediaType.APPLICATION_OCTET_STREAM_VALUE,"application/vnd.oasis.opendocument.text"}, - produces = {MediaType.MULTIPART_FORM_DATA_VALUE,MediaType.APPLICATION_OCTET_STREAM_VALUE,"application/vnd.oasis.opendocument.text" } + @PostMapping(value = "/operation/codebook/checkCodeBookContent", + consumes = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"}, + produces = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"} ) @PreAuthorize("hasRole('Administrateur_RMESGOPS')") @Operation(operationId = "getCodeBookCheck", summary = "Check the DDI before made the codebook export") - public ResponseEntity getCodeBookCheck( + public ResponseEntity getCodeBookCheck( - @RequestParam(value="file") MultipartFile isCodeBook // InputStream isDDI, + @RequestParam(value = "file") MultipartFile isCodeBook // InputStream isDDI, ) throws Exception { return colecticaService.getCodeBookCheck(isCodeBook); } + protected ResponseEntity returnRmesException(RmesException e) { logger.error(e.getMessageAndDetails(), e); return ResponseEntity.status(e.getStatus()).contentType(MediaType.TEXT_PLAIN).body(e.getMessage()); diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/RmesExceptionHandler.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/RmesExceptionHandler.java new file mode 100644 index 00000000..6afa8bf2 --- /dev/null +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/RmesExceptionHandler.java @@ -0,0 +1,26 @@ +package fr.insee.rmes.tocolecticaapi.controller; + +import fr.insee.rmes.exceptions.RmesException; +import fr.insee.rmes.exceptions.RmesExceptionIO; +import fr.insee.rmes.transfoxsl.controller.TransformationController; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; + +@ControllerAdvice(assignableTypes = {DeleteItem.class, GetItem.class, PostItem.class, TransformationController.class}) +public class RmesExceptionHandler extends ResponseEntityExceptionHandler { + + @ExceptionHandler({RmesException.class, RmesExceptionIO.class}) + public final ResponseEntity handleRmesException(RmesException exception){ + logger.error(exception.getMessageAndDetails(), exception); + return ResponseEntity.status(exception.getStatus()).body(exception.getMessage()); + } + + @ExceptionHandler({RmesExceptionIO.class}) + public final ResponseEntity handleRmesExceptionIO(RmesExceptionIO exception){ + logger.error(exception.toRestMessage().getDetails(), exception); + return ResponseEntity.status(exception.toRestMessage().getStatus()).body(exception.getMessage()); + } + +} diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java index 099a1d86..668eb9a2 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java @@ -6,6 +6,7 @@ import fr.insee.rmes.model.DDIItemType; import fr.insee.rmes.tocolecticaapi.models.TransactionType; import fr.insee.rmes.exceptions.RmesExceptionIO; +import org.json.JSONArray; import org.json.simple.parser.ParseException; import org.springframework.core.io.Resource; import org.springframework.http.ResponseEntity; @@ -17,52 +18,28 @@ import java.util.Map; public interface ColecticaService { - ResponseEntity findFragmentByUuid(String uuid) throws ExceptionColecticaUnreachable, IOException; + String findFragmentByUuid(String uuid); String sendDeleteColectica(String uuid, TransactionType transactionType) throws JsonProcessingException, ExceptionColecticaUnreachable, RmesExceptionIO, ParseException; - ResponseEntity findInstanceByUuid(String uuid) throws RmesExceptionIO, ParseException; - String findFragmentByUuidWithChildren(String uuid) throws Exception; - ResponseEntity filteredSearchText(String index, String texte); + String searchColecticaInstanceByUuid(String uuid) throws RmesException; + JSONArray findFragmentByUuidWithChildren(String uuid) throws Exception; + String filteredSearchText(String index, String texte) throws RmesException; - ResponseEntity searchTexteByType(String index, String texte, DDIItemType type); - ResponseEntity searchByType(String index, DDIItemType type); - ResponseEntity>> getJsonWithChild(String identifier, String outputField, String fieldLabelName) throws Exception; + String searchTexteByType(String index, String texte, DDIItemType type) throws RmesException; + String searchByType(String index, DDIItemType type) throws RmesException; + List> getJsonWithChild(String identifier, String outputField, String fieldLabelName) throws Exception; - String convertXmlToJson(String uuid) throws ExceptionColecticaUnreachable, JsonProcessingException, RmesExceptionIO, ParseException; - String replaceXmlParameters(String inputXml, DDIItemType type, String label, int version, String name, String idepUtilisateur); + String getRessourcePackage(String uuid) throws ExceptionColecticaUnreachable, JsonProcessingException, RmesExceptionIO, ParseException; - ResponseEntity getByType(DDIItemType type) throws IOException, ExceptionColecticaUnreachable, ParseException; + String getByType(DDIItemType type) throws IOException, ExceptionColecticaUnreachable, ParseException; - ResponseEntity sendUpdateColectica(String ddiUpdatingInJson, TransactionType transactionType) throws IOException; + void sendUpdateColectica(String ddiUpdatingInJson, TransactionType transactionType) throws RmesException; - ResponseEntity transformFile( - MultipartFile file, - String idValue, - String nomenclatureName, - String suggesterDescription, - String version, - String idepUtilisateur, - String timbre - ); - ResponseEntity transformFileForComplexList( - MultipartFile file, - String idValue, - String nomenclatureName, - String suggesterDescription, - String version, - String idepUtilisateur, - String timbre, - String principale, - List secondaire, - List labelSecondaire - ); - - ResponseEntity getCodeBookExport(String ddiFile, File dicoVar, String acceptHeader) throws RmesException; + Resource exportCodeBookAsOdt(String ddiFile, File dicoVar) throws RmesException; ResponseEntity getCodeBookExportV2(String ddiFile, String xslPatternFile) throws RmesException, Exception; ResponseEntity getCodeBookCheck(MultipartFile isCodeBook) throws RmesException, Exception; - ResponseEntity uploadItem(MultipartFile file) throws IOException, ExceptionColecticaUnreachable; } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java index 106d2249..1edc73ee 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java @@ -8,46 +8,34 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import fr.insee.rmes.config.keycloak.KeycloakServices; -import fr.insee.rmes.exceptions.ExceptionColecticaUnreachable; import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.exceptions.RmesExceptionIO; import fr.insee.rmes.model.DDIItemType; -import fr.insee.rmes.tocolecticaapi.models.*; -import fr.insee.rmes.tocolecticaapi.randomUUID; -import fr.insee.rmes.transfoxsl.service.XsltTransformationService; +import fr.insee.rmes.tocolecticaapi.models.FragmentData; +import fr.insee.rmes.tocolecticaapi.models.NamespaceContextMap; +import fr.insee.rmes.tocolecticaapi.models.RessourcePackage; +import fr.insee.rmes.tocolecticaapi.models.TransactionType; +import fr.insee.rmes.transfoxsl.service.internal.DDIDerefencer; import fr.insee.rmes.utils.ExportUtils; -import fr.insee.rmes.utils.FilesUtils; import fr.insee.rmes.utils.XsltUtils; import fr.insee.rmes.utils.export.XDocReport; import lombok.NonNull; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import net.sf.saxon.TransformerFactoryImpl; -import net.sf.saxon.s9api.ExtensionFunction; -import net.sf.saxon.s9api.Processor; import org.json.JSONArray; import org.json.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.ByteArrayResource; import org.springframework.core.io.Resource; -import org.springframework.http.ResponseEntity; +import org.springframework.http.*; import org.springframework.http.client.SimpleClientHttpRequestFactory; import org.springframework.stereotype.Service; -import org.springframework.web.client.HttpClientErrorException; -import org.springframework.web.client.HttpServerErrorException; -import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestClient; import org.springframework.web.client.RestTemplate; import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.server.ResponseStatusException; -import org.springframework.web.util.UriUtils; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; import org.xml.sax.SAXException; import javax.xml.XMLConstants; @@ -55,551 +43,200 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.*; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import javax.xml.transform.stream.StreamSource; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; import java.io.*; +import java.net.URI; import java.net.URLEncoder; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; -import static org.springframework.security.oauth2.core.endpoint.OAuth2ParameterNames.ERROR; +import static fr.insee.rmes.transfoxsl.utils.RestClientUtils.readBodySafely; @Service -@RequiredArgsConstructor @Slf4j -public class ColecticaServiceImpl implements ColecticaService { +public record ColecticaServiceImpl(@NonNull KeycloakServices kc, + ElasticService elasticService, + RestClient restClient, + XDocReport xdr, + ExportUtils exportUtils, + DDIDerefencer ddiDerefencer, + @Value("${fr.insee.rmes.api.remote.metadata.url}") + String serviceUrl, + @Value("${fr.insee.rmes.api.remote.metadata.agency}") + String agency + +) implements ColecticaService { - private static final String CONTENT_TYPE = "Content-Type"; - private static final String APPLICATION_XML = "application/xml"; - private static final String APPLICATION_JSON = "application/json"; - private static final String AUTHORIZATION = "Authorization"; private static final String BEARER = "Bearer "; - private static final String VERSION = "version"; - private static final String SEARCH = "/_search"; - private static final String APIKEYHEADER = "apiKey "; - private static final String ERREUR_COLECTICA = "Une erreur s'est produite lors de la requête vers Colectica."; - private static final String ERREUR_ELASTICSEARCH = "Une erreur s'est produite lors de la requête Elasticsearch."; private static final String TRANSACTIONID = "{\"TransactionId\":"; - private static final String ATTACHMENT = "attachment"; - - public static final String DISALLOW_DOCTYPE_DECL = "http://javax.xml.transform.TransformerFactory/feature/disallow-doctype-decl"; - public static final String DISALLOW_DOCTYPE_DECL1 = "http://apache.org/xml/features/disallow-doctype-decl"; - public static final String EXTERNAL_GENERAL_ENTITIES = "http://xml.org/sax/features/external-general-entities"; - public static final String EXTERNAL_PARAMETER_ENTITIES = "http://xml.org/sax/features/external-parameter-entities"; - - - @NonNull - private final KeycloakServices kc; - - @Value("${auth.api.url}") - private String authApiUrl; - - - @Value("${auth.username}") - private String username; - - @Value("${auth.password}") - private String password; - - @Value("${fr.insee.rmes.api.remote.metadata.url}") - private String serviceUrl; - - @Value("${fr.insee.rmes.api.remote.metadata.agency}") - private String agency; - - @Value("${fr.insee.rmes.elasticsearch.host}") - private String elasticHost; - - @Value("${fr.insee.rmes.elasticsearch.port}") - private int elasticHostPort; - @Value("${fr.insee.rmes.elasticsearch.url}") - private String elasticUrl; - - @Value("${fr.insee.rmes.elasticsearch.apikey}") - private String apiKey; - - public RestTemplate restTemplate; - @Autowired - private final XDocReport xdr; - @Autowired - private final ExportUtils exportUtils; - - @Autowired - private final XsltTransformationService xsltTransformationService; - - @Override - public ResponseEntity findFragmentByUuid(String uuid) throws ExceptionColecticaUnreachable, IOException { - ResponseEntity responseEntity = searchColecticaFragmentByUuid(uuid); - if (responseEntity.getStatusCode().is2xxSuccessful()) { - String responseBody = responseEntity.getBody(); - return ResponseEntity.ok(responseBody); - } else { - return responseEntity; - } - } - - private ResponseEntity searchColecticaFragmentByUuid(String uuid) throws - ExceptionColecticaUnreachable, IOException { - try (HttpClient httpClient = HttpClient.newHttpClient()) { - HttpRequest getRequest = getGetSearchColecticaFragmentByUuid(uuid); - - try (CloseableHttpResponse response = httpClient.execute(httpGet)) { - return getResponseEntitySearchColecticaFragmentByUuid(response); - } catch (IOException e) { - throw new IOException("Erreur IO lors de la récupération des données pour UUID: " + uuid, e); - } - } catch (ExceptionColecticaUnreachable e) { - throw new ExceptionColecticaUnreachable("Le service Colectica est injoignable pour UUID: " + uuid, e); - } catch (ParseException e) { - throw new RmesExceptionIO(400,"erreur durant la lecture du jeton",""); - } - } - - private HttpRequest getGetSearchColecticaFragmentByUuid(String uuid) throws ExceptionColecticaUnreachable, JsonProcessingException, RmesExceptionIO, ParseException { - String url = String.format("%s/api/v1/item/%s/%s", serviceUrl, agency, uuid); - HttpRequest httpGet = HttpRequest.newBuilder().build(); - httpGet.headers() - httpGet.addHeader(CONTENT_TYPE, APPLICATION_XML); - httpGet.addHeader("Accept", APPLICATION_JSON); - - httpGet.setHeader(AUTHORIZATION, BEARER + resolveBearerToken()); - return httpGet; - } - - private String resolveBearerToken() throws ExceptionColecticaUnreachable, JsonProcessingException, RmesExceptionIO, ParseException { - if (serviceUrl.contains("kube")) { - return extractAccessToken(getAuthToken()); - } - return kc.getFreshToken(); - } - - protected static ResponseEntity getResponseEntitySearchColecticaFragmentByUuid(CloseableHttpResponse response) throws IOException { - if (response.getStatusLine().getStatusCode() != HttpStatus.OK.value()) { - return ResponseEntity.status(response.getStatusLine().getStatusCode()) - .body("Erreur lors de la requête vers Colectica."); - } - - String responseBody = EntityUtils.toString(response.getEntity()); - JSONObject jsonResponse = new JSONObject(responseBody); - String xmlContent = jsonResponse.get("Item").toString(); - xmlContent = xmlContent.replace("\\\"", "\"").replace("", ""); - return ResponseEntity.ok(xmlContent); - } - - - @Override - public ResponseEntity findInstanceByUuid(String uuid) throws RmesExceptionIO, ParseException { - ResponseEntity responseEntity = searchColecticaInstanceByUuid(uuid); - if (responseEntity.getStatusCode().is2xxSuccessful()) { - String responseBody = responseEntity.getBody(); - return ResponseEntity.ok(responseBody); - } else { - return responseEntity; - } + private RestClient initRestClient() { + SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); + factory.setConnectTimeout(100_000); // Temps de connexion en millisecondes + factory.setReadTimeout(100_000); // Temps de lecture en millisecondes + + return RestClient.builder() + .requestFactory(factory) + .baseUrl(URI.create(serviceUrl + "/").resolve("api/v1/")) + .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_XML_VALUE) + .defaultHeader(HttpHeaders.AUTHORIZATION, BEARER) + .defaultHeaders(headers -> headers.setBearerAuth(kc.getFreshToken())) + .build(); } @Override - public String findFragmentByUuidWithChildren(String uuid) throws Exception { - // Appel au service Colectica pour obtenir l'instance - ResponseEntity responseEntity = searchColecticaInstanceByUuid(uuid); + public String findFragmentByUuid(String uuid) { + String response = getWithRestClient(URI.create("item/" + agency + "/" + uuid), MediaType.APPLICATION_JSON); + return (new JSONObject(response)).getString("Item"); - if (responseEntity.getStatusCode().is2xxSuccessful()) { - String responseBody = (String) responseEntity.getBody(); - - // Transformer la réponse XML avec le service de déréférencement - String dereferencedXml = dereferenceXml(responseBody); - - // Convertir le XML transformé en InputStream pour l'analyse - InputStream inputStream = new ByteArrayInputStream(dereferencedXml.getBytes(StandardCharsets.UTF_8)); - - // Appeler la méthode parseDereferencedXmlWithEnum pour générer le tableau JSON - JSONArray resultArray = parseDereferencedXmlWithEnum(inputStream); - - // Retourner le résultat final en JSON formaté - return resultArray.toString(4); - } else { - return "L'objet n'existe pas"; - } } - // Nouvelle méthode pour déréférencer le XML directement - private String dereferenceXml(String xmlResponse) throws Exception { - // Convertir le XML en InputStream - InputStream inputStream = new ByteArrayInputStream(xmlResponse.getBytes(StandardCharsets.UTF_8)); - // Appeler le service de transformation XSLT pour effectuer le déréférencement - List outputText = xsltTransformationService.transform(inputStream, DEREFERENCE_XSL, false); - - // Combiner le résultat en une seule chaîne de caractères - return String.join("\n", outputText); + private String getWithRestClient(URI relativeUri, MediaType acceptedMediaType) { + return restClient.get().uri(relativeUri) + .accept(acceptedMediaType) + .retrieve() + .onStatus(HttpStatusCode::is4xxClientError, (request, response) -> { + throw new RmesExceptionIO(response.getStatusCode().value(), "Bad request or inexisting resource", readBodySafely(response)); + }) + .onStatus(HttpStatusCode::is5xxServerError, (request, response) -> { + throw new RmesExceptionIO(response.getStatusCode().value(), "Error while calling Colectica", readBodySafely(response)); + }) + .body(String.class); } - public JSONArray parseDereferencedXmlWithEnum(InputStream inputStream) throws Exception { - // Créer et configurer le DocumentBuilderFactory de manière sécurisée - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - - // Désactiver l'accès aux entités externes pour éviter les attaques XXE - factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - factory.setFeature("http://xml.org/sax/features/external-general-entities", false); - factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - - // Construire le document XML - DocumentBuilder builder = factory.newDocumentBuilder(); - Document document = builder.parse(inputStream); - - // Initialiser XPath avec le contexte de l'espace de noms - XPathFactory xpathFactory = XPathFactory.newInstance(); - XPath xpath = xpathFactory.newXPath(); - - // Configurer les espaces de noms en utilisant NamespaceContextMap existant - NamespaceContext nsContext = new NamespaceContextMap( - "ddi", "ddi:instance:3_3", - "r", "ddi:reusable:3_3" - ); - xpath.setNamespaceContext(nsContext); - - JSONArray jsonArray = new JSONArray(); - - // Parcourir toutes les balises dans le document XML - NodeList allNodes = document.getElementsByTagName("*"); - - for (int i = 0; i < allNodes.getLength(); i++) { - Element element = (Element) allNodes.item(i); - String tagName = element.getLocalName(); // Utiliser getLocalName pour ignorer l'espace de noms - // Vérifier si le nom de la balise existe dans l'enum - DDIItemType itemType = DDIItemType.searchByName(tagName); - - if (itemType != null) { - // Extraire les informations r:ID, r:Agency, r:Version en tenant compte de l'espace de noms - String identifier = xpath.evaluate(".//r:ID", element); - String agencyId = xpath.evaluate(".//r:Agency", element); - String version = xpath.evaluate(".//r:Version", element); - - // Si les valeurs sont trouvées, les ajouter à l'objet JSON - JSONObject jsonObject = new JSONObject(); - jsonObject.put("Identifier", identifier != null ? identifier : ""); - jsonObject.put("AgencyId", agencyId != null ? agencyId : ""); - jsonObject.put("Version", version != null ? version : ""); - jsonObject.put("ItemType", itemType.getName()); - - // Ajouter l'objet JSON au tableau - jsonArray.put(jsonObject); - } - } - - return jsonArray; + private String postWithRestClient(URI relativeUri, Optional requestBody, MediaType contentType) { + RestClient.RequestBodySpec postRequest = restClient.post() + .uri(relativeUri) + .contentType(contentType); + requestBody.ifPresent(postRequest::body); + return postRequest + .retrieve() + .onStatus(HttpStatusCode::is4xxClientError, (request, response) -> { + throw new RmesExceptionIO(response.getStatusCode().value(), "Bad request or inexisting resource", readBodySafely(response)); + }) + .onStatus(HttpStatusCode::is5xxServerError, (request, response) -> { + throw new RmesExceptionIO(response.getStatusCode().value(), "Error while calling Colectica", readBodySafely(response)); + }) + .body(String.class); } - - - public Set extractUniqueIdentifiers(String xmlResponse) throws Exception { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - - // Désactivation de l'accès aux entités externes pour la sécurité (prévention des attaques XXE) - factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - factory.setFeature(DISALLOW_DOCTYPE_DECL1, true); - factory.setFeature(EXTERNAL_GENERAL_ENTITIES, false); - factory.setFeature(EXTERNAL_PARAMETER_ENTITIES, false); - - factory.setNamespaceAware(true); - - DocumentBuilder builder = factory.newDocumentBuilder(); - Document document = builder.parse(new ByteArrayInputStream(xmlResponse.getBytes())); - - XPathFactory xpathFactory = XPathFactory.newInstance(); - XPath xpath = xpathFactory.newXPath(); - - xpath.setNamespaceContext(new NamespaceContextMap( - "r", "ddi:reusable:3_3", - "ddi", "ddi:instance:3_3" - )); - - Set identifiers = new HashSet<>(); - - // Extraire tous les UUIDs - NodeList idNodes = (NodeList) xpath.evaluate("//r:ID", document, XPathConstants.NODESET); - - for (int i = 0; i < idNodes.getLength(); i++) { - identifiers.add(idNodes.item(i).getTextContent()); + @Override + public JSONArray findFragmentByUuidWithChildren(String uuid) throws RmesException { + String instance = searchColecticaInstanceByUuid(uuid); + try { + return XmlProcessing.parseDereferencedXmlWithEnum(this.ddiDerefencer.intermediateDereference(instance)); + } catch (ParserConfigurationException | IOException | SAXException | XPathExpressionException e) { + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Error while processing XML from Colectica", e.getMessage()); } - - return identifiers; } - public JSONArray processUuidsAndFetchData(Set uuids) throws ExceptionColecticaUnreachable, IOException { - JSONArray dataArray = new JSONArray(); + static class XmlProcessing { - for (String uuid : uuids) { - ResponseEntity responseEntity = searchColecticaFragmentByUuid(uuid); - if (responseEntity.getStatusCode().is2xxSuccessful()) { - String fragmentXml = (String) responseEntity.getBody(); - if (fragmentXml != null) { - JSONObject fragmentData = extractDataFromFragment(fragmentXml); - dataArray.put(fragmentData); - } - } else { - String message = "Erreur ou élément non trouvé pour UUID:" + uuid; - log.log( Level.ERROR,message); - } - } - - return dataArray; - } - - private JSONObject extractDataFromFragment(String fragmentXml) { - try { + private static JSONArray parseDereferencedXmlWithEnum(byte[] input) throws ParserConfigurationException, IOException, SAXException, XPathExpressionException { + // Créer et configurer le DocumentBuilderFactory de manière sécurisée DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); - // Désactiver l'accès aux entités externes pour des raisons de sécurité (prévention des attaques XXE) + // Désactiver l'accès aux entités externes pour éviter les attaques XXE factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - factory.setFeature(DISALLOW_DOCTYPE_DECL1, true); - factory.setFeature(EXTERNAL_GENERAL_ENTITIES, false); - factory.setFeature(EXTERNAL_PARAMETER_ENTITIES, false); - - factory.setNamespaceAware(true); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + // Construire le document XML DocumentBuilder builder = factory.newDocumentBuilder(); - Document document = builder.parse(new ByteArrayInputStream(fragmentXml.getBytes(StandardCharsets.UTF_8))); + Document document = builder.parse(new ByteArrayInputStream(input)); + // Initialiser XPath avec le contexte de l'espace de noms XPathFactory xpathFactory = XPathFactory.newInstance(); XPath xpath = xpathFactory.newXPath(); - xpath.setNamespaceContext(new NamespaceContextMap("r", "ddi:reusable:3_3", "ddi", "ddi:instance:3_3")); + // Configurer les espaces de noms en utilisant NamespaceContextMap existant + NamespaceContext nsContext = new NamespaceContextMap( + "ddi", "ddi:instance:3_3", + "r", "ddi:reusable:3_3" + ); + xpath.setNamespaceContext(nsContext); - Node firstElement = (Node) xpath.evaluate("/*/*[1]", document, XPathConstants.NODE); + JSONArray jsonArray = new JSONArray(); - String itemType = firstElement.getLocalName(); - String id = xpath.evaluate("//r:ID", firstElement); - String agencyFragment = xpath.evaluate("//r:Agency", firstElement); - String version = xpath.evaluate("//r:Version", firstElement); + // Parcourir toutes les balises dans le document XML + NodeList allNodes = document.getElementsByTagName("*"); - JSONObject fragmentData = new JSONObject(); - fragmentData.put("Identifier", id); - fragmentData.put("AgencyId", agencyFragment); - fragmentData.put("Version", version); - fragmentData.put("ItemType", itemType); + for (int i = 0; i < allNodes.getLength(); i++) { + Element element = (Element) allNodes.item(i); + String tagName = element.getLocalName(); // Utiliser getLocalName pour ignorer l'espace de noms - return fragmentData; - } catch (ParserConfigurationException | SAXException | IOException | XPathExpressionException e) { - // Log the exception - log.error("Error extracting data from fragment XML: {}", e.getMessage()); + // Vérifier si le nom de la balise existe dans l'enum + DDIItemType itemType = DDIItemType.searchByName(tagName); - // Return an appropriate response to the client - return new JSONObject(); - } - } + if (itemType != null) { + // Extraire les informations r:ID, r:Agency, r:Version en tenant compte de l'espace de noms + String identifier = xpath.evaluate(".//r:ID", element); + String agencyId = xpath.evaluate(".//r:Agency", element); + String version = xpath.evaluate(".//r:Version", element); + // Si les valeurs sont trouvées, les ajouter à l'objet JSON + JSONObject jsonObject = new JSONObject(); + jsonObject.put("Identifier", identifier != null ? identifier : ""); + jsonObject.put("AgencyId", agencyId != null ? agencyId : ""); + jsonObject.put("Version", version != null ? version : ""); + jsonObject.put("ItemType", itemType.getName()); - private ResponseEntity searchColecticaInstanceByUuid(String uuid) throws RmesExceptionIO, ParseException { - // Configuration pour ignorer les cookies invalides - RequestConfig globalConfig = RequestConfig.custom() - .setCookieSpec(CookieSpecs.IGNORE_COOKIES) - .build(); - - try (CloseableHttpClient httpClient = HttpClients.custom() - .setDefaultRequestConfig(globalConfig) - .build()) { + // Ajouter l'objet JSON au tableau + jsonArray.put(jsonObject); + } + } - return getStringResponseEntity(uuid, httpClient); - } catch (IOException e) { - throw new RmesExceptionIO(400, "Erreur lors de la communication avec le service : " + e.getMessage(), ""); - } catch (ExceptionColecticaUnreachable e) { - throw new RmesExceptionIO(400, "Le service Colectica est injoignable : " + e.getMessage(), ""); + return jsonArray; } } - protected ResponseEntity getStringResponseEntity(String uuid, CloseableHttpClient httpClient) throws ExceptionColecticaUnreachable, JsonProcessingException, RmesExceptionIO, ParseException { - if (!uuid.matches("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89ABab][0-9a-fA-F]{3}-[0-9a-fA-F]{12}(\\/\\d+)?$")) { - throw new IllegalArgumentException("UUID invalide"); - } - HttpGet httpGet = getHttpGet(uuid); + @Override + public String searchColecticaInstanceByUuid(String uuid) throws RmesException { - try (CloseableHttpResponse response = httpClient.execute(httpGet)) { - return getStringResponseEntity(response); - } catch (IOException e) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body(ERREUR_COLECTICA); + if (!uuid.matches("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89ABab][0-9a-fA-F]{3}-[0-9a-fA-F]{12}(\\/\\d+)?$")) { + throw new RmesException(HttpStatus.BAD_REQUEST, "UUID invalide", uuid + " ne respecte pas le pattern d'un uuid"); } + return getWithRestClient(URI.create("ddiset/" + agency + "/" + uuid), MediaType.APPLICATION_XML); } - private HttpGet getHttpGet(String uuid) throws ExceptionColecticaUnreachable, JsonProcessingException, RmesExceptionIO, ParseException { - HttpGet httpGet; - String authentToken; - String url = String.format("%s/api/v1/ddiset/%s/%s", serviceUrl, agency, uuid); - httpGet = new HttpGet(url); - httpGet.addHeader(CONTENT_TYPE, APPLICATION_XML); - httpGet.addHeader("Accept", APPLICATION_XML); - if (!serviceUrl.contains("kube")){ - httpGet.setHeader(AUTHORIZATION, BEARER + getFreshToken()); - } else { - String token2 = getAuthToken(); - authentToken = extractAccessToken(token2); - httpGet.setHeader(AUTHORIZATION, BEARER + authentToken); - } - return httpGet; - } - - private static ResponseEntity getStringResponseEntity(CloseableHttpResponse response) throws IOException { - byte[] bytes = EntityUtils.toByteArray(response.getEntity()); - String responseBody = new String(bytes, StandardCharsets.UTF_8); - return ResponseEntity.ok(responseBody); - } - - @Override - public ResponseEntity filteredSearchText(String index, String texte) { - ResponseEntity responseEntity = searchText(index, texte); - if (responseEntity.getStatusCode().is2xxSuccessful()) { - String responseBody = responseEntity.getBody(); - String filteredResponse = filterAndTransformResponse(responseBody); - log.info("Réponse filtrée avec succès pour searchTexte"); - return ResponseEntity.ok(filteredResponse); - } else { - return responseEntity; - } - } @Override - public ResponseEntity searchByType(String index, DDIItemType type) { - ResponseEntity responseEntity = searchType(index, String.valueOf(type.getUUID()).toLowerCase()); - if (responseEntity.getStatusCode().is2xxSuccessful()) { - String responseBody = responseEntity.getBody(); - String filteredResponse = filterAndTransformResponse(responseBody); - log.info("Réponse filtrée avec succès pour searchByType"); - return ResponseEntity.ok(filteredResponse); - } else { - return responseEntity; - } + public String filteredSearchText(String index, String texte) throws RmesException { + String filteredResponse = filterAndTransformResponse( + elasticService.search(index, URLEncoder.encode(texte, StandardCharsets.UTF_8)) + ); + log.info("Réponse filtrée avec succès pour searchTexte"); + return filteredResponse; } @Override - public ResponseEntity searchTexteByType(String index, String texte, DDIItemType type) { - ResponseEntity responseEntity = searchTextByType(index, texte, String.valueOf(type.getUUID()).toLowerCase()); - if (responseEntity.getStatusCode().is2xxSuccessful()) { - String responseBody = responseEntity.getBody(); - String filteredResponse = filterAndTransformResponse(responseBody); - log.info("Réponse filtrée avec succès pour searchTexteByType"); - return ResponseEntity.ok(filteredResponse); - } else { - return responseEntity; - } - } - - - private ResponseEntity searchType(String index,String type) { - try { - HttpPost httpPost; - String jsonBody = "{\"query\": {\"match\": {\"itemType\":\""+ type + "\"}}, \"_source\": true, \"size\": 10000, \"from\": 0}"; - StringEntity entity = new StringEntity(jsonBody, ContentType.APPLICATION_JSON); - - httpPost = getHttpPostToElastic(index, entity); - - - try (CloseableHttpResponse response = httpClient.execute(httpPost)) { - String responseBody = EntityUtils.toString(response.getEntity()); - - return ResponseEntity.ok(responseBody); - } - } catch (IOException e) { - return ResponseEntity.status(500).body(ERREUR_ELASTICSEARCH); - } - } + public String searchByType(String index, DDIItemType type) throws RmesException { + String filteredResponse = filterAndTransformResponse(elasticService.searchType(index, type)); + log.info("Réponse filtrée avec succès pour searchByType"); + return filteredResponse; - private HttpPost getHttpPostToElastic(String index, StringEntity entity) { - HttpPost httpPost; - if (elasticHost.contains("kube")) { - httpPost = new HttpPost("https://" + elasticHost + ":" + elasticHostPort + "/" + index + SEARCH); - httpPost.setEntity(entity); - } - else { - httpPost = new HttpPost(elasticUrl + "/" + index + SEARCH); - httpPost.setHeader(AUTHORIZATION, APIKEYHEADER + apiKey); - httpPost.setHeader(CONTENT_TYPE, APPLICATION_JSON); - httpPost.setEntity(entity); - } - return httpPost; } - private ResponseEntity searchText(String index, String texte) { - try { - String encodedTexte = URLEncoder.encode(texte, StandardCharsets.UTF_8.toString()); - HttpGet httpGet = getHttpGetToElastic(index,encodedTexte); - - try (CloseableHttpResponse response = httpClient.execute(httpGet)) { - String responseBody = EntityUtils.toString(response.getEntity()); - return ResponseEntity.ok(responseBody); - } - } catch (IOException e) { - e.printStackTrace(); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Erreur lors de la connexion au service."); - } - } - - private HttpGet getHttpGetToElastic(String index, String encodedTexte) { - HttpGet httpGet; - if (elasticHost.contains("kube")) { - httpGet = new HttpGet("https://" + elasticHost + ":" + elasticHostPort + "/" + index + "/_search?q=" + encodedTexte); - } else { - httpGet = new HttpGet(elasticUrl + "/" + index + "/_search?q=*" + encodedTexte + "*"); - httpGet.addHeader(AUTHORIZATION, APIKEYHEADER + apiKey); - } - return httpGet; - } - - private ResponseEntity searchTextByType(String index, String texte, String type) { - try { - HttpPost httpPost = new HttpPost(elasticUrl + "/" + index + SEARCH); - - if (!elasticHost.contains("kube")) { - httpPost.setHeader(AUTHORIZATION, APIKEYHEADER + apiKey); - httpPost.setHeader(CONTENT_TYPE, APPLICATION_JSON); - } - - // Construction de la requête wildcard pour chercher 'texte' dans tous les champs indexés - StringEntity entity = new StringEntity( - "{\n" + - " \"query\": {\n" + - " \"bool\": {\n" + - " \"must\": [\n" + - " { \"match\": { \"itemType\":\"" + type + "\" }},\n" + - " { \"query_string\": {\n" + - " \"query\": \"*" + texte + "*\",\n" + - " \"fields\": [\"*\"]\n" + - " }\n" + - " }\n" + - " ]\n" + - " }\n" + - " }\n" + - "}", ContentType.APPLICATION_JSON); - - httpPost.setEntity(entity); - - try (CloseableHttpResponse response = httpClient.execute(httpPost)) { - String responseBody = EntityUtils.toString(response.getEntity()); - return ResponseEntity.ok(responseBody); - } - } catch (IOException e) { - return ResponseEntity.status(500).body(ERREUR_ELASTICSEARCH); - } + @Override + public String searchTexteByType(String index, String texte, DDIItemType type) throws RmesException { + String filteredResponse = filterAndTransformResponse(elasticService.searchTextByType(index, texte, type)); + log.info("Réponse filtrée avec succès pour searchTexteByType"); + return filteredResponse; } - private String filterAndTransformResponse(String responseBody) { + private static String filterAndTransformResponse(String json) throws RmesException { try { ObjectMapper objectMapper = new ObjectMapper(); - JsonNode jsonResponse = objectMapper.readTree(responseBody); + JsonNode jsonResponse = objectMapper.readTree(json); JsonNode hitsArray = jsonResponse.path("hits").path("hits"); ArrayNode filteredHitsArray = objectMapper.createArrayNode(); @@ -628,618 +265,115 @@ private String filterAndTransformResponse(String responseBody) { } return filteredHitsArray.toString(); - } catch (IOException e) { - return "Une erreur s'est produite lors de la manipulation du JSON."; + } catch (JsonProcessingException e) { + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Error `" + e.getMessage() + "` while processing json", json); } } @Override - public ResponseEntity>> getJsonWithChild(String identifier, String outputField, String fieldLabelName) throws Exception { - String apiUrl = serviceUrl+"/api/v1/jsonset/fr.insee/" + identifier; + public List> getJsonWithChild(String identifier, String outputField, String fieldLabelName) throws Exception { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + JsonNode jsonNode = objectMapper.readTree(findJsonsetByIdentifier(identifier)); + JsonNode codesNode = jsonNode.get("Codes"); - HttpHeaders headers = new HttpHeaders(); - headers.set(AUTHORIZATION, BEARER + resolveBearerToken()); - HttpEntity requestEntity = new HttpEntity<>(headers); + List> idLabelPairs = new ArrayList<>(); + for (JsonNode codeNode : codesNode) { + String id = codeNode.get("Value").asText(); + String label = codeNode.get("Category").get("Label").get("fr-FR").asText(); - try { - ResponseEntity response = restTemplate.exchange(apiUrl, HttpMethod.GET, requestEntity, String.class); - if (response.getStatusCode() == HttpStatus.OK) { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - JsonNode jsonNode = objectMapper.readTree(response.getBody()); - JsonNode codesNode = jsonNode.get("Codes"); - - List> idLabelPairs = new ArrayList<>(); - for (JsonNode codeNode : codesNode) { - String id = codeNode.get("Value").asText(); - String label = codeNode.get("Category").get("Label").get("fr-FR").asText(); - - Map idLabelPair = new HashMap<>(); - idLabelPair.put(outputField, id); - idLabelPair.put(fieldLabelName, label); - idLabelPairs.add(idLabelPair); - } - - return ResponseEntity.ok(idLabelPairs); - } else { - return ResponseEntity.status(response.getStatusCode()).body(null); - } - } catch (HttpClientErrorException|HttpServerErrorException e) { - return ResponseEntity.status(e.getStatusCode()).body(null); - } catch (Exception e) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(null); + Map idLabelPair = new HashMap<>(); + idLabelPair.put(outputField, id); + idLabelPair.put(fieldLabelName, label); + idLabelPairs.add(idLabelPair); } + return idLabelPairs; } - @Override - public String convertXmlToJson(String uuid) throws ExceptionColecticaUnreachable, JsonProcessingException, RmesExceptionIO, ParseException { - - String apiUrl = serviceUrl + "/api/v1/jsonset/fr.insee/" + uuid; - - - HttpHeaders headers = new HttpHeaders(); - headers.set(AUTHORIZATION, BEARER + token); - HttpEntity requestEntity = new HttpEntity<>(headers); - - ResponseEntity response = restTemplate.exchange( - apiUrl, - HttpMethod.GET, - requestEntity, - String.class - ); - - String result = null; - if (response.getStatusCode() == HttpStatus.OK) { - try { - ObjectMapper mapper = new ObjectMapper(); - RessourcePackage ressourcePackage = mapper.readValue(response.getBody(), RessourcePackage.class); - List> countriesWithCodesAndLabels = ressourcePackage.mapCodesToLabels(); - ObjectMapper mapperFinal= new ObjectMapper(); - result = mapperFinal.writeValueAsString(countriesWithCodesAndLabels); - } catch (RestClientException e) { - // Log the exception - log.error("Error while calling the REST API: {}", e.getMessage()); - // Rethrow the exception or handle it appropriately - throw new ExceptionColecticaUnreachable("Error while calling the REST API", e); - } - } - return result; + private String findJsonsetByIdentifier(String identifier) { + return getWithRestClient(URI.create("jsonset/fr.insee/" + identifier), MediaType.APPLICATION_JSON); } @Override - public String replaceXmlParameters(String inputXml, DDIItemType type, String label, int version, String name, String idepUtilisateur) { - try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - - // Désactivation des entités externes pour des raisons de sécurité (prévention des attaques XXE) - factory.setNamespaceAware(true); - factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - factory.setFeature(DISALLOW_DOCTYPE_DECL1, true); - factory.setFeature(EXTERNAL_GENERAL_ENTITIES, false); - factory.setFeature(EXTERNAL_PARAMETER_ENTITIES, false); - - DocumentBuilder builder = factory.newDocumentBuilder(); - Document document = builder.parse(new InputSource(new StringReader(inputXml))); - - String typeName = type.getName(); - NodeList typeNodes = document.getElementsByTagNameNS("ddi:logicalproduct:3_3", typeName); - - if (typeNodes.getLength() == 0) { - return "Erreur : Aucun élément correspondant trouvé pour le type " + typeName; - } - - Element typeNameDocument = (Element) typeNodes.item(0); - - if (!typeName.equals(typeNameDocument.getNodeName())) { - return "Erreur : Attention ce n'est pas le bon type. L'item chargé n'est pas du type que vous avez sélectionné."; - } - - Document document2 = (Document) document.cloneNode(true); - - NodeList versionNodes = document2.getElementsByTagName("r:Version"); - for (int i = 0; i < versionNodes.getLength(); i++) { - Node versionNode = versionNodes.item(i); - if (versionNode instanceof Element) { - Element versionElement = (Element) versionNode; - versionElement.setTextContent(String.valueOf(version)); - } - } - - Node nameNode = document2.getElementsByTagName("r:String").item(0); - nameNode.setTextContent(name); - - Node labelNode = document2.getElementsByTagName("r:Content").item(0); - labelNode.setTextContent(label); - - NodeList urnNodes = document2.getElementsByTagName("r:URN"); - for (int i = 0; i < urnNodes.getLength(); i++) { - Node urnNode = urnNodes.item(i); - if (urnNode instanceof Element) { - Element urnElement = (Element) urnNode; - String urnCode = urnElement.getTextContent(); - urnElement.setTextContent(urnCode.substring(0, urnCode.lastIndexOf(":")) + ":" + version); - } - } - - // Désactivation des entités externes dans TransformerFactory pour éviter les attaques XXE - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - transformerFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - transformerFactory.setFeature(DISALLOW_DOCTYPE_DECL, true); - - Transformer transformer = transformerFactory.newTransformer(); - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - - StringWriter writer = new StringWriter(); - transformer.transform(new DOMSource(document2), new StreamResult(writer)); - - // Ajout de la transformation XML vers JSON avec XSLT - InputStream xsltStream2 = getClass().getResourceAsStream("/DDIxmltojsonForOneObject.xsl"); - String jsonResult = transformToJson(new ByteArrayResource(writer.toString().getBytes(StandardCharsets.UTF_8)), xsltStream2, idepUtilisateur, version); - - return jsonResult; - - } catch (Exception e) { - return "Error processing XML"; - } - } - - private String transformToJson(Resource resultResource, InputStream xsltFileJson, String idepUtilisateur, int version) throws IOException, TransformerException { - // Créer un transformateur XSLT sécurisé - TransformerFactory factory = TransformerFactory.newInstance(); - - // Désactiver l'accès aux entités externes pour des raisons de sécurité (prévention des attaques XXE) - factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - factory.setFeature(DISALLOW_DOCTYPE_DECL, true); - - Source xslt = new StreamSource(xsltFileJson); - - Transformer transformer = factory.newTransformer(xslt); - transformer.setParameter("idepUtilisateur", idepUtilisateur); - transformer.setParameter(VERSION, version); - - // Effectuer la transformation - StreamSource text = new StreamSource(resultResource.getInputStream()); - StringWriter xmlWriter = new StringWriter(); - StreamResult xmlResult = new StreamResult(xmlWriter); - - transformer.transform(text, xmlResult); - - return xmlWriter.toString(); + public String getRessourcePackage(String uuid) throws JsonProcessingException { + ObjectMapper mapper = new ObjectMapper(); + RessourcePackage ressourcePackage = mapper.readValue(findJsonsetByIdentifier(uuid), + RessourcePackage.class); + List> countriesWithCodesAndLabels = ressourcePackage.mapCodesToLabels(); + ObjectMapper mapperFinal = new ObjectMapper(); + return mapperFinal.writeValueAsString(countriesWithCodesAndLabels); } @Override - public ResponseEntity getByType(DDIItemType type) throws IOException, ExceptionColecticaUnreachable, ParseException { - - HttpHeaders headers = new HttpHeaders(); - headers.set(AUTHORIZATION, BEARER + token); - - try (CloseableHttpClient httpClient = HttpClients.createDefault()) { - String url = serviceUrl + "/api/v1/_query"; - HttpPost httpPost = new HttpPost(url); - httpPost.setHeader(AUTHORIZATION, BEARER + token); - httpPost.setHeader(CONTENT_TYPE, "application/json-patch+json"); - String requestBody = "{ \"ItemTypes\": [\"" + type.getUUID().toLowerCase() + "\"] }"; - httpPost.setEntity(new StringEntity(requestBody)); - - String responseBody; - try (CloseableHttpResponse response = httpClient.execute(httpPost)) { - responseBody = EntityUtils.toString(response.getEntity()); - } catch (IOException e) { - return ResponseEntity.status(500).body(ERREUR_COLECTICA); - } - - return ResponseEntity.ok(responseBody); - } + public String getByType(DDIItemType type) { + return postWithRestClient(URI.create("_query"), + Optional.of("{ \"ItemTypes\": [\"" + type.getUUID().toLowerCase() + "\"] }"), + MediaType.valueOf("application/json-patch+json")); } @Override - public ResponseEntity sendUpdateColectica(String ddiUpdatingInJson, TransactionType transactionType) { - try { - RestTemplate restTemplate=new RestTemplate(); - String initTransactionUrl = serviceUrl + "/api/v1/transaction"; - String authentToken=resolveBearerToken(); - - HttpHeaders initTransactionHeaders = new HttpHeaders(); - initTransactionHeaders.setContentType(MediaType.APPLICATION_JSON); - if (authentToken != null && !authentToken.isEmpty()) { - initTransactionHeaders.setBearerAuth(authentToken); - } - HttpEntity initTransactionRequest = new HttpEntity<>(initTransactionHeaders); - ResponseEntity initTransactionResponse = restTemplate.exchange( - initTransactionUrl, - HttpMethod.POST, - initTransactionRequest, - String.class - ); + public void sendUpdateColectica(String ddiUpdatingInJson, TransactionType transactionType) throws RmesException { - if (initTransactionResponse.getStatusCode() == HttpStatus.OK) { - String responseBody = initTransactionResponse.getBody(); - int transactionId = extractTransactionId(responseBody); - // Étape 2: Peupler la transaction - String populateTransactionUrl = serviceUrl + "/api/v1/transaction/_addItemsToTransaction"; - HttpHeaders populateTransactionHeaders = new HttpHeaders(); - populateTransactionHeaders.setContentType(MediaType.APPLICATION_JSON); - if (authentToken != null && !authentToken.isEmpty()) { - populateTransactionHeaders.setBearerAuth(authentToken); - } - // Créez un objet de demande pour peupler la transaction ici avec transactionId et DdiUpdatingInJson - String updatedJson = ddiUpdatingInJson.replaceFirst("\\{", TRANSACTIONID + transactionId + ","); - HttpEntity populateTransactionRequest = new HttpEntity<>(updatedJson, populateTransactionHeaders); - ResponseEntity populateTransactionResponse = restTemplate.exchange( - populateTransactionUrl, - HttpMethod.POST, - populateTransactionRequest, - String.class - ); - - if (populateTransactionResponse.getStatusCode() == HttpStatus.OK) { - // Étape 3: Commit la transaction - String commitTransactionUrl = serviceUrl + "/api/v1/transaction/_commitTransaction"; - HttpHeaders commitTransactionHeaders = new HttpHeaders(); - commitTransactionHeaders.setContentType(MediaType.APPLICATION_JSON); - if (authentToken != null && !authentToken.isEmpty()) { - commitTransactionHeaders.setBearerAuth(authentToken); - } - // Créez un objet de demande pour commettre la transaction ici avec transactionId et transactionType - String commitTransactionRequestBody = TRANSACTIONID + transactionId + ",\"TransactionType\":\"" + transactionType + "\"}"; - HttpEntity commitTransactionRequest = new HttpEntity<>(commitTransactionRequestBody, commitTransactionHeaders); - ResponseEntity commitTransactionResponse = restTemplate.exchange( - commitTransactionUrl, - HttpMethod.POST, - commitTransactionRequest, - String.class - ); - - if (commitTransactionResponse.getStatusCode() == HttpStatus.OK) { - return ResponseEntity.ok("Transaction réussie."); - } else { - // Échec de la transaction, annuler la transaction - cancelTransaction(transactionId,authentToken); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Échec de la transaction."); - } - } else { - // Échec du peuplement de la transaction, annuler la transaction - cancelTransaction(transactionId,authentToken); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Échec du peuplement de la transaction."); - } - } else { - // Échec de l'initialisation de la transaction - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Échec de l'initialisation de la transaction."); + int transactionId = extractTransactionId(postWithRestClient(URI.create("transaction"), Optional.empty(), MediaType.APPLICATION_JSON)); + if (populateTransaction(ddiUpdatingInJson, transactionId)) { + if (!commitTransaction(transactionType, transactionId)) { + cancelTransaction(transactionId); + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Transaction failed", "Transaction id : " + transactionId); } - } catch (Exception e) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Une erreur s'est produite."); - } - } - - private int extractTransactionId(String responseBody) { - try { - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode jsonNode = objectMapper.readTree(responseBody); - return jsonNode.get("TransactionId").asInt(); - } catch (Exception e) { - return -1; - } - } - - private String cancelTransaction(int transactionId, String authentToken) { - // Étape 4: Annuler la transaction en cas d'échec - String cancelTransactionUrl = serviceUrl + "/api/v1/transaction/_cancelTransaction"; - HttpHeaders cancelTransactionHeaders = new HttpHeaders(); - cancelTransactionHeaders.setContentType(MediaType.APPLICATION_JSON); - cancelTransactionHeaders.setBearerAuth(authentToken); - // Créez un objet de demande pour annuler la transaction ici avec transactionId - String cancelTransactionRequestBody = TRANSACTIONID + transactionId + "}"; - HttpEntity cancelTransactionRequest = new HttpEntity<>(cancelTransactionRequestBody, cancelTransactionHeaders); - ResponseEntity cancelTransactionResponse = restTemplate.exchange( - cancelTransactionUrl, - HttpMethod.POST, - cancelTransactionRequest, - String.class - ); - - if (cancelTransactionResponse.getStatusCode() == HttpStatus.OK) { - return "Transaction annulée."; } else { - return "Échec de l'annulation de la transaction."; + cancelTransaction(transactionId); + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Fail to populate transaction", "Transaction id : " + transactionId); } } - @Override - public ResponseEntity transformFile( - MultipartFile file, - String idValue, - String nomenclatureName, - String suggesterDescription, - String version, - String idepUtilisateur, - String timbre - ) { + private boolean commitTransaction(TransactionType transactionType, int transactionId) { + String commitTransactionRequestBody = TRANSACTIONID + transactionId + ",\"TransactionType\":\"" + transactionType + "\"}"; try { - if (file.isEmpty()) { - return ResponseEntity.badRequest().build(); - } - - String resultFileName2 = UUID.randomUUID().toString() + ".json"; - - MultipartFile outputFile = processFile(file); - log.log( Level.ERROR,"Le fichier a été modifié avec succès (ajout balise data) !"); - - InputStream xsltStream1 = getClass().getResourceAsStream("/jsontoDDIXML.xsl"); - String xmlContent = transformToXml(outputFile, xsltStream1, idValue, nomenclatureName, suggesterDescription, timbre,version); - - InputStream xsltStream2 = getClass().getResourceAsStream("/DDIxmltojson.xsl"); - String jsonContent = transformToJson(new ByteArrayResource(xmlContent.getBytes(StandardCharsets.UTF_8)), xsltStream2, idepUtilisateur); - - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_OCTET_STREAM); - headers.setContentDispositionFormData("attachment", UriUtils.encode(resultFileName2, StandardCharsets.UTF_8)); - - return new ResponseEntity<>(jsonContent, headers, HttpStatus.OK); + postWithRestClient(URI.create("transaction/_commitTransaction"), Optional.of(commitTransactionRequestBody), MediaType.APPLICATION_JSON); + return true; } catch (Exception e) { - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Erreur lors de la transformation du fichier.", e); + log.error(e.getMessage(), e); + return false; } } - @Override - public ResponseEntity transformFileForComplexList( - MultipartFile file, - String idValue, - String nomenclatureName, - String suggesterDescription, - String version, - String idepUtilisateur, - String timbre, - String principale, - List secondaire, - List labelSecondaire - ) { - try { - if (file.isEmpty()) { - return ResponseEntity.badRequest().build(); - } - - String resultFileName2 = UUID.randomUUID() + ".json"; - MultipartFile outputFile = processFile(file); - log.log(Level.getLevel(ERROR),"Le fichier a été modifié avec succès (ajout balise data) !"); - InputStream xsltStream1 = getClass().getResourceAsStream("/jsonToDdiXmlForComplexList.xsl"); - String xmlContent = transformToXmlForComplexList(outputFile, xsltStream1, idValue, nomenclatureName, suggesterDescription, timbre,version,principale,secondaire,labelSecondaire); - - InputStream xsltStream2 = getClass().getResourceAsStream("/DDIComplexxmltojson.xsl"); - String jsonContent = transformToJson(new ByteArrayResource(xmlContent.getBytes(StandardCharsets.UTF_8)), xsltStream2, idepUtilisateur); - - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_OCTET_STREAM); - headers.setContentDispositionFormData("attachment", UriUtils.encode(resultFileName2, StandardCharsets.UTF_8)); - - return new ResponseEntity<>(jsonContent, headers, HttpStatus.OK); + private boolean populateTransaction(String ddiUpdatingInJson, int transactionId) { + String updatedJson = ddiUpdatingInJson.replaceFirst("\\{", TRANSACTIONID + transactionId + ","); + try { + postWithRestClient(URI.create("transaction/_addItemsToTransaction"), Optional.of(updatedJson), MediaType.APPLICATION_JSON); + return true; } catch (Exception e) { - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Erreur lors de la transformation du fichier.", e); + log.error(e.getMessage(), e); + return false; } } - public static MultipartFile processFile(MultipartFile inputFile) throws IOException { - byte[] modifiedContent; - - // Modifie le contenu en ajoutant des balises - try (InputStream inputStream = inputFile.getInputStream(); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { - - // Écriture de la balise d'ouverture "" - outputStream.write("".getBytes()); - - // Copie du contenu du fichier d'entrée dans le flux de sortie - byte[] buffer = new byte[1024]; - int bytesRead; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - } - - // Écriture de la balise de fermeture "" - outputStream.write("".getBytes()); - - // Récupération du contenu modifié sous forme de tableau de bytes - modifiedContent = outputStream.toByteArray(); - } - - // Création d'un fichier temporaire pour stocker le contenu modifié - File tempFile = File.createTempFile("upload_", ".tmp"); - try (FileOutputStream fos = new FileOutputStream(tempFile)) { - fos.write(modifiedContent); - } - - // Créer un MultipartFile personnalisé à partir du fichier temporaire - return new CustomMultipartFile(tempFile, inputFile.getOriginalFilename(), inputFile.getContentType()); - } - - private String transformToXml(MultipartFile file, InputStream xsltFile, String idValue, String nomenclatureName, - String suggesterDescription, String timbre, String version) throws IOException, TransformerException { - - TransformerFactory factory = new net.sf.saxon.TransformerFactoryImpl(); - - // Saxon ne supporte pas certaines fonctionnalités comme disallow-doctype-decl, il n'est pas nécessaire de définir cette propriété - - TransformerFactoryImpl tFactoryImpl = (TransformerFactoryImpl) factory; - net.sf.saxon.Configuration saxonConfig = tFactoryImpl.getConfiguration(); - Processor processor = (Processor) saxonConfig.getProcessor(); - - ExtensionFunction randomUUID = new randomUUID(); - processor.registerExtensionFunction(randomUUID); - - Source xslt = new StreamSource(xsltFile); - Transformer transformer = factory.newTransformer(xslt); - transformer.setParameter("idValue", idValue); - transformer.setParameter("suggesterName", nomenclatureName); - transformer.setParameter("suggesterDescription", suggesterDescription); - transformer.setParameter("timbre", timbre); - transformer.setParameter(VERSION, version); - - StreamSource text = new StreamSource(file.getInputStream()); - StringWriter xmlWriter = new StringWriter(); - StreamResult xmlResult = new StreamResult(xmlWriter); - - transformer.transform(text, xmlResult); - - return xmlWriter.toString(); - } - - private String transformToXmlForComplexList(MultipartFile file, InputStream xsltFile, String idValue, String nomenclatureName, - String suggesterDescription, String timbre, String version, String principale, - List secondaire, List labelSecondaire) - throws IOException, TransformerException { - - // Créer un transformateur XSLT sécurisé - TransformerFactory factory = TransformerFactory.newInstance(); - - // Désactiver l'accès aux entités externes pour des raisons de sécurité (prévention des attaques XXE) - factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - factory.setFeature(DISALLOW_DOCTYPE_DECL, true); - - // Utiliser TransformerFactoryImpl pour permettre des personnalisations supplémentaires via Saxon - TransformerFactoryImpl tFactoryImpl = (TransformerFactoryImpl) factory; - - // Obtenir la configuration Saxon - net.sf.saxon.Configuration saxonConfig = tFactoryImpl.getConfiguration(); - - // Obtenir le processeur Saxon - Processor processor = (Processor) saxonConfig.getProcessor(); - - // Enregistrer une fonction d'extension personnalisée - ExtensionFunction randomUUID = new randomUUID(); - processor.registerExtensionFunction(randomUUID); - - // Préparer la transformation XSLT - Source xslt = new StreamSource(xsltFile); - Transformer transformer = factory.newTransformer(xslt); - - // Passer les paramètres à la transformation - transformer.setParameter("idValue", idValue); - transformer.setParameter("suggesterName", nomenclatureName); - transformer.setParameter("suggesterDescription", suggesterDescription); - transformer.setParameter("timbre", timbre); - transformer.setParameter(VERSION, version); - transformer.setParameter("principale", principale); - transformer.setParameter("secondaire", secondaire); - transformer.setParameter("labelSecondaire", labelSecondaire); - - // Exécuter la transformation - StreamSource text = new StreamSource(file.getInputStream()); - StringWriter xmlWriter = new StringWriter(); - StreamResult xmlResult = new StreamResult(xmlWriter); - - transformer.transform(text, xmlResult); - - return xmlWriter.toString(); - } - - public String transformToJson(Resource resultResource, InputStream xsltFileJson, String idepUtilisateur) throws IOException, TransformerException { - TransformerFactory factory = new net.sf.saxon.TransformerFactoryImpl(); - Source xslt = new StreamSource(xsltFileJson); - - Transformer transformer = factory.newTransformer(xslt); - transformer.setParameter("idepUtilisateur", idepUtilisateur); - - StreamSource text = new StreamSource(resultResource.getInputStream()); - StringWriter xmlWriter = new StringWriter(); - StreamResult xmlResult = new StreamResult(xmlWriter); - - transformer.transform(text, xmlResult); - - return xmlWriter.toString(); - } - - @Override - public ResponseEntity uploadItem(MultipartFile file) { + private int extractTransactionId(String json) throws RmesException { try { - String authentToken; - if (serviceUrl.contains("kube")) { - String token2 = getAuthToken(); - authentToken = extractAccessToken(token2); - } else { - authentToken = getFreshToken(); - } - String fileContent = new String(file.getBytes()); - String itemApiUrl = serviceUrl + "/api/v1/item"; - boolean success = sendFileToApi(fileContent, authentToken, itemApiUrl); - - if (success) { - return ResponseEntity.ok("Le fichier a été envoyé avec succès à l'API."); - } else { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body("Une erreur s'est produite lors de l'envoi du fichier à l'API."); - } + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode = objectMapper.readTree(json); + return jsonNode.get("TransactionId").asInt(); } catch (Exception e) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body("Une erreur s'est produite."); - } - } - - private boolean sendFileToApi(String fileContent, String token, String url) { - - - SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); - factory.setConnectTimeout(100000); // Temps de connexion en millisecondes - factory.setReadTimeout(100000); // Temps de lecture en millisecondes - - restTemplate.setRequestFactory(factory); - - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - headers.setBearerAuth(token); - - HttpEntity request = new HttpEntity<>(fileContent, headers); - - ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST,request, String.class); - - return response.getStatusCode() == HttpStatus.OK; - } - - private String getAuthToken() throws JsonProcessingException, RmesExceptionIO { - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - - // Créer un objet représentant les informations d'identification - AuthRequest authRequest = new AuthRequest(username, password); - - // Convertir l'objet en JSON - ObjectMapper mapper = new ObjectMapper(); - String requestBody = mapper.writeValueAsString(authRequest); - - HttpEntity request = new HttpEntity<>(requestBody, headers); - - ResponseEntity response = restTemplate.postForEntity(authApiUrl, request, String.class); - - if (response.getStatusCode() == HttpStatus.OK) { - return response.getBody(); - } else { - throw new RmesExceptionIO(401, "Impossible d'obtenir le token d'authentification.", "toto"); + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Fail transacation initialisation", e.getMessage()); } } - private static String extractAccessToken(String token) throws ParseException { + private void cancelTransaction(int transactionId) throws RmesException { + String cancelTransactionRequestBody = TRANSACTIONID + transactionId + "}"; try { - JSONParser parser = new JSONParser(); - JSONObject json = (JSONObject) parser.parse(token); - return (String) json.get("access_token"); - } catch (ParseException e) { - // Log the exception - log.error("Error parsing JSON token: {}", e.getMessage()); - // Rethrow the exception - throw e; - } catch (ClassCastException e) { - // Log the exception - log.error("Error casting JSON token: {}", e.getMessage()); - // Convert the exception to a ParseException - throw new ParseException(ParseException.ERROR_UNEXPECTED_TOKEN, e); + postWithRestClient(URI.create("transaction/_cancelTransaction"), Optional.of(cancelTransactionRequestBody), MediaType.APPLICATION_JSON); + }catch (Exception e) { + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "GRAVE : fail to rollback transaction : database may be inconsistent", "transaction id "+transactionId); } } @Override - public String sendDeleteColectica(String uuid, TransactionType transactionType) throws JsonProcessingException, ExceptionColecticaUnreachable, RmesExceptionIO, ParseException { + public String sendDeleteColectica(String uuid, TransactionType transactionType) { RestTemplate restTemplate = new RestTemplate(); String initTransactionUrl = serviceUrl + "/api/v1/transaction"; - String authentToken = resolveBearerToken(); + String authentToken = kc.getFreshToken(); HttpHeaders initTransactionHeaders = new HttpHeaders(); initTransactionHeaders.setContentType(MediaType.APPLICATION_JSON); @@ -1294,6 +428,7 @@ public String sendDeleteColectica(String uuid, TransactionType transactionType) } return ("error no TransactionId"); } + private JSONArray extractIdentifiersFromDiiInstance(String fragmentXml) { Set uniqueData = new HashSet<>(); @@ -1349,28 +484,12 @@ private JSONArray extractIdentifiersFromDiiInstance(String fragmentXml) { } - @Override - public ResponseEntity getCodeBookExport(String ddiFile, File dicoVar, String accept) throws RmesException { + public Resource exportCodeBookAsOdt(String ddiFile, File dicoVar) throws RmesException { //Prepare file byte[] odt = xdr.exportVariableBookInOdt(ddiFile, dicoVar); - ByteArrayResource resource = new ByteArrayResource(odt); - - //Prepare response headers - String fileName = "Codebook"+ FilesUtils.getExtension(accept); - ContentDisposition content = ContentDisposition.builder(ATTACHMENT).filename(fileName).build(); - HttpHeaders responseHeaders = new HttpHeaders(); - responseHeaders.setAccept(List.of(MediaType.ALL)); - responseHeaders.setContentDisposition(content); - responseHeaders.setContentType(new MediaType("application","vnd.oasis.opendocument.text")); - - return ResponseEntity.ok() - .headers(responseHeaders) - .contentLength(resource.contentLength()) - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .body(resource); - + return new ByteArrayResource(odt); } @Override @@ -1428,17 +547,18 @@ public ResponseEntity getCodeBookExportV2(String ddi, String xslPatter @Override public ResponseEntity getCodeBookCheck(MultipartFile isCodeBook) throws Exception { - if (isCodeBook == null) throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Can't generate codebook","Codebook is null"); - InputStream codeBook= new BufferedInputStream(isCodeBook.getInputStream()); + if (isCodeBook == null) + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Can't generate codebook", "Codebook is null"); + InputStream codeBook = new BufferedInputStream(isCodeBook.getInputStream()); InputStream xslRemoveNameSpaces = getClass().getResourceAsStream("/xslTransformerFiles/remove-namespaces.xsl"); File ddiRemoveNameSpaces = File.createTempFile("ddiRemoveNameSpaces", ".xml"); ddiRemoveNameSpaces.deleteOnExit(); - XsltUtils.transformerInputStreamWithXsl(codeBook,xslRemoveNameSpaces,ddiRemoveNameSpaces); + XsltUtils.transformerInputStreamWithXsl(codeBook, xslRemoveNameSpaces, ddiRemoveNameSpaces); InputStream xslCodeBookCheck = getClass().getResourceAsStream("/xslTransformerFiles/dico-codes-test-ddi-content.xsl"); File codeBookCheck = File.createTempFile("codeBookCheck", ".xml"); codeBookCheck.deleteOnExit(); - XsltUtils.transformerFileWithXsl(ddiRemoveNameSpaces,xslCodeBookCheck,codeBookCheck); + XsltUtils.transformerFileWithXsl(ddiRemoveNameSpaces, xslCodeBookCheck, codeBookCheck); HttpHeaders headers = new HttpHeaders(); headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + codeBookCheck.getName()); @@ -1448,7 +568,7 @@ public ResponseEntity getCodeBookCheck(MultipartFile isCodeBook) throws Excep ByteArrayResource resourceByte = new ByteArrayResource(Files.readAllBytes(codeBookCheck.toPath())); - if (resourceByte.contentLength()==0) { + if (resourceByte.contentLength() == 0) { return ResponseEntity.ok("Nothing is missing in the DDI file " + isCodeBook.getOriginalFilename()); } return ResponseEntity.ok() diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ElasticService.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ElasticService.java new file mode 100644 index 00000000..f0bbe892 --- /dev/null +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ElasticService.java @@ -0,0 +1,84 @@ +package fr.insee.rmes.tocolecticaapi.service; + +import fr.insee.rmes.exceptions.RmesExceptionIO; +import fr.insee.rmes.model.DDIItemType; +import lombok.NonNull; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.*; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestClient; + +import java.io.IOException; +import java.net.URI; + +import static fr.insee.rmes.transfoxsl.utils.RestClientUtils.readBodySafely; + +@Component +public record ElasticService(@Value("${fr.insee.rmes.elasticsearch.url}") String elasticUrl, + @Value("${fr.insee.rmes.elasticsearch.apikey}") String elasticApiKey, + RestClient elasticClient + ) { + + private static final String APIKEYHEADER = "apiKey "; + private static final String SEARCH = "/_search"; + + private RestClient initRestClient() { + return RestClient.builder() + .defaultHeader(HttpHeaders.AUTHORIZATION, APIKEYHEADER+elasticApiKey) + .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) + .baseUrl(URI.create(this.elasticUrl+"/")) + .build(); + } + + public String search(@NonNull String index, @NonNull String texte) { + return elasticClient.get() + .uri(index + "/_search?q=*" + texte + "*") + .retrieve() + .onStatus(HttpStatusCode::is4xxClientError, (request, response) -> { + throw new RmesExceptionIO(response.getStatusCode().value(), "Bad request or inexisting resource", readBodySafely(response)); + }) + .onStatus(HttpStatusCode::is5xxServerError, (request, response) -> { + throw new RmesExceptionIO(response.getStatusCode().value(), "Error while searching into Colectica", readBodySafely(response)); + }) + .body(String.class); + } + + private String postSearch(String index, String postedSearchQuery) { + return elasticClient.post() + .uri("/" + index + SEARCH) + .body(postedSearchQuery) + .retrieve() + .onStatus(HttpStatusCode::is4xxClientError, (request, response) -> { + throw new RmesExceptionIO(response.getStatusCode().value(), "Bad request or inexisting resource", readBodySafely(response)); + }) + .onStatus(HttpStatusCode::is5xxServerError, (request, response) -> { + throw new RmesExceptionIO(response.getStatusCode().value(), "Error while searching into Colectica", readBodySafely(response)); + }) + .body(String.class); + } + + public String searchType(@NonNull String index, @NonNull DDIItemType type) { + String typeUuid = String.valueOf(type.getUUID()).toLowerCase(); + return postSearch(index, "{\"query\": {\"match\": {\"itemType\":\"" + typeUuid + "\"}}, \"_source\": true, \"size\": 10000, \"from\": 0}"); + } + + public String searchTextByType(String index, String texte, DDIItemType type) { + + String typeUuid = String.valueOf(type.getUUID()).toLowerCase(); + return postSearch(index, "{\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + " \"must\": [\n" + + " { \"match\": { \"itemType\":\"" + typeUuid + "\" }},\n" + + " { \"query_string\": {\n" + + " \"query\": \"*" + texte + "*\",\n" + + " \"fields\": [\"*\"]\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + "}"); + } + +} diff --git a/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java b/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java index 5e7410c1..4cf5d836 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java @@ -2,8 +2,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.exceptions.VtlTransformationException; import fr.insee.rmes.exceptions.XsltTransformationException; +import fr.insee.rmes.model.DDIItemType; import fr.insee.rmes.transfoxsl.service.XsltTransformationService; import fr.insee.rmes.transfoxsl.service.internal.DDIDerefencer; import fr.insee.rmes.transfoxsl.service.internal.DDITransformerToVtl; @@ -19,17 +21,20 @@ import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.util.UriUtils; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.UUID; + +import static fr.insee.rmes.transfoxsl.service.internal.DDIDerefencer.DEREFERENCE_XSL; @Controller @RequestMapping("/xsl") @@ -168,5 +173,65 @@ public ResponseEntity ddi2vtlBrut(@RequestParam("file") MultipartFile fi return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(null); } } + + + @PutMapping("/replace-xml-parameters") + @PreAuthorize("hasRole('Administrateur_RMESGOPS')") + @Operation(summary = "Modify a fragment DDI", description = "Modify a fragment DDI. All field need to be filled with the same data if there are no changes, except for the version number, which takes a plus 1.") + public String replaceXmlParameters(@RequestBody String inputXml, + @RequestParam ("Type") DDIItemType type, + @RequestParam ("Label") String label, + @RequestParam ("Version") int version, + @RequestParam ("Name") String name, + @RequestParam ("VersionResponsibility") String idepUtilisateur) { + return xsltTransformationService.replaceXmlParameters(inputXml, type, label, version, name, idepUtilisateur); + } + + @GetMapping(value = "/transformJsonToJsonForAPI", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @Operation(summary = "transform an JSON Codelist (Id,Label) to an another json for Colectica API ", + description = "tranform a codeList in json to another json with DDI item inside") + @PreAuthorize("hasRole('Administrateur_RMESGOPS')") + public ResponseEntity transformFile(@RequestParam("file") MultipartFile file, + @RequestParam("nom metier") String idValue, + @RequestParam("label") String nomenclatureName, + @RequestParam("description") String suggesterDescription, + @RequestParam(value = "version", defaultValue = "1") String version, + @RequestParam("idepUtilisateur") String idepUtilisateur, + // peut-être lire le jeton pour recup le timbre directement + @RequestParam("timbre") String timbre) throws RmesException { + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_OCTET_STREAM); + headers.setContentDispositionFormData("attachment", UriUtils.encode(UUID.randomUUID() + ".json", StandardCharsets.UTF_8)); + return ResponseEntity.ok() + .headers(headers) + .body(xsltTransformationService.transformFile(file, idValue, nomenclatureName, suggesterDescription, version, idepUtilisateur, timbre)); + } + + @GetMapping(value = "/transformJsonToJsonForApiForComplexCodeList", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @Operation(summary = "transform an JSON Codelist with multiple field for one id to an another json for Colectica API ", + description = "tranform a codeList in json to another json with DDI item inside") + @PreAuthorize("hasRole('Administrateur_RMESGOPS')") + public ResponseEntity transformFileForComplexCodeList(@RequestParam("file") MultipartFile file, + @RequestParam("nom metier") String idValue, + @RequestParam("label") String nomenclatureName, + @RequestParam("description") String suggesterDescription, + @RequestParam(value = "version", defaultValue = "1") String version, + @RequestParam("idepUtilisateur") String idepUtilisateur, + // peut-être lire le jeton pour recup le timbre directement et l'idep + @RequestParam("timbre") String timbre, + @RequestParam("principale") String principale, + @RequestParam("secondaire") List secondaire, + @RequestParam("labelSecondaire") List labelSecondaire) throws RmesException { + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_OCTET_STREAM); + headers.setContentDispositionFormData("attachment", UriUtils.encode(UUID.randomUUID() + ".json", StandardCharsets.UTF_8)); + + return ResponseEntity.ok() + .headers(headers) + .body(xsltTransformationService.transformFileForComplexList(file, idValue, nomenclatureName, suggesterDescription, version, idepUtilisateur, timbre, principale, secondaire, labelSecondaire); + } + } diff --git a/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java b/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java index 235bac67..6fa4553d 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java @@ -1,29 +1,187 @@ package fr.insee.rmes.transfoxsl.service; +import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.exceptions.XsltTransformationException; +import fr.insee.rmes.model.DDIItemType; +import fr.insee.rmes.tocolecticaapi.RandomUUIDExtensionFunction; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; +import net.sf.saxon.TransformerFactoryImpl; import net.sf.saxon.s9api.*; +import org.springframework.core.io.ByteArrayResource; import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.server.ServerErrorException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.*; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; +import java.util.List; +import java.util.Map; @Service @Slf4j public class XsltTransformationService { + public static final String DISALLOW_DOCTYPE_DECL = "http://javax.xml.transform.TransformerFactory/feature/disallow-doctype-decl"; + public static final String DISALLOW_DOCTYPE_DECL1 = "http://apache.org/xml/features/disallow-doctype-decl"; + public static final String EXTERNAL_GENERAL_ENTITIES = "http://xml.org/sax/features/external-general-entities"; + public static final String EXTERNAL_PARAMETER_ENTITIES = "http://xml.org/sax/features/external-parameter-entities"; + private static final String VERSION = "version"; + static final byte[] DATA_TAG_BEGIN = "".getBytes(); + static final byte[] DATA_TAG_END = "".getBytes(); + static final TransformerFactory saxonFactoryWithRandomUUIDExtension = initSaxonFactoryWithRandomUUIDExtension(); + private final Processor processor; public XsltTransformationService() { this.processor = new Processor(false); } + + private static TransformerFactory initSaxonFactoryWithRandomUUIDExtension() { + TransformerFactoryImpl factory = new net.sf.saxon.TransformerFactoryImpl(); + ((Processor) factory.getConfiguration().getProcessor()).registerExtensionFunction(new RandomUUIDExtensionFunction()); + + return factory; + } + + + public String transformFileForComplexList( + MultipartFile file, + String idValue, + String nomenclatureName, + String suggesterDescription, + String version, + String idepUtilisateur, + String timbre, + String principale, + List secondaire, + List labelSecondaire + ) throws RmesException { + + DDIProcessor ddiProcessor = (inputData, xsltStream) -> + transformToXmlForComplexList(inputData, xsltStream, idValue, nomenclatureName, suggesterDescription, timbre, version, principale, secondaire, labelSecondaire); + return doTransformFile(file, idepUtilisateur, "/jsonToDdiXmlForComplexList.xsl", ddiProcessor, "/DDIComplexxmltojson.xsl"); + } + + public String transformFile( + @NonNull MultipartFile file, + String idValue, + String nomenclatureName, + String suggesterDescription, + String version, + String idepUtilisateur, + String timbre + ) throws RmesException { + + DDIProcessor ddiProcessor = (inputData, xsltStream) -> transformToXml(inputData, xsltStream, idValue, nomenclatureName, suggesterDescription, timbre, version); + return doTransformFile(file, idepUtilisateur, "/jsontoDDIXML.xsl", ddiProcessor, "/DDIxmltojson.xsl"); + } + + + + public static byte[] transformToJson(Resource resultResource, InputStream xsltFileJson, String idepUtilisateur) throws + IOException, TransformerException { + return transformWithSaxon(xsltFileJson, new StreamSource(resultResource.getInputStream()), + Map.of("idepUtilisateur", idepUtilisateur) + ); + } + + private byte[] transformToXml(byte[] inputData, InputStream xsltFile, String idValue, String + nomenclatureName, + String suggesterDescription, String timbre, String version) throws TransformerException { + + return transformWithSaxon(xsltFile, new StreamSource(new ByteArrayInputStream(inputData)), + Map.of( + "idValue", idValue, + "suggesterName", nomenclatureName, + "suggesterDescription", suggesterDescription, + "timbre", timbre, + VERSION, version) + ); + } + + private static byte[] transformWithSaxon(InputStream xsltFile, StreamSource input, Map parameters) throws TransformerException { + + Transformer transformer = saxonFactoryWithRandomUUIDExtension.newTransformer(new StreamSource(xsltFile)); + parameters.forEach(transformer::setParameter); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + StreamResult xmlResult = new StreamResult(outputStream); + + transformer.transform(input, xmlResult); + + return outputStream.toByteArray(); + } + + private byte[] transformToXmlForComplexList(byte[] inputData, InputStream xsltFile, String idValue, String + nomenclatureName, + String suggesterDescription, String timbre, String version, String principale, + List secondaire, List labelSecondaire) + throws TransformerException { + + return transformWithSaxon(xsltFile, new StreamSource(new ByteArrayInputStream(inputData)), + Map.of( + "idValue", idValue, + "suggesterName", nomenclatureName, + "suggesterDescription", suggesterDescription, + "timbre", timbre, + VERSION, version, + "principale", principale, + "secondaire", secondaire, + "labelSecondaire", labelSecondaire + ) + ); + } + + + private String doTransformFile(MultipartFile file, String idepUtilisateur, String jsonToDdiXslFilename, DDIProcessor ddiProcessor, String DdiToJsonXslFilename) throws RmesException { + if (file.isEmpty()) { + throw new RmesException(HttpStatus.BAD_REQUEST, "Uploaded file is empty", null); + } + try { + byte[] inputData = contentInDataTag(file); + log.debug("Le contenu du fichier a été modifié avec succès (ajout balise data) !"); + + + InputStream xsltStream1 = getClass().getResourceAsStream(jsonToDdiXslFilename); + byte[] xmlContent = ddiProcessor.process(inputData, xsltStream1); + + + InputStream xsltStream2 = getClass().getResourceAsStream(DdiToJsonXslFilename); + return new String(transformToJson(new ByteArrayResource(xmlContent), xsltStream2, idepUtilisateur)); + } catch (Exception e) { + throw new ServerErrorException("Erreur lors de la transformation du fichier.", e); + } + } + + private byte[] contentInDataTag(@NonNull MultipartFile file) throws IOException { + var fileContent = file.getBytes(); + byte[] retour = new byte[DATA_TAG_BEGIN.length + fileContent.length + DATA_TAG_END.length]; + System.arraycopy(DATA_TAG_BEGIN, 0, retour, 0, DATA_TAG_BEGIN.length); + System.arraycopy(fileContent, 0, retour, DATA_TAG_BEGIN.length, fileContent.length); + System.arraycopy(DATA_TAG_END, 0, retour, DATA_TAG_BEGIN.length + fileContent.length, DATA_TAG_END.length); + return retour; + } + + byte[] transform(InputStream inputStream, String xslFileName, SerializerConfigurer serializerConfigurer) throws XsltTransformationException, IOException { try { - log.atDebug().log(()->"Starting transformation with XSLT file: "+ xslFileName); + log.atDebug().log(() -> "Starting transformation with XSLT file: " + xslFileName); XsltCompiler compiler = processor.newXsltCompiler(); @@ -42,21 +200,37 @@ byte[] transform(InputStream inputStream, String xslFileName, SerializerConfigur transformer.setDestination(serializer); transformer.transform(); - return outputStream.toByteArray(); - -// // Si sortie texte, on peut découper par ligne et retourner une liste de String -// if (isTextOutput) { -// return Arrays.asList(result.split("\n")); -// } else { -// // Pour XML, on retourne une seule chaîne sous forme de liste (vous pouvez adapter selon vos besoins) -// return Collections.singletonList(result); -// } + return outputStream.toByteArray(); } catch (SaxonApiException e) { throw new XsltTransformationException("Error during XSLT transformation", e); } } - public String transformToXmlString(InputStream inputStream, String xslFileName)throws XsltTransformationException, IOException { + static byte[] transformWithParameters(byte[] source, InputStream xsltFileJson, Map parameters) throws javax.xml.transform.TransformerException { + // Créer un transformateur XSLT sécurisé + // Désactiver l'accès aux entités externes pour des raisons de sécurité (prévention des attaques XXE) + TransformerFactory factory = TransformerFactory.newInstance(); + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setFeature(DISALLOW_DOCTYPE_DECL, true); + factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "file,jar:file"); + + Source xslt = new StreamSource(xsltFileJson); + + Transformer transformer = factory.newTransformer(xslt); + parameters.forEach(transformer::setParameter); + + // Effectuer la transformation + StreamSource text = new StreamSource(new ByteArrayInputStream(source)); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + StreamResult xmlResult = new StreamResult(outputStream); + + transformer.transform(text, xmlResult); + + return outputStream.toByteArray(); + } + + public String transformToXmlString(InputStream inputStream, String xslFileName) throws XsltTransformationException, IOException { return toXmlString(transformToXml(inputStream, xslFileName)); } @@ -71,4 +245,93 @@ private String toXmlString(byte[] result) { public byte[] transformToRawText(InputStream stream, String ddi2VtlXsl) throws IOException { return transform(stream, ddi2VtlXsl, SerializerConfigurer.forRawText(processor)); } + + + public String replaceXmlParameters(String inputXml, DDIItemType type, String label, int version, String name, String idepUtilisateur) { + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + + // Désactivation des entités externes pour des raisons de sécurité (prévention des attaques XXE) + factory.setNamespaceAware(true); + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setFeature(DISALLOW_DOCTYPE_DECL1, true); + factory.setFeature(EXTERNAL_GENERAL_ENTITIES, false); + factory.setFeature(EXTERNAL_PARAMETER_ENTITIES, false); + + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(new InputSource(new StringReader(inputXml))); + + String typeName = type.getName(); + NodeList typeNodes = document.getElementsByTagNameNS("ddi:logicalproduct:3_3", typeName); + + if (typeNodes.getLength() == 0) { + return "Erreur : Aucun élément correspondant trouvé pour le type " + typeName; + } + + Element typeNameDocument = (Element) typeNodes.item(0); + + if (!typeName.equals(typeNameDocument.getNodeName())) { + return "Erreur : Attention ce n'est pas le bon type. L'item chargé n'est pas du type que vous avez sélectionné."; + } + + Document copy = (Document) document.cloneNode(true); + + NodeList versionNodes = copy.getElementsByTagName("r:Version"); + for (int i = 0; i < versionNodes.getLength(); i++) { + Node versionNode = versionNodes.item(i); + if (versionNode instanceof Element) { + Element versionElement = (Element) versionNode; + versionElement.setTextContent(String.valueOf(version)); + } + } + + Node nameNode = copy.getElementsByTagName("r:String").item(0); + nameNode.setTextContent(name); + + Node labelNode = copy.getElementsByTagName("r:Content").item(0); + labelNode.setTextContent(label); + + NodeList urnNodes = copy.getElementsByTagName("r:URN"); + for (int i = 0; i < urnNodes.getLength(); i++) { + Node urnNode = urnNodes.item(i); + if (urnNode instanceof Element) { + Element urnElement = (Element) urnNode; + String urnCode = urnElement.getTextContent(); + urnElement.setTextContent(urnCode.substring(0, urnCode.lastIndexOf(":")) + ":" + version); + } + } + + ByteArrayOutputStream outputStreamResult = onlyIndent(copy); + + // Ajout de la transformation XML vers JSON avec XSLT + InputStream xsltStream = getClass().getResourceAsStream("/DDIxmltojsonForOneObject.xsl"); + return transformToJson(outputStreamResult.toByteArray(), xsltStream, idepUtilisateur, version); + + } catch (Exception e) { + return "Error processing XML"; + } + } + + private static ByteArrayOutputStream onlyIndent(Document document) throws TransformerException { + // Désactivation des entités externes dans TransformerFactory pour éviter les attaques XXE + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + transformerFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + transformerFactory.setFeature(DISALLOW_DOCTYPE_DECL, true); + + Transformer transformer = transformerFactory.newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + + ByteArrayOutputStream outputStreamResult = new ByteArrayOutputStream(); + transformer.transform(new DOMSource(document), new StreamResult(outputStreamResult)); + return outputStreamResult; + } + + private static String transformToJson(byte[] source, InputStream xsltFileJson, String idepUtilisateur, int version) throws TransformerException { + return new String(transformWithParameters(source, xsltFileJson, Map.of("idepUtilisateur", idepUtilisateur, VERSION, String.valueOf(version)))); + } + + @FunctionalInterface + private interface DDIProcessor { + byte[] process(byte[] inputData, InputStream xsltStream) throws TransformerException, IOException; + } } diff --git a/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDIDerefencer.java b/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDIDerefencer.java index 345049e5..9fbed08c 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDIDerefencer.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDIDerefencer.java @@ -1,21 +1,29 @@ package fr.insee.rmes.transfoxsl.service.internal; import fr.insee.rmes.transfoxsl.service.XsltTransformationService; +import lombok.NonNull; +import org.springframework.stereotype.Service; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +@Service public record DDIDerefencer(XsltTransformationService xsltTransformationService) { public static final String DEREFERENCE_XSL = "dereference.xsl"; - public String dereferenceToString(InputStream inputStream) throws IOException { + public String dereferenceToString(@NonNull InputStream inputStream) throws IOException { // Première transformation - XML en sortie return xsltTransformationService.transformToXmlString(inputStream, DEREFERENCE_XSL); } - public byte[] intermediateDereference(InputStream inputStream) throws IOException { + public byte[] intermediateDereference(@NonNull InputStream inputStream) throws IOException { return xsltTransformationService.transformToXml(inputStream, DEREFERENCE_XSL); } + public byte[] intermediateDereference(@NonNull String xmlString) throws IOException { + return intermediateDereference(new ByteArrayInputStream(xmlString.getBytes())); + } + } diff --git a/src/main/java/fr/insee/rmes/transfoxsl/utils/RestClientUtils.java b/src/main/java/fr/insee/rmes/transfoxsl/utils/RestClientUtils.java new file mode 100644 index 00000000..b7915341 --- /dev/null +++ b/src/main/java/fr/insee/rmes/transfoxsl/utils/RestClientUtils.java @@ -0,0 +1,22 @@ +package fr.insee.rmes.transfoxsl.utils; + +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; + +public class RestClientUtils { + + private RestClientUtils() {} + + public static String readBodySafely(ClientHttpResponse response) { + if (response ==null){ + return "null response"; + }; + try { + return new String(response.getBody().readAllBytes()); + } catch (IOException e) { + return "Error "+e.getMessage()+" while reading error response body"; + } + } + +} diff --git a/src/main/java/fr/insee/rmes/utils/FileExtension.java b/src/main/java/fr/insee/rmes/utils/FileExtension.java index d88d7612..63e5285e 100644 --- a/src/main/java/fr/insee/rmes/utils/FileExtension.java +++ b/src/main/java/fr/insee/rmes/utils/FileExtension.java @@ -24,7 +24,7 @@ public MediaType getMediaTypeFromExtension() { return mediaType; } - public static FileExtension get(String acceptHeader) { + public static FileExtension forHeader(String acceptHeader) { return switch (acceptHeader) { case "application/octet-stream" -> PDF_EXTENSION; case "flatODT" -> FODT_EXTENSION; diff --git a/src/main/java/fr/insee/rmes/utils/FilesUtils.java b/src/main/java/fr/insee/rmes/utils/FilesUtils.java index 6aaaa10c..c7d71ad7 100644 --- a/src/main/java/fr/insee/rmes/utils/FilesUtils.java +++ b/src/main/java/fr/insee/rmes/utils/FilesUtils.java @@ -1,25 +1,21 @@ package fr.insee.rmes.utils; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.nio.file.FileSystem; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.text.Normalizer; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; public class FilesUtils { - private static final Logger log = LoggerFactory.getLogger(FilesUtils.class); - - public static String reduceFileNameSize(String fileName, int maxLength) { + public static String reduceFileNameSize(String fileName, int maxLength) { return fileName.substring(0, Math.min(fileName.length(), maxLength)); } @@ -33,19 +29,6 @@ public static File streamToFile(InputStream in, String fileName, String fileExte return tempFile; } - public static String cleanFileNameAndAddExtension(String fileName, String extension) { - fileName = fileName.toLowerCase().trim(); - fileName = StringUtils.normalizeSpace(fileName); - fileName = fileName.replace(" ", "-"); - fileName = Normalizer.normalize(fileName, Normalizer.Form.NFD).replace("[^\\p{ASCII}]", ""); - if (extension.startsWith(".")) { - fileName += extension; - } else { - fileName += "." + extension; - } - return fileName; - } - public static void addFileToZipFolder(Path fileToAdd, Path zipArchive) throws IOException { try(FileSystem fs = FileSystems.newFileSystem(zipArchive)){ Files.write(fs.getPath(fileToAdd.getFileName().toString()), Files.readAllBytes(fileToAdd)); @@ -56,20 +39,6 @@ private FilesUtils() { throw new IllegalStateException("Utility class"); } - public static void zipDirectory(File directoryToZip) throws IOException { - FileOutputStream fos = new FileOutputStream(directoryToZip + "/" + directoryToZip.getName() + ".zip"); - ZipOutputStream zipOut = new ZipOutputStream(fos); - - zipFile(directoryToZip, directoryToZip.getName(), zipOut); - - try { - zipOut.close(); - fos.close(); - } catch(IOException e ) { - log.warn("outputStream already closed"); - } - - } public static String removeAsciiCharacters(String fileName) { return Normalizer.normalize(fileName, Normalizer.Form.NFD) .replaceAll("œ", "oe") @@ -78,34 +47,4 @@ public static String removeAsciiCharacters(String fileName) { .replaceAll("\\p{Punct}", ""); } - private static void zipFile(File fileToZip, String fileName, ZipOutputStream zipOut) throws IOException { - if (fileToZip.isHidden() || fileName.endsWith(".zip")) { - return; - } - if (fileToZip.isDirectory()) { - if (fileName.endsWith("/")) { - zipOut.putNextEntry(new ZipEntry(fileName)); - zipOut.closeEntry(); - } else { - zipOut.putNextEntry(new ZipEntry(fileName + "/")); - zipOut.closeEntry(); - } - File[] children = fileToZip.listFiles(); - for (File childFile : children) { - zipFile(childFile, fileName + "/" + childFile.getName(), zipOut); - } - - return; - } - try (FileInputStream fis = new FileInputStream(fileToZip)) { - ZipEntry zipEntry = new ZipEntry(fileName); - zipOut.putNextEntry(zipEntry); - byte[] bytes = new byte[1024]; - int length; - while ((length = fis.read(bytes)) >= 0) { - zipOut.write(bytes, 0, length); - } - } - } - } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index c23e8e79..4861ed60 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -46,6 +46,7 @@ fr.insee.rmes.elasticsearch.apikey= cle # Metadata service fr.insee.rmes.api.remote.metadata.MetadataService.impl = MetadataServiceImpl +## Colectica Repository API fr.insee.rmes.api.remote.metadata.url = http://ddiApi fr.insee.rmes.api.remote.metadata.agency = xxxx fr.insee.rmes.api.remote.metadata.key = xxxx diff --git a/src/test/java/service/ColecticaServiceImplTest.java b/src/test/java/service/ColecticaServiceImplTest.java index 9d845c18..b3e1eddb 100644 --- a/src/test/java/service/ColecticaServiceImplTest.java +++ b/src/test/java/service/ColecticaServiceImplTest.java @@ -4,6 +4,7 @@ import fr.insee.rmes.config.keycloak.KeycloakServices; import fr.insee.rmes.model.DDIItemType; import fr.insee.rmes.tocolecticaapi.service.ColecticaServiceImpl; +import fr.insee.rmes.transfoxsl.service.XsltTransformationService; import fr.insee.rmes.utils.DocumentBuilderUtils; import org.apache.http.ProtocolVersion; import org.apache.http.client.methods.CloseableHttpResponse; @@ -150,7 +151,7 @@ void testTransformToJson() throws Exception { InputStream xsltStream = new ByteArrayInputStream(xsltContent.getBytes()); // Call the method to be tested - String result = colecticaService.transformToJson(mockResource, xsltStream, "user"); + String result = XsltTransformationService.transformToJson(mockResource, xsltStream, "user"); // Assertions assertNotNull(result); @@ -159,33 +160,6 @@ void testTransformToJson() throws Exception { assertTrue(result.contains("Test")); } - @Test - void testUploadItem() { - ColecticaServiceImpl colecticaService = mock(ColecticaServiceImpl.class); - String jsonContent = "{\n" + - " \"Items\":\n" + - "[\n" + - "{\n" + - "\"ItemType\": \"8b108ef8-b642-4484-9c49-f88e4bf7cf1d\",\n" + - "\"AgencyId\": \"fr.insee\",\n" + - "\"Version\": 1,\n" + - "\"Identifier\": \"4aea85af-27c9-4b87-a057-ccbd78ec95bf\",\n" + - "\"Item\": \" \",\n" + - "\"VersionDate\": \"2024-02-21T09:31:22.0300000Z\",\n" + - "\"VersionResponsibility\": \"QZ6ICW\",\n" + - "\"IsPublished\": false,\n" + - "\"IsDeprecated\": false,\n" + - "\"IsProvisional\": false,\n" + - "\"ItemFormat\": \"DC337820-AF3A-4C0B-82F9-CF02535CDE83\"\n" + - "}]"; - MockMultipartFile file = new MockMultipartFile("file", "test.json", "application/json", jsonContent.getBytes()); - ResponseEntity mockedResponse = ResponseEntity.ok("Le fichier a été envoyé avec succès à l'API."); - when(colecticaService.uploadItem(file)).thenReturn(mockedResponse); - ResponseEntity result = colecticaService.uploadItem(file); - assertNotNull(result); - assertEquals(HttpStatus.OK, result.getStatusCode()); - assertEquals("Le fichier a été envoyé avec succès à l'API.", result.getBody()); - } @Test void testGetDocument() throws Exception { From 27ce69cb1aaa0e07e43bd7480ce69b20608783de Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Fri, 20 Dec 2024 17:01:00 +0100 Subject: [PATCH 07/25] refactor (mutulualize, separate concerns, remove useless, remove redondant, remove elastic search) WIP - fr.insee.rmes.utils.export.XDocReport.getCodeBookExportV2 in progress --- .../controller/ControllerUtils.java | 4 +- .../tocolecticaapi/controller/PostItem.java | 124 +-------- .../models/CustomMultipartFile.java | 65 ----- .../service/ColecticaService.java | 16 +- .../service/ColecticaServiceImpl.java | 256 +++--------------- .../controller/TransformationController.java | 106 +++++++- .../service/XsltTransformationService.java | 104 +++++-- .../java/fr/insee/rmes/utils/XMLUtils.java | 129 ++------- .../java/fr/insee/rmes/utils/XsltUtils.java | 62 +---- .../insee/rmes/utils/export/XDocReport.java | 82 +++++- 10 files changed, 341 insertions(+), 607 deletions(-) delete mode 100644 src/main/java/fr/insee/rmes/tocolecticaapi/models/CustomMultipartFile.java diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/ControllerUtils.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/ControllerUtils.java index 3c11fe83..d9288bbe 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/ControllerUtils.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/ControllerUtils.java @@ -11,7 +11,7 @@ public class ControllerUtils { private ControllerUtils() {} - static ResponseEntity xmltoResponseEntity(Resource response, String fileNameWithExtension, MediaType contentType) throws IOException { + public static ResponseEntity xmltoResponseEntity(Resource response, String fileNameWithExtension, MediaType contentType) throws IOException { return ResponseEntity.ok() .header(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.attachment().filename(fileNameWithExtension).build().toString()) .contentLength(response.contentLength()) @@ -19,7 +19,7 @@ static ResponseEntity xmltoResponseEntity(Resource response, String fi .body(response); } - static ResponseEntity xmltoResponseEntity(String xmlContent) { + public static ResponseEntity xmltoResponseEntity(String xmlContent) { return ResponseEntity.ok(xmlContent.replace("\\\"", "\"").replace("", "")); } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java index 9c91a2ae..bbc15b12 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java @@ -1,13 +1,11 @@ package fr.insee.rmes.tocolecticaapi.controller; -import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.exceptions.ExceptionColecticaUnreachable; +import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.model.DDIItemType; import fr.insee.rmes.tocolecticaapi.models.TransactionType; import fr.insee.rmes.tocolecticaapi.service.ColecticaService; -import fr.insee.rmes.utils.FileExtension; -import fr.insee.rmes.utils.FilesUtils; import io.swagger.v3.oas.annotations.Hidden; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -20,23 +18,20 @@ import org.apache.logging.log4j.Logger; import org.json.simple.parser.ParseException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.Resource; -import org.springframework.http.*; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.security.oauth2.core.oidc.user.OidcUser; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.util.UriUtils; import java.io.BufferedInputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.UUID; import static fr.insee.rmes.tocolecticaapi.controller.ControllerUtils.xmltoResponseEntity; @@ -57,7 +52,6 @@ public class PostItem { static final Logger logger = LogManager.getLogger(PostItem.class); - public static final MediaType APPLICATION_ODT = MediaType.valueOf("application/vnd.oasis.opendocument.text"); private final ColecticaService colecticaService; @Autowired @@ -95,117 +89,9 @@ public ResponseEntity uploadItem(@RequestParam("file") MultipartFile fil @Operation(summary = "Get JSON for a type of DDI item", description = "Get a JSON list of item for a type of DDI items .") @PreAuthorize("hasRole('Administrateur_RMESGOPS')") public ResponseEntity byType( - @PathVariable("type") DDIItemType type, @AuthenticationPrincipal - OidcUser principal) + @PathVariable("type") DDIItemType type) throws IOException, ExceptionColecticaUnreachable, ParseException { return colecticaService.getByType(type); } - - - @PostMapping(value = "/operation/codebook", - consumes = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"}, - produces = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"} - ) - @PreAuthorize("hasRole('Administrateur_RMESGOPS')") - @Operation(operationId = "getCodeBook", summary = "Produce a codebook from a DDI") - - public ResponseEntity getCodeBook( - - @Parameter(schema = @Schema(type = "string", format = "String", description = "Accept")) - @RequestHeader(required = false) String accept, - - @Parameter(schema = @Schema(type = "string", format = "binary", description = "file in DDI")) - @RequestParam(value = "file") MultipartFile isDDI, // InputStream isDDI, - - @Parameter(schema = @Schema(type = "string", format = "binary", description = "file for structure")) - @RequestParam(value = "dicoVar") MultipartFile isCodeBook //InputStream isCodeBook - - ) - throws RmesException { - logger.info("Generate CodeBook from DDI {}, {}", isDDI.getOriginalFilename(), isCodeBook.getOriginalFilename()); - String ddi; - File codeBookFile; - try { - ddi = new String(isDDI.getBytes(), StandardCharsets.UTF_8); - codeBookFile = FilesUtils.streamToFile(isCodeBook.getInputStream(), "dicoVar", ".odt"); - Resource response = colecticaService.exportCodeBookAsOdt(ddi, codeBookFile); - logger.debug("Codebook is generated"); - return xmltoResponseEntity(response, "Codebook" + FileExtension.forHeader(accept).extension(), APPLICATION_ODT); - } catch (IOException e) { - throw new RmesException(HttpStatus.BAD_REQUEST, e.getMessage(), "Files can't be read"); - } - } - - @PostMapping(value = "/operation/codebook/V2", - consumes = {MediaType.MULTIPART_FORM_DATA_VALUE}, - produces = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"} - ) - @PreAuthorize("hasRole('Administrateur_RMESGOPS')") - @Operation(operationId = "getCodeBookV2", summary = "Produce a codebook from a DDI") - public ResponseEntity getCodeBookV2( - - @Parameter(schema = @Schema(type = "string", format = "String", description = "Accept")) - @RequestHeader(required = false) String accept, - - @Parameter(schema = @Schema(type = "string", allowableValues = {"concis", "concis avec expression", "scindable", "non scindable"})) - @RequestParam(value = "dicoVar") String isCodeBook, //InputStream isCodeBook, - - @RequestParam(value = "file") MultipartFile isDDI // InputStream isDDI, - - ) - throws Exception { - if (isDDI == null) - throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Can't generate codebook", "Stream is null"); - InputStream ddiInputStream = new BufferedInputStream(isDDI.getInputStream()); - String ddi = new String(ddiInputStream.readAllBytes(), StandardCharsets.UTF_8); - String xslPatternFile = null; - switch (isCodeBook) { - case "concis": - String xmlFileConcis = "/xslTransformerFiles/dicoCodes/dicoConcisPatternContent.xml"; - xslPatternFile = xmlFileConcis; - break; - case "concis avec expression": - String xmlFileConcisAvecExpression = "/xslTransformerFiles/dicoCodes/dicoConcisDescrPatternContent.xml"; - xslPatternFile = xmlFileConcisAvecExpression; - break; - case "scindable": - String xmlFileScindable = "/xslTransformerFiles/dicoCodes/dicoScindablePatternContent.xml"; - xslPatternFile = xmlFileScindable; - break; - case "non scindable": - String xmlFileNonScindable = "/xslTransformerFiles/dicoCodes/dicoNonScindablePatternContent.xml"; - xslPatternFile = xmlFileNonScindable; - break; - default: - logger.error("Choix incorrect"); - break; - } - - return colecticaService.getCodeBookExportV2(ddi, xslPatternFile); - } - - @PostMapping(value = "/operation/codebook/checkCodeBookContent", - consumes = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"}, - produces = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"} - ) - @PreAuthorize("hasRole('Administrateur_RMESGOPS')") - @Operation(operationId = "getCodeBookCheck", summary = "Check the DDI before made the codebook export") - - public ResponseEntity getCodeBookCheck( - - @RequestParam(value = "file") MultipartFile isCodeBook // InputStream isDDI, - - ) - throws Exception { - - return colecticaService.getCodeBookCheck(isCodeBook); - } - - protected ResponseEntity returnRmesException(RmesException e) { - logger.error(e.getMessageAndDetails(), e); - return ResponseEntity.status(e.getStatus()).contentType(MediaType.TEXT_PLAIN).body(e.getMessage()); - } - - } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/models/CustomMultipartFile.java b/src/main/java/fr/insee/rmes/tocolecticaapi/models/CustomMultipartFile.java deleted file mode 100644 index e6271b55..00000000 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/models/CustomMultipartFile.java +++ /dev/null @@ -1,65 +0,0 @@ -package fr.insee.rmes.tocolecticaapi.models; - -import org.springframework.web.multipart.MultipartFile; - -import java.io.*; - -public class CustomMultipartFile implements MultipartFile { - - private final File file; - private final String originalFilename; - private final String contentType; - - public CustomMultipartFile(File file, String originalFilename, String contentType) { - this.file = file; - this.originalFilename = originalFilename; - this.contentType = contentType; - } - - @Override - public String getName() { - return originalFilename; - } - - @Override - public String getOriginalFilename() { - return originalFilename; - } - - @Override - public String getContentType() { - return contentType; - } - - @Override - public boolean isEmpty() { - return file.length() == 0; - } - - @Override - public long getSize() { - return file.length(); - } - - @Override - public byte[] getBytes() throws IOException { - return java.nio.file.Files.readAllBytes(file.toPath()); - } - - @Override - public InputStream getInputStream() throws IOException { - return new FileInputStream(file); - } - - @Override - public void transferTo(File dest) throws IOException { - try (InputStream in = new FileInputStream(file); - OutputStream out = new FileOutputStream(dest)) { - byte[] buffer = new byte[1024]; - int length; - while ((length = in.read(buffer)) > 0) { - out.write(buffer, 0, length); - } - } - } -} diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java index 668eb9a2..b63873aa 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java @@ -1,18 +1,14 @@ package fr.insee.rmes.tocolecticaapi.service; import com.fasterxml.jackson.core.JsonProcessingException; -import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.exceptions.ExceptionColecticaUnreachable; +import fr.insee.rmes.exceptions.RmesException; +import fr.insee.rmes.exceptions.RmesExceptionIO; import fr.insee.rmes.model.DDIItemType; import fr.insee.rmes.tocolecticaapi.models.TransactionType; -import fr.insee.rmes.exceptions.RmesExceptionIO; import org.json.JSONArray; import org.json.simple.parser.ParseException; -import org.springframework.core.io.Resource; -import org.springframework.http.ResponseEntity; -import org.springframework.web.multipart.MultipartFile; -import java.io.File; import java.io.IOException; import java.util.List; import java.util.Map; @@ -20,7 +16,7 @@ public interface ColecticaService { String findFragmentByUuid(String uuid); - String sendDeleteColectica(String uuid, TransactionType transactionType) throws JsonProcessingException, ExceptionColecticaUnreachable, RmesExceptionIO, ParseException; + String sendDeleteColectica(String uuid, TransactionType transactionType) throws JsonProcessingException, ExceptionColecticaUnreachable, RmesExceptionIO, ParseException, RmesException; String searchColecticaInstanceByUuid(String uuid) throws RmesException; JSONArray findFragmentByUuidWithChildren(String uuid) throws Exception; String filteredSearchText(String index, String texte) throws RmesException; @@ -36,10 +32,4 @@ public interface ColecticaService { void sendUpdateColectica(String ddiUpdatingInJson, TransactionType transactionType) throws RmesException; - Resource exportCodeBookAsOdt(String ddiFile, File dicoVar) throws RmesException; - - ResponseEntity getCodeBookExportV2(String ddiFile, String xslPatternFile) throws RmesException, Exception; - - ResponseEntity getCodeBookCheck(MultipartFile isCodeBook) throws RmesException, Exception; - } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java index 1edc73ee..c2fccfac 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java @@ -4,37 +4,31 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import fr.insee.rmes.config.keycloak.KeycloakServices; import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.exceptions.RmesExceptionIO; import fr.insee.rmes.model.DDIItemType; -import fr.insee.rmes.tocolecticaapi.models.FragmentData; import fr.insee.rmes.tocolecticaapi.models.NamespaceContextMap; import fr.insee.rmes.tocolecticaapi.models.RessourcePackage; import fr.insee.rmes.tocolecticaapi.models.TransactionType; import fr.insee.rmes.transfoxsl.service.internal.DDIDerefencer; import fr.insee.rmes.utils.ExportUtils; -import fr.insee.rmes.utils.XsltUtils; -import fr.insee.rmes.utils.export.XDocReport; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.json.JSONArray; import org.json.JSONObject; import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.ByteArrayResource; -import org.springframework.core.io.Resource; -import org.springframework.http.*; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.HttpStatusCode; +import org.springframework.http.MediaType; import org.springframework.http.client.SimpleClientHttpRequestFactory; import org.springframework.stereotype.Service; import org.springframework.web.client.RestClient; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.multipart.MultipartFile; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; @@ -44,17 +38,16 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; -import java.io.*; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.net.URI; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.*; +import static fr.insee.rmes.transfoxsl.service.XsltTransformationService.extractIdentifiersFromDiiInstance; import static fr.insee.rmes.transfoxsl.utils.RestClientUtils.readBodySafely; @Service @@ -62,7 +55,6 @@ public record ColecticaServiceImpl(@NonNull KeycloakServices kc, ElasticService elasticService, RestClient restClient, - XDocReport xdr, ExportUtils exportUtils, DDIDerefencer ddiDerefencer, @Value("${fr.insee.rmes.api.remote.metadata.url}") @@ -139,6 +131,8 @@ public JSONArray findFragmentByUuidWithChildren(String uuid) throws RmesExceptio static class XmlProcessing { + private XmlProcessing(){} + private static JSONArray parseDereferencedXmlWithEnum(byte[] input) throws ParserConfigurationException, IOException, SAXException, XPathExpressionException { // Créer et configurer le DocumentBuilderFactory de manière sécurisée DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); @@ -314,7 +308,7 @@ public String getByType(DDIItemType type) { @Override public void sendUpdateColectica(String ddiUpdatingInJson, TransactionType transactionType) throws RmesException { - int transactionId = extractTransactionId(postWithRestClient(URI.create("transaction"), Optional.empty(), MediaType.APPLICATION_JSON)); + int transactionId = createTransaction(); if (populateTransaction(ddiUpdatingInJson, transactionId)) { if (!commitTransaction(transactionType, transactionId)) { cancelTransaction(transactionId); @@ -326,6 +320,10 @@ public void sendUpdateColectica(String ddiUpdatingInJson, TransactionType transa } } + private int createTransaction() throws RmesException { + return extractTransactionId(postWithRestClient(URI.create("transaction"), Optional.empty(), MediaType.APPLICATION_JSON)); + } + private boolean commitTransaction(TransactionType transactionType, int transactionId) { String commitTransactionRequestBody = TRANSACTIONID + transactionId + ",\"TransactionType\":\"" + transactionType + "\"}"; try { @@ -370,212 +368,38 @@ private void cancelTransaction(int transactionId) throws RmesException { } @Override - public String sendDeleteColectica(String uuid, TransactionType transactionType) { - RestTemplate restTemplate = new RestTemplate(); - String initTransactionUrl = serviceUrl + "/api/v1/transaction"; - String authentToken = kc.getFreshToken(); - - HttpHeaders initTransactionHeaders = new HttpHeaders(); - initTransactionHeaders.setContentType(MediaType.APPLICATION_JSON); - if (authentToken != null && !authentToken.isEmpty()) { - initTransactionHeaders.setBearerAuth(authentToken); - } - HttpEntity initTransactionRequest = new HttpEntity<>(initTransactionHeaders); - ResponseEntity initTransactionResponse = restTemplate.exchange( - initTransactionUrl, - HttpMethod.POST, - initTransactionRequest, - String.class - ); - - if (initTransactionResponse.getStatusCode() == HttpStatus.OK) { - String responseBodyInit = initTransactionResponse.getBody(); - int transactionId = extractTransactionId(responseBodyInit); - - //appel au service DDI instance par uuuid - ResponseEntity test = searchColecticaInstanceByUuid(uuid); - // transformation de la reponse à la requete précédente grâce à une transformation xslt: on obtient une String - // Transformation de la réponse en JSON - String responseBody = (String) test.getBody(); - JSONArray result = extractIdentifiersFromDiiInstance(responseBody); - // Créer l'objet JSON pour la requête - JSONObject deleteTransactionBody = new JSONObject(); - deleteTransactionBody.put("identifiers", result); // Ajouter les identifiants extraits - deleteTransactionBody.put("setIdentifiers", new JSONArray()); // Tableau vide si nécessaire - deleteTransactionBody.put("transactionIds", new JSONArray(Arrays.asList(transactionId))); // Ajouter l'ID de transaction - deleteTransactionBody.put("deleteType", 0); // Ajouter le type de suppression - - // Convertir en chaîne JSON pour la requête - JsonMapper resultFinal = new JsonMapper(); - String deleteTransactionRequestBody = deleteTransactionBody.toString(); - System.out.println("deleteTransactionRequestBody: " + deleteTransactionRequestBody); - // lancement de la requete à destination de l'api colectica - String deleteTransactionUrl = serviceUrl + "/api/v1/item/_delete"; - // Configuration des en-têtes de la requête - HttpHeaders deleteTransactionHeaders = new HttpHeaders(); - deleteTransactionHeaders.setContentType(MediaType.APPLICATION_JSON); - deleteTransactionHeaders.setBearerAuth(authentToken); - - // Création et envoi de la requête - HttpEntity deleteTransactionRequest = new HttpEntity<>(deleteTransactionRequestBody, deleteTransactionHeaders); - ResponseEntity deleteTransactionResponse = restTemplate.exchange( - deleteTransactionUrl, - HttpMethod.POST, - deleteTransactionRequest, - String.class - ); - return deleteTransactionResponse.getStatusCode().toString(); + public String sendDeleteColectica(String uuid, TransactionType transactionType) throws RmesException { + int transactionId; + try{ + transactionId = createTransaction(); + }catch (RmesException e){ + log.error("Error while creating delete transaction : "+e.getMessage(), e); + return "error no TransactionId"; } - return ("error no TransactionId"); - } - - private JSONArray extractIdentifiersFromDiiInstance(String fragmentXml) { - Set uniqueData = new HashSet<>(); - - try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - - // Désactiver l'accès aux entités externes pour des raisons de sécurité (prévention des attaques XXE) - factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - factory.setFeature(DISALLOW_DOCTYPE_DECL1, true); - factory.setFeature(EXTERNAL_GENERAL_ENTITIES, false); - factory.setFeature(EXTERNAL_PARAMETER_ENTITIES, false); - factory.setNamespaceAware(true); - - DocumentBuilder builder = factory.newDocumentBuilder(); - Document document = builder.parse(new ByteArrayInputStream(fragmentXml.getBytes(StandardCharsets.UTF_8))); - - XPathFactory xpathFactory = XPathFactory.newInstance(); - XPath xpath = xpathFactory.newXPath(); - - xpath.setNamespaceContext(new NamespaceContextMap("r", "ddi:reusable:3_3", "ddi", "ddi:instance:3_3")); - - // Sélectionner tous les nœuds ayant un ID, Agency ou Version - NodeList nodes = (NodeList) xpath.evaluate("//*[local-name()='ID' or local-name()='Agency' or local-name()='Version']", document, XPathConstants.NODESET); - - for (int i = 0; i < nodes.getLength(); i++) { - Node currentNode = nodes.item(i).getParentNode(); - - String id = xpath.evaluate("r:ID", currentNode); - String agency = xpath.evaluate("r:Agency", currentNode); - String version = xpath.evaluate("r:Version", currentNode); - - FragmentData fragmentData = new FragmentData(id, agency, version); - uniqueData.add(fragmentData); - } - - // Convertir Set en JSONArray - JSONArray allFragmentsData = new JSONArray(); - for (FragmentData data : uniqueData) { - JSONObject jsonObject = new JSONObject(); - jsonObject.put("Identifier", data.getIdentifier()); - jsonObject.put("AgencyId", data.getAgencyId()); - jsonObject.put("Version", data.getVersion()); - allFragmentsData.put(jsonObject); - } + String deleteTransactionRequestBody = bodyForDeleteWithPostRequest(uuid, transactionId); - return allFragmentsData; + log.debug("deleteTransactionRequestBody: {}", deleteTransactionRequestBody); + // lancement de la requete à destination de l'api colectica + postWithRestClient(URI.create("item/_delete"), Optional.of(deleteTransactionRequestBody), MediaType.APPLICATION_JSON); - } catch (Exception e) { - e.printStackTrace(); - return new JSONArray(); - } - } - - - @Override - public Resource exportCodeBookAsOdt(String ddiFile, File dicoVar) throws RmesException { - //Prepare file - byte[] odt = xdr.exportVariableBookInOdt(ddiFile, dicoVar); - - return new ByteArrayResource(odt); + return HttpStatus.OK.toString(); } - @Override - public ResponseEntity getCodeBookExportV2(String ddi, String xslPatternFile) throws Exception { - - InputStream xslRemoveNameSpaces = getClass().getResourceAsStream("/xslTransformerFiles/remove-namespaces.xsl"); - InputStream xslCheckReference = getClass().getResourceAsStream("/xslTransformerFiles/check-references.xsl"); - String dicoCode = "/xslTransformerFiles/dico-codes.xsl"; - String zipRmes = "/xslTransformerFiles/dicoCodes/toZipForDicoCodes.zip"; - - File ddiRemoveNameSpaces = File.createTempFile("ddiRemoveNameSpaces", ".xml"); - ddiRemoveNameSpaces.deleteOnExit(); - XsltUtils.transformerStringWithXsl(ddi, xslRemoveNameSpaces, ddiRemoveNameSpaces); - - File control = File.createTempFile("control", ".xml"); - control.deleteOnExit(); - XsltUtils.transformerFileWithXsl(ddiRemoveNameSpaces, xslCheckReference, control); - - // Créer un DocumentBuilderFactory sécurisé - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - - // Désactiver l'accès aux entités externes pour des raisons de sécurité (prévention des attaques XXE) - dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - dbf.setFeature(DISALLOW_DOCTYPE_DECL1, true); - dbf.setFeature(EXTERNAL_GENERAL_ENTITIES, false); - dbf.setFeature(EXTERNAL_PARAMETER_ENTITIES, false); - - DocumentBuilder db = dbf.newDocumentBuilder(); - Document doc = db.parse(control); - - String checkResult = doc.getDocumentElement().getNodeName(); - - if (!"OK".equals(checkResult)) { - HttpHeaders headers = new HttpHeaders(); - headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + control.getName()); - headers.add("Cache-Control", "no-cache, no-store, must-revalidate"); - headers.add("Pragma", "no-cache"); - headers.add("Expires", "0"); - - ByteArrayResource resourceByte = new ByteArrayResource(Files.readAllBytes(control.toPath())); - - return ResponseEntity.ok() - .headers(headers) - .contentLength(control.length()) - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .body(resourceByte); - } - - HashMap contentXML = new HashMap<>(); - contentXML.put("ddi-file", Files.readString(ddiRemoveNameSpaces.toPath())); - - return exportUtils.exportAsODT(Path.of("export.odt"), contentXML, dicoCode, xslPatternFile, zipRmes, "dicoVariable"); - } - - - @Override - public ResponseEntity getCodeBookCheck(MultipartFile isCodeBook) throws Exception { - if (isCodeBook == null) - throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Can't generate codebook", "Codebook is null"); - InputStream codeBook = new BufferedInputStream(isCodeBook.getInputStream()); - InputStream xslRemoveNameSpaces = getClass().getResourceAsStream("/xslTransformerFiles/remove-namespaces.xsl"); - File ddiRemoveNameSpaces = File.createTempFile("ddiRemoveNameSpaces", ".xml"); - ddiRemoveNameSpaces.deleteOnExit(); - XsltUtils.transformerInputStreamWithXsl(codeBook, xslRemoveNameSpaces, ddiRemoveNameSpaces); - - InputStream xslCodeBookCheck = getClass().getResourceAsStream("/xslTransformerFiles/dico-codes-test-ddi-content.xsl"); - File codeBookCheck = File.createTempFile("codeBookCheck", ".xml"); - codeBookCheck.deleteOnExit(); - XsltUtils.transformerFileWithXsl(ddiRemoveNameSpaces, xslCodeBookCheck, codeBookCheck); - - HttpHeaders headers = new HttpHeaders(); - headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + codeBookCheck.getName()); - headers.add("Cache-Control", "no-cache, no-store, must-revalidate"); - headers.add("Pragma", "no-cache"); - headers.add("Expires", "0"); - - ByteArrayResource resourceByte = new ByteArrayResource(Files.readAllBytes(codeBookCheck.toPath())); - - if (resourceByte.contentLength() == 0) { - return ResponseEntity.ok("Nothing is missing in the DDI file " + isCodeBook.getOriginalFilename()); - } - return ResponseEntity.ok() - .headers(headers) - .contentLength(codeBookCheck.length()) - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .body(resourceByte); + private String bodyForDeleteWithPostRequest(String uuid, int transactionId) throws RmesException { + //appel au service DDI instance par uuuid + String instance = searchColecticaInstanceByUuid(uuid); + // transformation de la reponse à la requete précédente grâce à une transformation xslt: on obtient une String + // Transformation de la réponse en JSON + JSONArray result = extractIdentifiersFromDiiInstance(instance); + // Créer l'objet JSON pour la requête + JSONObject deleteTransactionBody = new JSONObject(); + deleteTransactionBody.put("identifiers", result); // Ajouter les identifiants extraits + deleteTransactionBody.put("setIdentifiers", new JSONArray()); // Tableau vide si nécessaire + deleteTransactionBody.put("transactionIds", new JSONArray(List.of(transactionId))); // Ajouter l'ID de transaction + deleteTransactionBody.put("deleteType", 0); // Ajouter le type de suppression + // Convertir en chaîne JSON pour la requête + return deleteTransactionBody.toString(); } diff --git a/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java b/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java index 4cf5d836..8e99af21 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java @@ -10,12 +10,15 @@ import fr.insee.rmes.transfoxsl.service.internal.DDIDerefencer; import fr.insee.rmes.transfoxsl.service.internal.DDITransformerToVtl; import fr.insee.rmes.transfoxsl.utils.MultipartFileUtils; +import fr.insee.rmes.utils.FileExtension; +import fr.insee.rmes.utils.export.XDocReport; import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ByteArrayResource; -import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.Resource; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; @@ -34,6 +37,7 @@ import java.util.List; import java.util.UUID; +import static fr.insee.rmes.tocolecticaapi.controller.ControllerUtils.xmltoResponseEntity; import static fr.insee.rmes.transfoxsl.service.internal.DDIDerefencer.DEREFERENCE_XSL; @Controller @@ -44,14 +48,17 @@ public class TransformationController { public static final String FAILED_TO_PROCESS_THE_INPUT_FILE = "Failed to process the input file"; public static final String TRANSFORMATION_FAILED_DURING_THE_XSLT_PROCESSING = "Transformation failed during the XSLT processing"; + public static final MediaType APPLICATION_ODT = MediaType.valueOf("application/vnd.oasis.opendocument.text"); public static final String REL_TO_JSON_XSL = "dataRelToJson.xsl"; private final XsltTransformationService xsltTransformationService; private final MultipartFileUtils multipartFileUtils; + private final XDocReport xDocReport; @Autowired - public TransformationController(XsltTransformationService xsltTransformationService, MultipartFileUtils multipartFileUtils) { + public TransformationController(XsltTransformationService xsltTransformationService, MultipartFileUtils multipartFileUtils, XDocReport xDocReport) { this.xsltTransformationService = xsltTransformationService; this.multipartFileUtils = multipartFileUtils; + this.xDocReport = xDocReport; } @Operation(summary = "Déréférencer un objet DDI") @@ -233,5 +240,100 @@ public ResponseEntity transformFileForComplexCodeList(@RequestParam("fil .body(xsltTransformationService.transformFileForComplexList(file, idValue, nomenclatureName, suggesterDescription, version, idepUtilisateur, timbre, principale, secondaire, labelSecondaire); } + + @GetMapping(value = "/transform/ddi-to-codebook", + consumes = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"}, + produces = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"} + ) + @PreAuthorize("hasRole('Administrateur_RMESGOPS')") + @Operation(operationId = "getCodeBook", summary = "Produce a codebook from a DDI") + + public ResponseEntity ddiToCodeBook( + + @Parameter(schema = @Schema(type = "string", format = "String", description = "Accept")) + @RequestHeader(required = false) String accept, + + @Parameter(schema = @Schema(type = "string", format = "binary", description = "file in DDI")) + @RequestParam(value = "ddiFile") MultipartFile ddiFile, // InputStream isDDI, + + @Parameter(schema = @Schema(type = "string", format = "binary", description = "file for structure")) + @RequestParam(value = "codeBookFile") MultipartFile codeBookFile //InputStream isCodeBook + + ) + throws RmesException { + log.info("Generate CodeBook from DDI {}, {}", ddiFile.getOriginalFilename(), codeBookFile.getOriginalFilename()); + try { + String ddi = new String(ddiFile.getBytes(), StandardCharsets.UTF_8); + Resource response = new ByteArrayResource(xDocReport.exportVariableBookInOdt(ddi, codeBookFile.getBytes())); + log.debug("Codebook is generated"); + return xmltoResponseEntity(response, "Codebook" + FileExtension.forHeader(accept).extension(), APPLICATION_ODT); + } catch (IOException e) { + throw new RmesException(HttpStatus.BAD_REQUEST, e.getMessage(), "Files can't be read"); + } + } + + + @GetMapping(value = "/transform/ddi-to-codebook/V2", + consumes = {MediaType.MULTIPART_FORM_DATA_VALUE}, + produces = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"} + ) + @PreAuthorize("hasRole('Administrateur_RMESGOPS')") + @Operation(operationId = "getCodeBookV2", summary = "Produce a codebook from a DDI") + public ResponseEntity getCodeBookV2( + @Parameter(schema = @Schema(type = "string", allowableValues = {"concis", "concis avec expression", "scindable", "non scindable"})) + @RequestParam(value = "dicoVar") String isCodeBook, //InputStream isCodeBook, + @RequestParam(value = "ddiFile") MultipartFile ddiFile // InputStream isDDI, + ) throws Exception { + + String ddi = new String(ddiFile.getBytes(), StandardCharsets.UTF_8); + String xslPatternFile = null; + switch (isCodeBook) { + case "concis": + String xmlFileConcis = "/xslTransformerFiles/dicoCodes/dicoConcisPatternContent.xml"; + xslPatternFile = xmlFileConcis; + break; + case "concis avec expression": + String xmlFileConcisAvecExpression = "/xslTransformerFiles/dicoCodes/dicoConcisDescrPatternContent.xml"; + xslPatternFile = xmlFileConcisAvecExpression; + break; + case "scindable": + String xmlFileScindable = "/xslTransformerFiles/dicoCodes/dicoScindablePatternContent.xml"; + xslPatternFile = xmlFileScindable; + break; + case "non scindable": + String xmlFileNonScindable = "/xslTransformerFiles/dicoCodes/dicoNonScindablePatternContent.xml"; + xslPatternFile = xmlFileNonScindable; + break; + default: + log.error("Choix incorrect"); + break; + } + + return xDocReport.getCodeBookExportV2(ddi, xslPatternFile); + } + + @GetMapping(value = "/check/is-codebook", + consumes = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"}, + produces = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_OCTET_STREAM_VALUE, "application/vnd.oasis.opendocument.text"} + ) + @PreAuthorize("hasRole('Administrateur_RMESGOPS')") + @Operation(operationId = "getCodeBookCheck", summary = "Check the DDI before made the codebook export") + public ResponseEntity getCodeBookCheck( + @RequestParam(value = "ddiFile") MultipartFile ddiFile)throws Exception { + + HttpHeaders headers = new HttpHeaders(); + headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + codeBookCheck.getName()); + headers.add("Cache-Control", "no-cache, no-store, must-revalidate"); + headers.add("Pragma", "no-cache"); + headers.add("Expires", "0"); + + return xDocReport.getCodeBookCheck(ddiFile.getBytes()); + ResponseEntity.ok() + .headers(headers) + .contentLength(codeBookCheck.length()) + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(resourceByte); + } + } diff --git a/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java b/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java index 6fa4553d..ec136fb0 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java @@ -3,11 +3,14 @@ import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.exceptions.XsltTransformationException; import fr.insee.rmes.model.DDIItemType; -import fr.insee.rmes.tocolecticaapi.RandomUUIDExtensionFunction; +import fr.insee.rmes.tocolecticaapi.models.FragmentData; +import fr.insee.rmes.tocolecticaapi.models.NamespaceContextMap; +import fr.insee.rmes.utils.XMLUtils; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import net.sf.saxon.TransformerFactoryImpl; import net.sf.saxon.s9api.*; +import org.json.JSONArray; +import org.json.JSONObject; import org.springframework.core.io.ByteArrayResource; import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.Resource; @@ -24,26 +27,33 @@ import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.*; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Source; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; @Service @Slf4j public class XsltTransformationService { - public static final String DISALLOW_DOCTYPE_DECL = "http://javax.xml.transform.TransformerFactory/feature/disallow-doctype-decl"; public static final String DISALLOW_DOCTYPE_DECL1 = "http://apache.org/xml/features/disallow-doctype-decl"; public static final String EXTERNAL_GENERAL_ENTITIES = "http://xml.org/sax/features/external-general-entities"; public static final String EXTERNAL_PARAMETER_ENTITIES = "http://xml.org/sax/features/external-parameter-entities"; private static final String VERSION = "version"; static final byte[] DATA_TAG_BEGIN = "".getBytes(); static final byte[] DATA_TAG_END = "".getBytes(); - static final TransformerFactory saxonFactoryWithRandomUUIDExtension = initSaxonFactoryWithRandomUUIDExtension(); private final Processor processor; @@ -52,14 +62,6 @@ public XsltTransformationService() { } - private static TransformerFactory initSaxonFactoryWithRandomUUIDExtension() { - TransformerFactoryImpl factory = new net.sf.saxon.TransformerFactoryImpl(); - ((Processor) factory.getConfiguration().getProcessor()).registerExtensionFunction(new RandomUUIDExtensionFunction()); - - return factory; - } - - public String transformFileForComplexList( MultipartFile file, String idValue, @@ -117,7 +119,7 @@ private byte[] transformToXml(byte[] inputData, InputStream xsltFile, String idV private static byte[] transformWithSaxon(InputStream xsltFile, StreamSource input, Map parameters) throws TransformerException { - Transformer transformer = saxonFactoryWithRandomUUIDExtension.newTransformer(new StreamSource(xsltFile)); + Transformer transformer = XMLUtils.newTransformer(new StreamSource(xsltFile)); parameters.forEach(transformer::setParameter); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); @@ -149,7 +151,7 @@ private byte[] transformToXmlForComplexList(byte[] inputData, InputStream xsltFi } - private String doTransformFile(MultipartFile file, String idepUtilisateur, String jsonToDdiXslFilename, DDIProcessor ddiProcessor, String DdiToJsonXslFilename) throws RmesException { + private String doTransformFile(MultipartFile file, String idepUtilisateur, String jsonToDdiXslFilename, DDIProcessor ddiProcessor, String ddiToJsonXslFilename) throws RmesException { if (file.isEmpty()) { throw new RmesException(HttpStatus.BAD_REQUEST, "Uploaded file is empty", null); } @@ -162,7 +164,7 @@ private String doTransformFile(MultipartFile file, String idepUtilisateur, Strin byte[] xmlContent = ddiProcessor.process(inputData, xsltStream1); - InputStream xsltStream2 = getClass().getResourceAsStream(DdiToJsonXslFilename); + InputStream xsltStream2 = getClass().getResourceAsStream(ddiToJsonXslFilename); return new String(transformToJson(new ByteArrayResource(xmlContent), xsltStream2, idepUtilisateur)); } catch (Exception e) { throw new ServerErrorException("Erreur lors de la transformation du fichier.", e); @@ -209,15 +211,8 @@ byte[] transform(InputStream inputStream, String xslFileName, SerializerConfigur static byte[] transformWithParameters(byte[] source, InputStream xsltFileJson, Map parameters) throws javax.xml.transform.TransformerException { // Créer un transformateur XSLT sécurisé // Désactiver l'accès aux entités externes pour des raisons de sécurité (prévention des attaques XXE) - TransformerFactory factory = TransformerFactory.newInstance(); - factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - factory.setFeature(DISALLOW_DOCTYPE_DECL, true); - factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); - factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "file,jar:file"); - Source xslt = new StreamSource(xsltFileJson); - - Transformer transformer = factory.newTransformer(xslt); + Transformer transformer = XMLUtils.newTransformer(xslt); parameters.forEach(transformer::setParameter); // Effectuer la transformation @@ -313,12 +308,7 @@ public String replaceXmlParameters(String inputXml, DDIItemType type, String lab } private static ByteArrayOutputStream onlyIndent(Document document) throws TransformerException { - // Désactivation des entités externes dans TransformerFactory pour éviter les attaques XXE - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - transformerFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - transformerFactory.setFeature(DISALLOW_DOCTYPE_DECL, true); - - Transformer transformer = transformerFactory.newTransformer(); + Transformer transformer = XMLUtils.newTransformer(); transformer.setOutputProperty(OutputKeys.INDENT, "yes"); ByteArrayOutputStream outputStreamResult = new ByteArrayOutputStream(); @@ -330,6 +320,60 @@ private static String transformToJson(byte[] source, InputStream xsltFileJson, S return new String(transformWithParameters(source, xsltFileJson, Map.of("idepUtilisateur", idepUtilisateur, VERSION, String.valueOf(version)))); } + public static JSONArray extractIdentifiersFromDiiInstance(String fragmentXml) { + Set uniqueData = new HashSet<>(); + + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + + // Désactiver l'accès aux entités externes pour des raisons de sécurité (prévention des attaques XXE) + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setFeature(DISALLOW_DOCTYPE_DECL1, true); + factory.setFeature(EXTERNAL_GENERAL_ENTITIES, false); + factory.setFeature(EXTERNAL_PARAMETER_ENTITIES, false); + + factory.setNamespaceAware(true); + + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(new ByteArrayInputStream(fragmentXml.getBytes(StandardCharsets.UTF_8))); + + XPathFactory xpathFactory = XPathFactory.newInstance(); + XPath xpath = xpathFactory.newXPath(); + + xpath.setNamespaceContext(new NamespaceContextMap("r", "ddi:reusable:3_3", "ddi", "ddi:instance:3_3")); + + // Sélectionner tous les nœuds ayant un ID, Agency ou Version + NodeList nodes = (NodeList) xpath.evaluate("//*[local-name()='ID' or local-name()='Agency' or local-name()='Version']", document, XPathConstants.NODESET); + + for (int i = 0; i < nodes.getLength(); i++) { + Node currentNode = nodes.item(i).getParentNode(); + + String id = xpath.evaluate("r:ID", currentNode); + String agency = xpath.evaluate("r:Agency", currentNode); + String version = xpath.evaluate("r:Version", currentNode); + + FragmentData fragmentData = new FragmentData(id, agency, version); + uniqueData.add(fragmentData); + } + + // Convertir Set en JSONArray + JSONArray allFragmentsData = new JSONArray(); + for (FragmentData data : uniqueData) { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("Identifier", data.getIdentifier()); + jsonObject.put("AgencyId", data.getAgencyId()); + jsonObject.put("Version", data.getVersion()); + allFragmentsData.put(jsonObject); + } + + return allFragmentsData; + + } catch (Exception e) { + log.error("Error while extractIdentifiersFromDiiInstance "+e.getMessage(), e); + return new JSONArray(); + } + } + @FunctionalInterface private interface DDIProcessor { byte[] process(byte[] inputData, InputStream xsltStream) throws TransformerException, IOException; diff --git a/src/main/java/fr/insee/rmes/utils/XMLUtils.java b/src/main/java/fr/insee/rmes/utils/XMLUtils.java index 0c9ac644..cf938a6a 100644 --- a/src/main/java/fr/insee/rmes/utils/XMLUtils.java +++ b/src/main/java/fr/insee/rmes/utils/XMLUtils.java @@ -1,34 +1,27 @@ package fr.insee.rmes.utils; -import fr.insee.rmes.tocolecticaapi.service.Constants; -import org.apache.commons.lang3.StringEscapeUtils; +import fr.insee.rmes.tocolecticaapi.RandomUUIDExtensionFunction; +import lombok.extern.slf4j.Slf4j; +import net.sf.saxon.TransformerFactoryImpl; +import net.sf.saxon.s9api.Processor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Node; -import org.xml.sax.InputSource; import javax.xml.XMLConstants; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.stream.XMLInputFactory; import javax.xml.transform.*; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; -import java.io.StringReader; import java.io.StringWriter; import java.io.Writer; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +@Slf4j public class XMLUtils { + public static final String DISALLOW_DOCTYPE_DECL = "http://javax.xml.transform.TransformerFactory/feature/disallow-doctype-decl"; + private static final TransformerFactory factory = initTransformerFactory(); - private static final String AMP = "&"; - static final Logger logger = LoggerFactory.getLogger(XMLUtils.class); private XMLUtils() { throw new IllegalStateException("Utility class"); @@ -56,107 +49,25 @@ public static Node getChild(Node parent, String childName) { return null; } - public static String produceEmptyXML() { - return(Constants.XML_EMPTY_TAG); - } - - public static Document convertStringToDocument(String xmlStr) { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); // Compliant - factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); // compliant - // disable resolving of external DTD entities - factory.setAttribute(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE); - factory.setAttribute(XMLConstants.FEATURE_SECURE_PROCESSING, Boolean.TRUE); - DocumentBuilder builder; - try { - builder = factory.newDocumentBuilder(); - return builder.parse(new InputSource(new StringReader(xmlStr))); - } catch (Exception e) { - logger.error(e.getMessage()); - } - return null; - } - - public static List getTagValues(String text, String tag) { - final Pattern tagRegex = Pattern.compile("<" + tag + ">(.+?)", Pattern.DOTALL); - final List tagValues = new ArrayList<>(); - final Matcher matcher = tagRegex.matcher(text); - while (matcher.find()) { - tagValues.add(matcher.group(1)); - } - return tagValues; - } - - public static String encodeXml(String response) { - String ret = StringEscapeUtils.unescapeXml(response); - ret = StringEscapeUtils.unescapeHtml4(ret); - - final String regex = "&"; - final Pattern pattern = Pattern.compile(regex, Pattern.MULTILINE | Pattern.DOTALL | Pattern.CASE_INSENSITIVE); - ret = pattern.matcher(ret).replaceAll(AMP); - - final String regex2 = "&amp;"; - final Pattern pattern2 = Pattern.compile(regex2, Pattern.MULTILINE | Pattern.DOTALL | Pattern.CASE_INSENSITIVE); - ret = pattern2.matcher(ret).replaceAll(AMP); - - final String regex3 = "&gt;"; - final Pattern pattern3 = Pattern.compile(regex3, Pattern.MULTILINE | Pattern.DOTALL | Pattern.CASE_INSENSITIVE); - ret = pattern3.matcher(ret).replaceAll(">"); - - final String regex4 = "&lt;"; - final Pattern pattern4 = Pattern.compile(regex4, Pattern.MULTILINE | Pattern.DOTALL | Pattern.CASE_INSENSITIVE); - ret = pattern4.matcher(ret).replaceAll("<"); - - final String regex8 = "&quot;"; - final Pattern pattern8 = Pattern.compile(regex8, Pattern.MULTILINE | Pattern.DOTALL | Pattern.CASE_INSENSITIVE); - ret = pattern8.matcher(ret).replaceAll("""); - - final String regex5 = Constants.XML_ESPERLUETTE_REPLACEMENT; - final Pattern pattern5 = Pattern.compile(regex5, Pattern.MULTILINE | Pattern.DOTALL | Pattern.CASE_INSENSITIVE); - ret = pattern5.matcher(ret).replaceAll(AMP); - - final String regex6 = Constants.XML_SUP_REPLACEMENT; - final Pattern pattern6 = Pattern.compile(regex6, Pattern.MULTILINE | Pattern.DOTALL | Pattern.CASE_INSENSITIVE); - ret = pattern6.matcher(ret).replaceAll(">"); - - final String regex7 = Constants.XML_INF_REPLACEMENT; - final Pattern pattern7 = Pattern.compile(regex7, Pattern.MULTILINE | Pattern.DOTALL | Pattern.CASE_INSENSITIVE); - ret = pattern7.matcher(ret).replaceAll("<"); - - return new String(ret.getBytes(), StandardCharsets.UTF_8); - } - - public static String solveSpecialXmlcharacters(String rubric) { - String ret = rubric.replace(""", Constants.XML_ESPERLUETTE_REPLACEMENT + "quot;"); //Quotes are not authorized in Json - ret = StringEscapeUtils.unescapeXml(ret); - ret = StringEscapeUtils.unescapeHtml4(ret); - //ret=rubric - - final String regex = "&"; - final Pattern pattern = Pattern.compile(regex, Pattern.MULTILINE | Pattern.DOTALL | Pattern.CASE_INSENSITIVE); - ret = pattern.matcher(ret).replaceAll(Constants.XML_ESPERLUETTE_REPLACEMENT); - - final String regex2 = "<"; - final Pattern pattern2 = Pattern.compile(regex2, Pattern.MULTILINE | Pattern.DOTALL | Pattern.CASE_INSENSITIVE); - ret = pattern2.matcher(ret).replaceAll(Constants.XML_INF_REPLACEMENT); - - final String regex3 = ">"; - final Pattern pattern3 = Pattern.compile(regex3, Pattern.MULTILINE | Pattern.DOTALL | Pattern.CASE_INSENSITIVE); - ret = pattern3.matcher(ret).replaceAll(Constants.XML_SUP_REPLACEMENT); - - return new String(ret.getBytes(), StandardCharsets.UTF_8); - } - private static TransformerFactory initTransformerFactory() { - TransformerFactory factory = new net.sf.saxon.TransformerFactoryImpl(); + TransformerFactoryImpl factory = new net.sf.saxon.TransformerFactoryImpl(); try { factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - } catch (TransformerConfigurationException ignored) { - logger.info(XMLConstants.FEATURE_SECURE_PROCESSING+" unsuported for net.sf.saxon.TransformerFactoryImpl"); + factory.setFeature(DISALLOW_DOCTYPE_DECL, true); + } catch (TransformerConfigurationException e) { + log.info("unsuported feature for net.sf.saxon.TransformerFactoryImpl : {}", e.getMessage()); } factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "file,jar:file"); + ((Processor) factory.getConfiguration().getProcessor()).registerExtensionFunction(new RandomUUIDExtensionFunction()); return factory; } - + + public static Transformer newTransformer(Source xslt) throws TransformerConfigurationException { + return factory.newTransformer(xslt); + } + + public static Transformer newTransformer() throws TransformerConfigurationException { + return factory.newTransformer(); + } } diff --git a/src/main/java/fr/insee/rmes/utils/XsltUtils.java b/src/main/java/fr/insee/rmes/utils/XsltUtils.java index 4cf7ee29..067914fb 100644 --- a/src/main/java/fr/insee/rmes/utils/XsltUtils.java +++ b/src/main/java/fr/insee/rmes/utils/XsltUtils.java @@ -1,7 +1,6 @@ package fr.insee.rmes.utils; import fr.insee.rmes.exceptions.RmesException; -import fr.insee.rmes.tocolecticaapi.service.Constants; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +30,7 @@ public static void xsltTransform(Map xmlContent, InputStream odt PrintStream printStream, Path tempDir) throws TransformerException { // prepare transformer StreamSource xsrc = new StreamSource(xslFileIS); - Transformer xsltTransformer = XMLUtils.getTransformerFactory().newTransformer(xsrc); + Transformer xsltTransformer = XMLUtils.newTransformer(xsrc); // Pass parameters in a file to the transformer xmlContent.forEach((paramName, xmlData) -> { @@ -51,7 +50,6 @@ private static void addParameter(Transformer xsltTransformer, String paramName, CopyOption[] options = { StandardCopyOption.REPLACE_EXISTING }; try { Path tempFile = Files.createTempFile(tempDir, paramName, FileExtension.XML_EXTENSION.extension()); - String absolutePath = tempFile.toFile().getAbsolutePath(); InputStream is = IOUtils.toInputStream(paramData, StandardCharsets.UTF_8); Files.copy(is, tempFile, options); @@ -66,60 +64,30 @@ private static void addParameter(Transformer xsltTransformer, String paramName, public static void createOdtFromXml(Path outputPath, Path finalPath, InputStream zipToCompleteIS, Path tempDir) throws IOException { - Path contentPath = Paths.get(tempDir.toString() + "/content.xml"); + Path contentPath = Paths.get(tempDir + "/content.xml"); Files.copy(outputPath, contentPath, StandardCopyOption.REPLACE_EXISTING); - Path zipPath = Paths.get(tempDir.toString() + "/export.zip"); + Path zipPath = Paths.get(tempDir + "/export.zip"); Files.copy(zipToCompleteIS, zipPath, StandardCopyOption.REPLACE_EXISTING); FilesUtils.addFileToZipFolder(contentPath, zipPath); Files.copy(zipPath, finalPath, StandardCopyOption.REPLACE_EXISTING); } - - public static String buildParams(Boolean lg1, Boolean lg2, Boolean includeEmptyFields, String targetType) { - String includeEmptyFieldsString = (Boolean.TRUE.equals(includeEmptyFields) ? "true" : "false"); - String parametersXML = ""; - - parametersXML = parametersXML.concat(Constants.XML_OPEN_PARAMETERS_TAG); - - parametersXML = parametersXML.concat(Constants.XML_OPEN_LANGUAGES_TAG); - if (Boolean.TRUE.equals(lg1)) - parametersXML = parametersXML.concat("1"); - if (Boolean.TRUE.equals(lg2)) - parametersXML = parametersXML.concat("2"); - parametersXML = parametersXML.concat(Constants.XML_END_LANGUAGES_TAG); - - parametersXML = parametersXML.concat(Constants.XML_OPEN_INCLUDE_EMPTY_FIELDS_TAG); - parametersXML = parametersXML.concat(includeEmptyFieldsString); - parametersXML = parametersXML.concat(Constants.XML_END_INCLUDE_EMPTY_FIELDS_TAG); - parametersXML = parametersXML.concat(Constants.XML_OPEN_TARGET_TYPE_TAG); - parametersXML = parametersXML.concat(targetType); - parametersXML = parametersXML.concat(Constants.XML_END_TARGET_TYPE_TAG); + public static byte[] transformerInputStreamWithXsl(byte[] input,InputStream xslCheckReference) throws TransformerException { + return doTransform(new StreamSource(xslCheckReference),new StreamSource(new ByteArrayInputStream(input)) ); + } - parametersXML = parametersXML.concat(Constants.XML_END_PARAMETERS_TAG); - return XMLUtils.encodeXml(parametersXML); + public static byte[] transformerStringWithXsl(String ddi, InputStream xslRemoveNameSpaces) throws TransformerException { + return doTransform(new StreamSource(xslRemoveNameSpaces), new StreamSource(new StringReader(ddi))); } - public static void transformerInputStreamWithXsl(InputStream input,InputStream xslCheckReference, File output) throws Exception { - Source stylesheetSource = new StreamSource(xslCheckReference); - Transformer transformer = XMLUtils.getTransformerFactory().newTransformer(stylesheetSource); - Source inputSource = new StreamSource(input); - Result outputResult = new StreamResult(output); - transformer.transform(inputSource, outputResult); - } - - public static void transformerStringWithXsl(String ddi,InputStream xslRemoveNameSpaces, File output) throws Exception{ - Source stylesheetSource = new StreamSource(xslRemoveNameSpaces); - Transformer transformer = XMLUtils.getTransformerFactory().newTransformer(stylesheetSource); - Source inputSource = new StreamSource(new StringReader(ddi)); - Result outputResult = new StreamResult(output); - transformer.transform(inputSource, outputResult); + public static byte[] transformerFileWithXsl(File input, InputStream xslCheckReference) throws TransformerException { + return doTransform(new StreamSource(xslCheckReference), new StreamSource(input)); } - public static void transformerFileWithXsl(File input, InputStream xslCheckReference, File output) throws Exception { - Source stylesheetSource = new StreamSource(xslCheckReference); - Transformer transformer = XMLUtils.getTransformerFactory().newTransformer(stylesheetSource); - Source inputSource = new StreamSource(input); - Result outputResult = new StreamResult(output); - transformer.transform(inputSource, outputResult); + private static byte[] doTransform(Source stylesheetSource, Source inputSource) throws TransformerException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Result outputResult = new StreamResult(baos); + XMLUtils.newTransformer(stylesheetSource).transform(inputSource, outputResult); + return baos.toByteArray(); } } diff --git a/src/main/java/fr/insee/rmes/utils/export/XDocReport.java b/src/main/java/fr/insee/rmes/utils/export/XDocReport.java index 6746df65..7c9d3f85 100644 --- a/src/main/java/fr/insee/rmes/utils/export/XDocReport.java +++ b/src/main/java/fr/insee/rmes/utils/export/XDocReport.java @@ -1,7 +1,8 @@ package fr.insee.rmes.utils.export; -import fr.insee.rmes.tocolecticaapi.service.VarBookExportBuilder; import fr.insee.rmes.exceptions.RmesException; +import fr.insee.rmes.tocolecticaapi.service.VarBookExportBuilder; +import fr.insee.rmes.utils.XsltUtils; import fr.opensagres.xdocreport.core.XDocReportException; import fr.opensagres.xdocreport.document.IXDocReport; import fr.opensagres.xdocreport.document.registry.XDocReportRegistry; @@ -10,21 +11,94 @@ import freemarker.ext.dom.NodeModel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; +import org.w3c.dom.Document; import org.xml.sax.InputSource; import org.xml.sax.SAXException; +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; + +import static fr.insee.rmes.transfoxsl.service.XsltTransformationService.*; @Component public record XDocReport(VarBookExportBuilder varBookExport) { static final Logger logger = LoggerFactory.getLogger(XDocReport.class); + public byte[] getCodeBookExportV2(String ddi, String xslPatternFile) { + + InputStream xslRemoveNameSpaces = getClass().getResourceAsStream("/xslTransformerFiles/remove-namespaces.xsl"); + InputStream xslCheckReference = getClass().getResourceAsStream("/xslTransformerFiles/check-references.xsl"); + String dicoCode = "/xslTransformerFiles/dico-codes.xsl"; + String zipRmes = "/xslTransformerFiles/dicoCodes/toZipForDicoCodes.zip"; + + byte[] ddiRemoveNameSpace = XsltUtils.transformerStringWithXsl(ddi, xslRemoveNameSpaces); + byte[] control = XsltUtils.transformerInputStreamWithXsl(ddiRemoveNameSpace, xslCheckReference); + + // Créer un DocumentBuilderFactory sécurisé + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + + // Désactiver l'accès aux entités externes pour des raisons de sécurité (prévention des attaques XXE) + dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + dbf.setFeature(DISALLOW_DOCTYPE_DECL1, true); + dbf.setFeature(EXTERNAL_GENERAL_ENTITIES, false); + dbf.setFeature(EXTERNAL_PARAMETER_ENTITIES, false); + + DocumentBuilder db = dbf.newDocumentBuilder(); + + + Document doc = db.parse(new ByteArrayInputStream(control)); + + String checkResult = doc.getDocumentElement().getNodeName(); + + if (!"OK".equals(checkResult)) { + HttpHeaders headers = new HttpHeaders(); + headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + control.getName()); + headers.add("Cache-Control", "no-cache, no-store, must-revalidate"); + headers.add("Pragma", "no-cache"); + headers.add("Expires", "0"); + + ByteArrayResource resourceByte = new ByteArrayResource(Files.readAllBytes(((InputStream) new ByteArrayInputStream(XsltUtils.transformerInputStreamWithXsl(XsltUtils.transformerStringWithXsl(ddi, xslRemoveNameSpaces), xslCheckReference))).toPath())); + + return ResponseEntity.ok() + .headers(headers) + .contentLength(control.length()) + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(resourceByte); + } + + HashMap contentXML = new HashMap<>(); + contentXML.put("ddi-file", Files.readString(ddiRemoveNameSpaces.toPath())); + + return exportUtils.exportAsODT(Path.of("export.odt"), contentXML, dicoCode, xslPatternFile, zipRmes, "dicoVariable"); + } + + + public byte[] getCodeBookCheck(byte[] ddiContent) throws Exception { + + InputStream xslRemoveNameSpaces = getClass().getResourceAsStream("/xslTransformerFiles/remove-namespaces.xsl"); + File ddiRemoveNameSpaces = File.createTempFile("ddiRemoveNameSpaces", ".xml"); + ddiRemoveNameSpaces.deleteOnExit(); + XsltUtils.transformerInputStreamWithXsl(ddiContent, xslRemoveNameSpaces, ddiRemoveNameSpaces); + + InputStream xslCodeBookCheck = getClass().getResourceAsStream("/xslTransformerFiles/dico-codes-test-ddi-content.xsl"); + + return XsltUtils.transformerFileWithXsl(ddiRemoveNameSpaces, xslCodeBookCheck); + } - public byte[] exportVariableBookInOdt(String xml, File odtTemplate) throws RmesException { + public byte[] exportVariableBookInOdt(String xml, byte[] odtTemplate) throws RmesException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); IXDocReport report; @@ -63,8 +137,8 @@ private IContext getXmlData(IXDocReport report, String xmlInput) return context; } - private IXDocReport getReportTemplate(File odtTemplate) throws IOException, XDocReportException { - try(InputStream is = new FileInputStream(odtTemplate)){ + private IXDocReport getReportTemplate(byte[] odtTemplate) throws IOException, XDocReportException { + try(InputStream is = new ByteArrayInputStream(odtTemplate)){ return XDocReportRegistry.getRegistry().loadReport(is,TemplateEngineKind.Freemarker); } From 25cb1f4260cfd75c72eb2fd685b44ed39b097f03 Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Tue, 14 Jan 2025 15:49:03 +0100 Subject: [PATCH 08/25] refactor (mutulualize, separate concerns, remove useless, remove redondant, remove elastic search) : compile --- .../config/keycloak/KeycloakServices.java | 161 +++++------------- .../fr/insee/rmes/config/keycloak/Token.java | 15 +- .../tocolecticaapi/controller/DeleteItem.java | 10 +- .../tocolecticaapi/controller/GetItem.java | 41 +++-- .../tocolecticaapi/controller/PostItem.java | 27 +-- ...iFragment.java => DdiFragmentService.java} | 6 +- .../service/ColecticaService.java | 4 +- .../service/ColecticaServiceImpl.java | 13 +- .../service/ElasticService.java | 9 +- .../service/VarBookExportBuilder.java | 9 +- .../controller/TransformationController.java | 105 ++++-------- .../service/XsltTransformationService.java | 8 +- .../service/internal/DDITransformerToVtl.java | 4 +- .../rmes/utils/DocumentBuilderUtils.java | 38 ++--- .../fr/insee/rmes/utils/DocumentBuilders.java | 8 +- .../java/fr/insee/rmes/utils/ExportUtils.java | 106 ++++-------- .../java/fr/insee/rmes/utils/HttpUtils.java | 1 - .../java/fr/insee/rmes/utils/XsltUtils.java | 77 +++------ .../insee/rmes/utils/export/XDocReport.java | 85 ++++----- src/main/resources/application.properties | 3 +- .../controller/GetItemTest.java | 6 +- 21 files changed, 255 insertions(+), 481 deletions(-) rename src/main/java/fr/insee/rmes/tocolecticaapi/fragments/{DdiFragment.java => DdiFragmentService.java} (72%) diff --git a/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java b/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java index b2bc9032..86a60cea 100644 --- a/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java +++ b/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java @@ -1,162 +1,91 @@ package fr.insee.rmes.config.keycloak; import com.auth0.jwt.JWT; -import com.auth0.jwt.exceptions.JWTDecodeException; -import com.auth0.jwt.interfaces.DecodedJWT; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import fr.insee.rmes.tocolecticaapi.models.AuthRequest; -import fr.insee.rmes.exceptions.ExceptionColecticaUnreachable; import lombok.extern.slf4j.Slf4j; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.*; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; import org.springframework.stereotype.Service; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import org.springframework.web.client.RestClient; -import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; -import java.util.Date; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.atomic.AtomicReference; @Service @Slf4j public class KeycloakServices { + final String secret; - @Value("${fr.insee.rmes.metadata.keycloak.secret}") - String secret; + final String resource; - @Value("${fr.insee.rmes.metadata.keycloak.resource}") - String resource; + final String server; - @Value("${fr.insee.rmes.metadata.keycloak.server}") - String server; + final String realm; - @Value("${fr.insee.rmes.metadata.keycloak.realm}") - String realm; - - @Value("${fr.insee.rmes.api.remote.metadata.url}") - String testKube; - - @Value("${auth.api.url}") - private String authApiUrl; - - @Value("${item.api.url}") - private String itemApiUrl; - - @Value("${auth.username}") - private String username; - - @Value("${auth.password}") - private String password; - private static final Logger logger = LoggerFactory.getLogger(KeycloakServices.class); private String token; + //TODO is it the right way to synchronize ? + private final AtomicReference expiration; + + public KeycloakServices(@Value("${fr.insee.rmes.metadata.keycloak.secret}") String secret, @Value("${fr.insee.rmes.metadata.keycloak.resource}") String resource, @Value("${fr.insee.rmes.metadata.keycloak.server}") String server, @Value("${fr.insee.rmes.metadata.keycloak.realm}") String realm) { + this.secret = secret; + this.resource = resource; + this.server = server; + this.realm = realm; + this.expiration = new AtomicReference<>(); + } /** * Permet de récuperer un jeton keycloak * * @return jeton */ - public String getKeycloakAccessToken() { - - if (!testKube.contains(".dev.kube.insee.fr")) { - RestTemplate keycloakClient = new RestTemplate(); - String keycloakUrl = server + "/realms/" + realm + "/protocol/openid-connect/token"; - - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + private String getKeycloakAccessToken() { - MultiValueMap body = new LinkedMultiValueMap<>(); - body.add("grant_type", "client_credentials"); - body.add("client_id", resource); - body.add("client_secret", secret); - HttpEntity entity = new HttpEntity<>(body, headers); + RestTemplate keycloakClient = new RestTemplate(); + String keycloakUrl = server + "/realms/" + realm + "/protocol/openid-connect/token"; - Token accessToken = keycloakClient.postForObject(keycloakUrl, entity, Token.class); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); - log.trace("Keycloak token provided"); - return accessToken.getAccessToken(); + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("grant_type", "client_credentials"); + body.add("client_id", resource); + body.add("client_secret", secret); + HttpEntity entity = new HttpEntity<>(body, headers); - } else { + Token accessToken = keycloakClient.postForObject(keycloakUrl, entity, Token.class); - return extractAccessToken(getAuthToken()); - } + log.trace("Keycloak token provided"); + return accessToken.accessToken(); } /** - * Verifie si le jeton keycloak a expiré + * check if oidc token has expired * - * @param token * @return boolean */ - public boolean isTokenValid(String token) { - if (token == null) { - return false; - } - boolean isValid = false; - Date now = new Date(); - try { - DecodedJWT jwt = JWT.decode(token); - if (jwt.getExpiresAt().after(now)) { - log.info("Token is valid"); - isValid = true; - } - } catch (JWTDecodeException exception) { - System.out.println("erreur" + exception.toString()); - - } - return isValid; - } - - private String getAuthToken() { - RestTemplate restTemplate = new RestTemplate(); - - // TODO utiliser - //RestClient.create().post().contentType(MediaType.APPLICATION_JSON).body(new AuthRequest("","")).retrieve().body(String.class); - - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - - // Créer un objet représentant les informations d'identification - AuthRequest authRequest = new AuthRequest(username, password); - - // Convertir l'objet en JSON - ObjectMapper mapper = new ObjectMapper(); - String requestBody = mapper.writeValueAsString(authRequest); - - HttpEntity request = new HttpEntity<>(requestBody, headers); - - ResponseEntity response = restTemplate.postForEntity(authApiUrl, request, String.class); - - if (response.getStatusCode() == HttpStatus.OK) { - return response.getBody(); - } else { - throw new RuntimeException("Impossible d'obtenir le token d'authentification."); - } - } - - public static String extractAccessToken(String token) { - try { - JSONParser parser = new JSONParser(); - JSONObject json = (JSONObject) parser.parse(token); - return (String) json.get("access_token"); - } catch (Exception e) { - logger.error("Erreur lors de l'extraction de l'access token : {}", e.getMessage(), e); - } - return null; + private boolean isCurrentTokenValid() { + return expiration.get() != null && Instant.now().isBefore(expiration.get().minus(1, ChronoUnit.SECONDS)); } public String getFreshToken() { - if (!this.isTokenValid(this.token)) { - token = getKeycloakAccessToken(); + if (!this.isCurrentTokenValid()) { + synchronized (this.expiration) { + token = getKeycloakAccessToken(); + this.expiration.set(decodeToken(token)); + } } return token; } + private Instant decodeToken(String token) { + return JWT.decode(token).getExpiresAtAsInstant(); + } + } diff --git a/src/main/java/fr/insee/rmes/config/keycloak/Token.java b/src/main/java/fr/insee/rmes/config/keycloak/Token.java index 49fdb092..f3624d65 100644 --- a/src/main/java/fr/insee/rmes/config/keycloak/Token.java +++ b/src/main/java/fr/insee/rmes/config/keycloak/Token.java @@ -2,17 +2,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; -public class Token { - @JsonProperty("access_token") - String accessToken; - - private String tokentype; - - public String getAccessToken() { - return accessToken; - } - - public String getTokenType() { - return tokentype; - } +record Token ( @JsonProperty("access_token") + String accessToken){ } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/DeleteItem.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/DeleteItem.java index b06c54da..05ed1ad1 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/DeleteItem.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/DeleteItem.java @@ -1,6 +1,7 @@ package fr.insee.rmes.tocolecticaapi.controller; import fr.insee.rmes.exceptions.ExceptionColecticaUnreachable; +import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.tocolecticaapi.models.TransactionType; import fr.insee.rmes.tocolecticaapi.service.ColecticaService; import io.swagger.v3.oas.annotations.Operation; @@ -12,12 +13,12 @@ import org.apache.logging.log4j.Logger; import org.json.simple.parser.ParseException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.client.RestTemplate; import java.io.IOException; @@ -51,9 +52,10 @@ public DeleteItem(ColecticaService colecticaService) { @PreAuthorize("hasRole('Administrateur_RMESGOPS')") @Operation(summary = "Delete a CodeList via Colectica API", description = "Delete a CodeList and all their children.") - public String deleteCodeList(@RequestParam("uuid") String uuid) - throws IOException, ExceptionColecticaUnreachable, ParseException { - return colecticaService.sendDeleteColectica(uuid, TransactionType.COPYCOMMIT); + public ResponseEntity deleteCodeList(@RequestParam("uuid") String uuid) + throws IOException, ExceptionColecticaUnreachable, ParseException, RmesException { + colecticaService.sendDeleteColectica(uuid, TransactionType.COPYCOMMIT); + return ResponseEntity.noContent().build(); } } \ No newline at end of file diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java index 115eab28..d3ec8dc2 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java @@ -5,7 +5,7 @@ import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.exceptions.RmesExceptionIO; import fr.insee.rmes.model.DDIItemType; -import fr.insee.rmes.tocolecticaapi.fragments.DdiFragment; +import fr.insee.rmes.tocolecticaapi.fragments.DdiFragmentService; import fr.insee.rmes.tocolecticaapi.service.ColecticaService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -19,7 +19,6 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; -import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -32,7 +31,7 @@ public class GetItem { private final ColecticaService colecticaService; - private final DdiFragment ddiFragment; + private final DdiFragmentService ddiFragmentService; @GetMapping(value = "fragmentInstance/uuid", produces = MediaType.APPLICATION_XML_VALUE) @Operation(summary = "Get FragmentInstance by uuid", description = "Get an XML document for a Fragment:Instance from Colectica repository.") @@ -41,8 +40,8 @@ public ResponseEntity findInstanceByUuidColectica( description = "id de l'objet colectica sous la forme uuid/version", required = true, schema = @Schema( - type = "string", example="d6c08ec1-c4d2-4b9a-b358-b23aa4e0af93")) String uuid) throws RmesExceptionIO, ParseException { - return colecticaService.findInstanceByUuid(uuid); + type = "string", example="d6c08ec1-c4d2-4b9a-b358-b23aa4e0af93")) String uuid) throws RmesException { + return ResponseEntity.ok(colecticaService.searchColecticaInstanceByUuid(uuid)); } @GetMapping(value = "ddiFragment/{uuid}/dataRelationship", produces = MediaType.APPLICATION_JSON_VALUE) @@ -51,8 +50,8 @@ public ResponseEntity extractDataRelationshipFromFragment(@Parameter( description = "id du fragment DDI sous la forme uuid", required = true, schema = @Schema(type = "string", example="d6c08ec1-c4d2-4b9a-b358-b23aa4e0af93")) - @PathVariable String uuid){ - return ResponseEntity.ok(this.ddiFragment.extractDataRelationship(uuid)); + @PathVariable String uuid) throws RmesException { + return ResponseEntity.ok(this.ddiFragmentService.extractDataRelationship(uuid)); } @GetMapping(value = "ddiFragment/uuid", produces = MediaType.APPLICATION_XML_VALUE) @@ -62,22 +61,22 @@ public ResponseEntity findFragmentByUuidColectica( description = "id de l'objet colectica sous la forme uuid/version", required = true, schema = @Schema( - type = "string", example="d6c08ec1-c4d2-4b9a-b358-b23aa4e0af93")) String uuid) throws RmesException { - return getResponseEntitySearchColecticaFragmentByUuid(colecticaService.findFragmentByUuid(uuid)); + type = "string", example="d6c08ec1-c4d2-4b9a-b358-b23aa4e0af93")) String uuid) { + return ResponseEntity.ok(colecticaService.findFragmentByUuid(uuid)); } - @GetMapping(value = "FragmentInstance/uuid/withChildren", produces = MediaType.APPLICATION_XML_VALUE) + @GetMapping(value = "FragmentInstance/uuid/withChildren", produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Get Fragments by uuid", description = "Get an XML document for a Fragment:Instance from Colectica repository.") - public String findFragmentByUuidWithChildrenColectica( + public ResponseEntity findFragmentByUuidWithChildrenColectica( @Parameter( description = "id de l'objet colectica sous la forme uuid/version", required = true, schema = @Schema( - type = "string", example="d6c08ec1-c4d2-4b9a-b358-b23aa4e0af93/2")) String uuid) throws Exception { - return colecticaService.findFragmentByUuidWithChildren(uuid); + type = "string", example="d6c08ec1-c4d2-4b9a-b358-b23aa4e0af93/2")) String uuid) throws RmesException { + return ResponseEntity.ok(colecticaService.findFragmentByUuidWithChildren(uuid).toString()); } @@ -94,8 +93,8 @@ public ResponseEntity filteredSearchText( description = "texte à rechercher. le * sert de wildcard", required = true, schema = @Schema( - type = "string", example="sugg*")) String texte) { - return colecticaService.filteredSearchText(index, texte); + type = "string", example="sugg*")) String texte) throws RmesException { + return ResponseEntity.ok(colecticaService.filteredSearchText(index, texte)); } @GetMapping("/filtered-search/texteByType") @@ -117,7 +116,7 @@ public ResponseEntity filteredSearchTextByType( description = "type à selectionner", required = true) @RequestParam DDIItemType ddiItemType - ) throws UnsupportedEncodingException { + ) throws RmesException { // Validation des entrées if (!index.matches("^[a-zA-Z0-9_*]+$") || !texte.matches("^[a-zA-Z0-9_*]+$")) { @@ -125,11 +124,11 @@ public ResponseEntity filteredSearchTextByType( } // Encodage des entrées - String encodedIndex = URLEncoder.encode(index, StandardCharsets.UTF_8.toString()); - String encodedText = URLEncoder.encode(texte, StandardCharsets.UTF_8.toString()); + String encodedIndex = URLEncoder.encode(index, StandardCharsets.UTF_8); + String encodedText = URLEncoder.encode(texte, StandardCharsets.UTF_8); // Appel au service avec les valeurs encodées - return colecticaService.searchTexteByType(encodedIndex, encodedText, ddiItemType); + return ResponseEntity.ok(colecticaService.searchTexteByType(encodedIndex, encodedText, ddiItemType)); } @@ -144,8 +143,8 @@ public ResponseEntity searchByType( String index , @Parameter( description = "type à selectionner", - required = true) @RequestParam DDIItemType ddiItemType) { - return colecticaService.searchByType(index, ddiItemType); + required = true) @RequestParam DDIItemType ddiItemType) throws RmesException { + return ResponseEntity.ok(colecticaService.searchByType(index, ddiItemType)); } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java index bbc15b12..bc1a4e1c 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java @@ -8,32 +8,20 @@ import fr.insee.rmes.tocolecticaapi.service.ColecticaService; import io.swagger.v3.oas.annotations.Hidden; import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.json.simple.parser.ParseException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.security.core.annotation.AuthenticationPrincipal; -import org.springframework.security.oauth2.core.oidc.user.OidcUser; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; -import java.io.BufferedInputStream; import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -import static fr.insee.rmes.tocolecticaapi.controller.ControllerUtils.xmltoResponseEntity; @Controller @RequestMapping("/postItem") @@ -51,7 +39,6 @@ }) public class PostItem { - static final Logger logger = LogManager.getLogger(PostItem.class); private final ColecticaService colecticaService; @Autowired @@ -66,7 +53,7 @@ public PostItem(ColecticaService colecticaService) { @PreAuthorize("hasRole('Administrateur_RMESGOPS')") public ResponseEntity sendUpdateColectica( @RequestBody String ddiUpdatingInJson, - @RequestParam("transactionType") TransactionType transactionType) throws RmesException { + @PathVariable("transactionType") TransactionType transactionType) throws RmesException { colecticaService.sendUpdateColectica(ddiUpdatingInJson, transactionType); return ResponseEntity.ok("Transaction success"); } @@ -76,22 +63,22 @@ public ResponseEntity sendUpdateColectica( @Operation(summary = "Send suggester JSON to Colectica Repository via Colectica API", description = "Send a suggester JSON to /api/v1/item. This suggester must be simple, a list of Id,Label transform with transformJsonToJsonForAPi") @PreAuthorize("hasRole('Administrateur_RMESGOPS')") - public ResponseEntity uploadItem(@RequestParam("file") MultipartFile file, @AuthenticationPrincipal - OidcUser principal) - throws IOException, ExceptionColecticaUnreachable { + public ResponseEntity uploadItem(@RequestParam("file") MultipartFile file) + throws IOException, RmesException { String fileContent = new String(file.getBytes()); - return colecticaService.sendUpdateColectica(fileContent, TransactionType.COPYCOMMIT); + colecticaService.sendUpdateColectica(fileContent, TransactionType.COPYCOMMIT); + return ResponseEntity.noContent().build(); } @Hidden - @PostMapping("{type}/json") + @PostMapping(value = "{type}/json", produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Get JSON for a type of DDI item", description = "Get a JSON list of item for a type of DDI items .") @PreAuthorize("hasRole('Administrateur_RMESGOPS')") public ResponseEntity byType( @PathVariable("type") DDIItemType type) throws IOException, ExceptionColecticaUnreachable, ParseException { - return colecticaService.getByType(type); + return ResponseEntity.ok(colecticaService.getByType(type)); } } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragment.java b/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentService.java similarity index 72% rename from src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragment.java rename to src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentService.java index 85d34070..aa095345 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragment.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentService.java @@ -1,11 +1,13 @@ package fr.insee.rmes.tocolecticaapi.fragments; -public interface DdiFragment { +import fr.insee.rmes.exceptions.RmesException; + +public interface DdiFragmentService { /** * Retourne une projection json de l'objet datarelationship du fragment DDI dont l'id est passé en paramètre. * Le DDI fragment est fourni en interrogeant colectica * @param uuid : l'id du DDI fragment dont on doit extraire le datarelationship * @return une chaine de caractères représentant le datarelationship */ - String extractDataRelationship(String uuid); + String extractDataRelationship(String uuid) throws RmesException; } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java index b63873aa..239a9fdf 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java @@ -16,9 +16,9 @@ public interface ColecticaService { String findFragmentByUuid(String uuid); - String sendDeleteColectica(String uuid, TransactionType transactionType) throws JsonProcessingException, ExceptionColecticaUnreachable, RmesExceptionIO, ParseException, RmesException; + void sendDeleteColectica(String uuid, TransactionType transactionType) throws JsonProcessingException, ExceptionColecticaUnreachable, RmesExceptionIO, ParseException, RmesException; String searchColecticaInstanceByUuid(String uuid) throws RmesException; - JSONArray findFragmentByUuidWithChildren(String uuid) throws Exception; + JSONArray findFragmentByUuidWithChildren(String uuid) throws RmesException; String filteredSearchText(String index, String texte) throws RmesException; diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java index c2fccfac..35fd8299 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import fr.insee.rmes.config.keycloak.KeycloakServices; +import fr.insee.rmes.exceptions.RmesBadRequestException; import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.exceptions.RmesExceptionIO; import fr.insee.rmes.model.DDIItemType; @@ -67,6 +68,10 @@ public record ColecticaServiceImpl(@NonNull KeycloakServices kc, private static final String BEARER = "Bearer "; private static final String TRANSACTIONID = "{\"TransactionId\":"; + public ColecticaServiceImpl{ + restClient = initRestClient(); + } + private RestClient initRestClient() { SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); factory.setConnectTimeout(100_000); // Temps de connexion en millisecondes @@ -196,7 +201,7 @@ private static JSONArray parseDereferencedXmlWithEnum(byte[] input) throws Parse @Override public String searchColecticaInstanceByUuid(String uuid) throws RmesException { - if (!uuid.matches("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89ABab][0-9a-fA-F]{3}-[0-9a-fA-F]{12}(\\/\\d+)?$")) { + if (!uuid.matches("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89ABab][0-9a-fA-F]{3}-[0-9a-fA-F]{12}(/\\d+)?$")) { throw new RmesException(HttpStatus.BAD_REQUEST, "UUID invalide", uuid + " ne respecte pas le pattern d'un uuid"); } return getWithRestClient(URI.create("ddiset/" + agency + "/" + uuid), MediaType.APPLICATION_XML); @@ -368,13 +373,13 @@ private void cancelTransaction(int transactionId) throws RmesException { } @Override - public String sendDeleteColectica(String uuid, TransactionType transactionType) throws RmesException { + public void sendDeleteColectica(String uuid, TransactionType transactionType) throws RmesException { int transactionId; try{ transactionId = createTransaction(); }catch (RmesException e){ log.error("Error while creating delete transaction : "+e.getMessage(), e); - return "error no TransactionId"; + throw new RmesBadRequestException("error no TransactionId"); } String deleteTransactionRequestBody = bodyForDeleteWithPostRequest(uuid, transactionId); @@ -382,8 +387,6 @@ public String sendDeleteColectica(String uuid, TransactionType transactionType) log.debug("deleteTransactionRequestBody: {}", deleteTransactionRequestBody); // lancement de la requete à destination de l'api colectica postWithRestClient(URI.create("item/_delete"), Optional.of(deleteTransactionRequestBody), MediaType.APPLICATION_JSON); - - return HttpStatus.OK.toString(); } private String bodyForDeleteWithPostRequest(String uuid, int transactionId) throws RmesException { diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ElasticService.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ElasticService.java index f0bbe892..a168fcc9 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ElasticService.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ElasticService.java @@ -4,11 +4,12 @@ import fr.insee.rmes.model.DDIItemType; import lombok.NonNull; import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.*; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatusCode; +import org.springframework.http.MediaType; import org.springframework.stereotype.Component; import org.springframework.web.client.RestClient; -import java.io.IOException; import java.net.URI; import static fr.insee.rmes.transfoxsl.utils.RestClientUtils.readBodySafely; @@ -22,6 +23,10 @@ public record ElasticService(@Value("${fr.insee.rmes.elasticsearch.url}") String private static final String APIKEYHEADER = "apiKey "; private static final String SEARCH = "/_search"; + public ElasticService{ + elasticClient = initRestClient(); + } + private RestClient initRestClient() { return RestClient.builder() .defaultHeader(HttpHeaders.AUTHORIZATION, APIKEYHEADER+elasticApiKey) diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/VarBookExportBuilder.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/VarBookExportBuilder.java index 8fa14951..e7b74712 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/VarBookExportBuilder.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/VarBookExportBuilder.java @@ -5,8 +5,6 @@ import fr.insee.rmes.utils.DocumentBuilders; import fr.insee.rmes.utils.StringUtils; import fr.insee.rmes.utils.XMLUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; import org.w3c.dom.Document; @@ -27,6 +25,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.TreeMap; @Component @@ -34,8 +33,6 @@ public record VarBookExportBuilder() { private static final String REFERENCE = "Reference"; - static final Logger logger = LoggerFactory.getLogger(VarBookExportBuilder.class); - public String getData(String xml) throws RmesException { Document xmlReadyToExport = transformXml(xml); @@ -64,7 +61,7 @@ private Document dereferenceAll(Document xmlInput) throws RmesException { Node root = alls.item(0); Document xmlOutput = null; try { - DocumentBuilder builder = DocumentBuilders.createSaferDocumentBuilder(DocumentBuilderFactory::isIgnoringElementContentWhitespace); + DocumentBuilder builder = DocumentBuilders.createSaferDocumentBuilder(Optional.of(DocumentBuilderFactory::isIgnoringElementContentWhitespace)); xmlOutput = builder.newDocument(); } catch (ParserConfigurationException e) { @@ -192,7 +189,7 @@ private static Document getDocument(String xml) throws RmesException { Document xmlInitial = null; try (InputStream stream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) { - db = DocumentBuilders.createSaferDocumentBuilder(DocumentBuilderFactory::isIgnoringElementContentWhitespace); + db = DocumentBuilders.createSaferDocumentBuilder(Optional.of(DocumentBuilderFactory::isIgnoringElementContentWhitespace)); xmlInitial = db.parse(stream); } catch (ParserConfigurationException | SAXException | IOException e) { throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage(), diff --git a/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java b/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java index 8e99af21..2466e30d 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java @@ -6,18 +6,19 @@ import fr.insee.rmes.exceptions.VtlTransformationException; import fr.insee.rmes.exceptions.XsltTransformationException; import fr.insee.rmes.model.DDIItemType; +import fr.insee.rmes.model.DicoVar; import fr.insee.rmes.transfoxsl.service.XsltTransformationService; import fr.insee.rmes.transfoxsl.service.internal.DDIDerefencer; import fr.insee.rmes.transfoxsl.service.internal.DDITransformerToVtl; import fr.insee.rmes.transfoxsl.utils.MultipartFileUtils; import fr.insee.rmes.utils.FileExtension; +import fr.insee.rmes.utils.HttpUtils; import fr.insee.rmes.utils.export.XDocReport; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ByteArrayResource; import org.springframework.core.io.Resource; import org.springframework.http.HttpHeaders; @@ -39,6 +40,8 @@ import static fr.insee.rmes.tocolecticaapi.controller.ControllerUtils.xmltoResponseEntity; import static fr.insee.rmes.transfoxsl.service.internal.DDIDerefencer.DEREFERENCE_XSL; +import static fr.insee.rmes.transfoxsl.service.internal.DDITransformerToVtl.DDI_2_VTL_XSL; +import static fr.insee.rmes.utils.ExportUtils.FILENAME_MAX_LENGTH; @Controller @RequestMapping("/xsl") @@ -54,7 +57,6 @@ public class TransformationController { private final MultipartFileUtils multipartFileUtils; private final XDocReport xDocReport; - @Autowired public TransformationController(XsltTransformationService xsltTransformationService, MultipartFileUtils multipartFileUtils, XDocReport xDocReport) { this.xsltTransformationService = xsltTransformationService; this.multipartFileUtils = multipartFileUtils; @@ -63,7 +65,7 @@ public TransformationController(XsltTransformationService xsltTransformationServ @Operation(summary = "Déréférencer un objet DDI") @PostMapping(value = "/Derefddi", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public ResponseEntity dereferenceDDI(@RequestParam("file") MultipartFile file) { + public ResponseEntity dereferenceDDI(@RequestParam("file") MultipartFile file) { DDIDerefencer ddiDerefencer = new DDIDerefencer(xsltTransformationService); try { @@ -78,14 +80,14 @@ public ResponseEntity dereferenceDDI(@RequestParam("file") MultipartFile return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(null); } catch (XsltTransformationException e) { log.info("Exception with user input", e.getCause()); - return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(TRANSFORMATION_FAILED_DURING_THE_XSLT_PROCESSING+" : "+e.getXmlErrorMessage()); + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(TRANSFORMATION_FAILED_DURING_THE_XSLT_PROCESSING + " : " + e.getXmlErrorMessage()); } } @Operation(summary = "Générer un fichier texte contenant les règles VTL à partir d'une physicalInstance") @PostMapping(value = "/ddi2vtl", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public ResponseEntity ddi2vtl(@RequestParam("file") MultipartFile file) { + public ResponseEntity ddi2vtl(@RequestParam("file") MultipartFile file) { DDITransformerToVtl ddiTransformerToVtl = new DDITransformerToVtl(xsltTransformationService); @@ -93,16 +95,13 @@ public ResponseEntity ddi2vtl(@RequestParam("file") MultipartFile file InputStream inputStream = multipartFileUtils.convertToInputStream(file); // Première transformation - XML en sortie (on récupère la sortie en tant que chaîne) - List intermediateOutput = xsltTransformationService.transformToXml(inputStream, DEREFERENCE_XSL, false); + var intermediateOutput = xsltTransformationService.transformToXml(inputStream, DEREFERENCE_XSL); // Conversion de la sortie intermédiaire en InputStream pour la deuxième transformation - InputStream intermediateInputStream = new ByteArrayInputStream(String.join("\n", intermediateOutput).getBytes(StandardCharsets.UTF_8)); - - - + InputStream intermediateInputStream = new ByteArrayInputStream(intermediateOutput); // Préparation de la réponse avec le fichier texte - Resource resource = new ByteArrayResource(ddiTransformerToVtl.transform(inputStream)); + Resource resource = new ByteArrayResource(ddiTransformerToVtl.transform(intermediateInputStream)); HttpHeaders headers = new HttpHeaders(); headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=vtl.txt"); @@ -122,15 +121,13 @@ public ResponseEntity ddi2vtl(@RequestParam("file") MultipartFile file @Operation(summary = "Transformer un objet dataRelationShip en JSON") @PostMapping(value = "/dataRelationShiptoJson", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public ResponseEntity ddiDataRelationShiptoJson(@RequestParam("file") MultipartFile file) { + public ResponseEntity ddiDataRelationShiptoJson(@RequestParam("file") MultipartFile file) { try { // Conversion du MultipartFile en InputStream InputStream inputStream = multipartFileUtils.convertToInputStream(file); // Première transformation - XML en sortie - List outputText = xsltTransformationService.transform(inputStream, REL_TO_JSON_XSL, true); - // Joindre les éléments de la liste en une seule chaîne JSON propre - String jsonOutput = String.join("", outputText); + var jsonOutput = xsltTransformationService.transformToRawText(inputStream, REL_TO_JSON_XSL); // Utiliser Jackson pour vérifier la validité du JSON ObjectMapper mapper = new ObjectMapper(); @@ -154,24 +151,19 @@ public ResponseEntity ddiDataRelationShiptoJson(@RequestParam("file") Mu @Operation(summary = "Générer les règles VTL à partir d'une physicalInstance et renvoyer sous forme de texte brut") @PostMapping(value = "/ddi2vtlBrut", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public ResponseEntity ddi2vtlBrut(@RequestParam("file") MultipartFile file) { + public ResponseEntity ddi2vtlBrut(@RequestParam("file") MultipartFile file) { try { // Conversion du MultipartFile en InputStream InputStream inputStream = multipartFileUtils.convertToInputStream(file); // Première transformation - XML en sortie - List intermediateOutput = xsltTransformationService.transform(inputStream, DEREFERENCE_XSL, false); + var intermediateOutput = xsltTransformationService.transformToXml(inputStream, DEREFERENCE_XSL); - // Deuxième transformation - Texte en sortie - InputStream intermediateInputStream = new ByteArrayInputStream(String.join("\n", intermediateOutput).getBytes(StandardCharsets.UTF_8)); - List outputText = xsltTransformationService.transform(intermediateInputStream, DDI_2_VTL_XSL, true); - - // Retourner le texte directement dans la réponse HTTP - String finalOutput = String.join("\n", outputText); + var output = xsltTransformationService.transformToRawText(new ByteArrayInputStream(intermediateOutput), DDI_2_VTL_XSL); return ResponseEntity.ok() .contentType(MediaType.TEXT_PLAIN) - .body(finalOutput); + .body(new String(output)); } catch (IOException e) { throw new VtlTransformationException(FAILED_TO_PROCESS_THE_INPUT_FILE + ".", e); } catch (XsltTransformationException e) { @@ -186,11 +178,11 @@ public ResponseEntity ddi2vtlBrut(@RequestParam("file") MultipartFile fi @PreAuthorize("hasRole('Administrateur_RMESGOPS')") @Operation(summary = "Modify a fragment DDI", description = "Modify a fragment DDI. All field need to be filled with the same data if there are no changes, except for the version number, which takes a plus 1.") public String replaceXmlParameters(@RequestBody String inputXml, - @RequestParam ("Type") DDIItemType type, - @RequestParam ("Label") String label, - @RequestParam ("Version") int version, - @RequestParam ("Name") String name, - @RequestParam ("VersionResponsibility") String idepUtilisateur) { + @RequestParam("Type") DDIItemType type, + @RequestParam("Label") String label, + @RequestParam("Version") int version, + @RequestParam("Name") String name, + @RequestParam("VersionResponsibility") String idepUtilisateur) { return xsltTransformationService.replaceXmlParameters(inputXml, type, label, version, name, idepUtilisateur); } @@ -237,7 +229,7 @@ public ResponseEntity transformFileForComplexCodeList(@RequestParam("fil return ResponseEntity.ok() .headers(headers) - .body(xsltTransformationService.transformFileForComplexList(file, idValue, nomenclatureName, suggesterDescription, version, idepUtilisateur, timbre, principale, secondaire, labelSecondaire); + .body(xsltTransformationService.transformFileForComplexList(file, idValue, nomenclatureName, suggesterDescription, version, idepUtilisateur, timbre, principale, secondaire, labelSecondaire)); } @@ -279,37 +271,15 @@ public ResponseEntity ddiToCodeBook( ) @PreAuthorize("hasRole('Administrateur_RMESGOPS')") @Operation(operationId = "getCodeBookV2", summary = "Produce a codebook from a DDI") - public ResponseEntity getCodeBookV2( - @Parameter(schema = @Schema(type = "string", allowableValues = {"concis", "concis avec expression", "scindable", "non scindable"})) - @RequestParam(value = "dicoVar") String isCodeBook, //InputStream isCodeBook, - @RequestParam(value = "ddiFile") MultipartFile ddiFile // InputStream isDDI, + public ResponseEntity getCodeBookV2( + @RequestParam(value = "dicoVar") DicoVar dicoVar, + @RequestParam(value = "ddiFile") MultipartFile ddiFile ) throws Exception { String ddi = new String(ddiFile.getBytes(), StandardCharsets.UTF_8); - String xslPatternFile = null; - switch (isCodeBook) { - case "concis": - String xmlFileConcis = "/xslTransformerFiles/dicoCodes/dicoConcisPatternContent.xml"; - xslPatternFile = xmlFileConcis; - break; - case "concis avec expression": - String xmlFileConcisAvecExpression = "/xslTransformerFiles/dicoCodes/dicoConcisDescrPatternContent.xml"; - xslPatternFile = xmlFileConcisAvecExpression; - break; - case "scindable": - String xmlFileScindable = "/xslTransformerFiles/dicoCodes/dicoScindablePatternContent.xml"; - xslPatternFile = xmlFileScindable; - break; - case "non scindable": - String xmlFileNonScindable = "/xslTransformerFiles/dicoCodes/dicoNonScindablePatternContent.xml"; - xslPatternFile = xmlFileNonScindable; - break; - default: - log.error("Choix incorrect"); - break; - } - - return xDocReport.getCodeBookExportV2(ddi, xslPatternFile); + return ResponseEntity.ok() + .headers(HttpUtils.generateHttpHeaders("export.odt", FileExtension.ODT_EXTENSION, FILENAME_MAX_LENGTH)) + .body(new ByteArrayResource(xDocReport.getCodeBookExportV2(ddi, dicoVar.getTransformerFilePath()))); } @GetMapping(value = "/check/is-codebook", @@ -318,21 +288,12 @@ public ResponseEntity getCodeBookV2( ) @PreAuthorize("hasRole('Administrateur_RMESGOPS')") @Operation(operationId = "getCodeBookCheck", summary = "Check the DDI before made the codebook export") - public ResponseEntity getCodeBookCheck( - @RequestParam(value = "ddiFile") MultipartFile ddiFile)throws Exception { - - HttpHeaders headers = new HttpHeaders(); - headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + codeBookCheck.getName()); - headers.add("Cache-Control", "no-cache, no-store, must-revalidate"); - headers.add("Pragma", "no-cache"); - headers.add("Expires", "0"); + public ResponseEntity getCodeBookCheck( + @RequestParam(value = "ddiFile") MultipartFile ddiFile) throws Exception { - return xDocReport.getCodeBookCheck(ddiFile.getBytes()); - ResponseEntity.ok() - .headers(headers) - .contentLength(codeBookCheck.length()) - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .body(resourceByte); + return ResponseEntity.ok() + .headers(HttpUtils.generateHttpHeaders("codeBookCheck.odt", FileExtension.ODT_EXTENSION, FILENAME_MAX_LENGTH)) + .body(new ByteArrayResource(xDocReport.getCodeBookCheck(ddiFile.getBytes()))); } } diff --git a/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java b/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java index ec136fb0..d7f3628c 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java @@ -96,9 +96,9 @@ public String transformFile( - public static byte[] transformToJson(Resource resultResource, InputStream xsltFileJson, String idepUtilisateur) throws + public static byte[] transformToJson(Resource resultResource, InputStream xsltFile, String idepUtilisateur) throws IOException, TransformerException { - return transformWithSaxon(xsltFileJson, new StreamSource(resultResource.getInputStream()), + return transformWithSaxon(xsltFile, new StreamSource(resultResource.getInputStream()), Map.of("idepUtilisateur", idepUtilisateur) ); } @@ -237,8 +237,8 @@ private String toXmlString(byte[] result) { return new String(result); } - public byte[] transformToRawText(InputStream stream, String ddi2VtlXsl) throws IOException { - return transform(stream, ddi2VtlXsl, SerializerConfigurer.forRawText(processor)); + public byte[] transformToRawText(InputStream stream, String xslFilename) throws IOException { + return transform(stream, xslFilename, SerializerConfigurer.forRawText(processor)); } diff --git a/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDITransformerToVtl.java b/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDITransformerToVtl.java index 457844d4..30687766 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDITransformerToVtl.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDITransformerToVtl.java @@ -2,9 +2,9 @@ import fr.insee.rmes.transfoxsl.service.XsltTransformationService; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.List; public record DDITransformerToVtl(XsltTransformationService xsltTransformationService, DDIDerefencer ddiDerefencer) { @@ -18,7 +18,7 @@ public byte[] transform(InputStream inputStream) throws IOException { // Deuxième transformation - Texte en sortie (on récupère directement une liste de lignes de texte) - return xsltTransformationService.transformToRawText(ddiDerefencer.intermediateDereference(inputStream), DDI_2_VTL_XSL); + return xsltTransformationService.transformToRawText(new ByteArrayInputStream(ddiDerefencer.intermediateDereference(inputStream)), DDI_2_VTL_XSL); } } diff --git a/src/main/java/fr/insee/rmes/utils/DocumentBuilderUtils.java b/src/main/java/fr/insee/rmes/utils/DocumentBuilderUtils.java index 11715943..f447033f 100644 --- a/src/main/java/fr/insee/rmes/utils/DocumentBuilderUtils.java +++ b/src/main/java/fr/insee/rmes/utils/DocumentBuilderUtils.java @@ -2,44 +2,36 @@ import org.springframework.stereotype.Service; import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; import org.xml.sax.InputSource; +import org.xml.sax.SAXException; import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; import java.io.StringReader; +import java.util.Optional; @Service public class DocumentBuilderUtils { + private DocumentBuilderUtils(){} - public static Document getDocument(String fragment) throws Exception { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - - // Désactiver l'accès aux entités externes pour des raisons de sécurité (prévention des attaques XXE) - factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - factory.setFeature("http://xml.org/sax/features/external-general-entities", false); - factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - - DocumentBuilder builder = factory.newDocumentBuilder(); + public static Document getDocument(String fragment) throws ParserConfigurationException, IOException, SAXException { + return getDocument(new ByteArrayInputStream(fragment.getBytes())); + } - if (null == fragment || fragment.isEmpty()) { + public static Document getDocument(InputStream fragment) throws ParserConfigurationException, IOException, SAXException { + DocumentBuilder builder = DocumentBuilders.createSaferDocumentBuilder(Optional.empty()); + if (null == fragment) { return builder.newDocument(); } - - InputSource ddiSource = new InputSource(new StringReader(fragment)); + InputSource ddiSource = new InputSource(fragment); return builder.parse(ddiSource); } - - public static Node getNode(String fragment, Document doc) throws Exception { - Element node = DocumentBuilderUtils.getDocument(fragment).getDocumentElement(); - Node newNode = node.cloneNode(true); - // Transfer ownership of the new node into the destination document - doc.adoptNode(newNode); - return newNode; - } + } diff --git a/src/main/java/fr/insee/rmes/utils/DocumentBuilders.java b/src/main/java/fr/insee/rmes/utils/DocumentBuilders.java index 024c9987..05a03852 100644 --- a/src/main/java/fr/insee/rmes/utils/DocumentBuilders.java +++ b/src/main/java/fr/insee/rmes/utils/DocumentBuilders.java @@ -3,6 +3,7 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import java.util.Optional; public interface DocumentBuilders { @@ -11,7 +12,7 @@ interface ParserConfigurer void configureParser(DocumentBuilderFactory factory) throws ParserConfigurationException; } - static DocumentBuilder createSaferDocumentBuilder(ParserConfigurer parserConfigurer) throws ParserConfigurationException + static DocumentBuilder createSaferDocumentBuilder(Optional parserConfigurer) throws ParserConfigurationException { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); @@ -21,8 +22,9 @@ static DocumentBuilder createSaferDocumentBuilder(ParserConfigurer parserConfigu factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); factory.setXIncludeAware(false); factory.setExpandEntityReferences(false); - parserConfigurer.configureParser(factory); - + if (parserConfigurer.isPresent()){ + parserConfigurer.get().configureParser(factory); + } return factory.newDocumentBuilder(); } } \ No newline at end of file diff --git a/src/main/java/fr/insee/rmes/utils/ExportUtils.java b/src/main/java/fr/insee/rmes/utils/ExportUtils.java index 5a9b73d3..1c2a4f52 100644 --- a/src/main/java/fr/insee/rmes/utils/ExportUtils.java +++ b/src/main/java/fr/insee/rmes/utils/ExportUtils.java @@ -1,107 +1,67 @@ package fr.insee.rmes.utils; import fr.insee.rmes.exceptions.RmesException; -import fr.insee.rmes.tocolecticaapi.service.Constants; -import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.ByteArrayResource; -import org.springframework.core.io.Resource; -import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; +import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; -import java.io.PrintStream; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Map; @Component public class ExportUtils { private static final Logger logger = LoggerFactory.getLogger(ExportUtils.class); - private static final String CAN_T_GENERATE_CODEBOOK = "Can't generate codebook"; - private static final String NULL_STREAM = "Stream is null"; - final int maxLength; + public static final int FILENAME_MAX_LENGTH = 50; - public ExportUtils(@Value("${fr.insee.rmes.ddias.filenames.maxlength}") int maxLength) { - this.maxLength = maxLength; + public byte[] exportAsODT(Map xmlContent, String xslFile, String xmlPattern, String zip) throws RmesException { + return exportAsBytes(xmlContent, xslFile, xmlPattern, zip); } - public ResponseEntity exportAsODT(Path fileName, Map xmlContent, String xslFile, String xmlPattern, String zip, String objectType) throws RmesException { - return exportAsFileByExtension(fileName, xmlContent, xslFile, xmlPattern, zip, objectType, FileExtension.ODT_EXTENSION); + public byte[] exportAsODS(Map xmlContent, String xslFile, String xmlPattern, String zip) throws RmesException { + return exportAsBytes(xmlContent, xslFile, xmlPattern, zip); } - public ResponseEntity exportAsODS(Path fileName, Map xmlContent, String xslFile, String xmlPattern, String zip, String objectType) throws RmesException { - return exportAsFileByExtension(fileName, xmlContent, xslFile, xmlPattern, zip, objectType, FileExtension.ODS_EXTENSION); - } - - private ResponseEntity exportAsFileByExtension(Path fileName, Map xmlContent, String xslFile, String xmlPattern, String zip, String objectType, FileExtension extension) throws RmesException { - logger.debug("Begin To export {} as Response", objectType); - + private byte[] exportAsBytes(Map xmlParameters, String xslFile, String xmlPattern, String zip) throws RmesException { + try ( + InputStream xslFileIS = getClass().getResourceAsStream(xslFile); + InputStream odtFileIS = getClass().getResourceAsStream(xmlPattern); + InputStream zipToCompleteIS = getClass().getResourceAsStream(zip)) { - InputStream input = exportAsInputStream(fileName, xmlContent, xslFile, xmlPattern, zip, objectType); - if (input == null) - throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, CAN_T_GENERATE_CODEBOOK, NULL_STREAM); - - ByteArrayResource resource; - try { - resource = new ByteArrayResource(IOUtils.toByteArray(input)); - input.close(); - } catch (IOException e) { - logger.error("Failed to getBytes of resource"); - throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage(), "IOException"); + Path tempDir = Files.createTempDirectory("forExport"); + var contentXml = XsltUtils.transformerInputStreamWithXslWithParameters(odtFileIS, xslFileIS, transformer -> addParameters(xmlParameters, tempDir, transformer)); + // create odt + return XsltUtils.createOdtFromXml(contentXml, zipToCompleteIS.readAllBytes()); + } catch (IOException | TransformerException e) { + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage(), e.getClass().getSimpleName()); } - logger.debug("End To export {} as Response", objectType); - - HttpHeaders responseHeaders = HttpUtils.generateHttpHeaders(fileName.toString(), extension, this.maxLength); - - return ResponseEntity.ok() - .headers(responseHeaders) - .contentLength(resource.contentLength()) - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .body(resource); } - public InputStream exportAsInputStream(Path fileName, Map xmlContent, String xslFile, String xmlPattern, String zip, String objectType) throws RmesException { - logger.debug("Begin To export {} as InputStream", objectType); + private static void addParameters(Map xmlContent, Path tempDir, Transformer xsltTransformer) { + xmlContent.forEach((paramName, xmlData) -> addParameter(xsltTransformer, paramName, xmlData, tempDir)); + } + private static void addParameter(Transformer xsltTransformer, String paramName, byte[] paramData, Path tempDir) { try { - Path outputPath = Files.createTempFile(Constants.OUTPUT, FileExtension.XML_EXTENSION.extension()); - - try ( - InputStream xslFileIS = getClass().getResourceAsStream(xslFile); - InputStream odtFileIS = getClass().getResourceAsStream(xmlPattern); - InputStream zipToCompleteIS = getClass().getResourceAsStream(zip); - OutputStream osOutputFile = Files.newOutputStream(outputPath); - PrintStream printStream = new PrintStream(osOutputFile)) { - - outputPath.toFile().deleteOnExit(); - - - Path tempDir = Files.createTempDirectory("forExport"); - Path finalPath = Paths.get(tempDir.toString()).resolve(fileName); - - // transform - XsltUtils.xsltTransform(xmlContent, odtFileIS, xslFileIS, printStream, tempDir); - // create odt - XsltUtils.createOdtFromXml(outputPath, finalPath, zipToCompleteIS, tempDir); - - logger.debug("End To export {} as InputStream", objectType); - - return Files.newInputStream(finalPath); - } - } catch (IOException | TransformerException e) { - throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage(), e.getClass().getSimpleName()); + Path tempFile = Files.createTempFile(tempDir, paramName, FileExtension.XML_EXTENSION.extension()); + Files.write(tempFile, paramData); + tempFile.toFile().deleteOnExit(); + + // Convert to file URI + String fileUri = tempFile.toUri().toString(); + logger.debug("Setting XSLT parameter '{}' to '{}'", paramName, fileUri); + xsltTransformer.setParameter(paramName, fileUri); + } catch (IOException e) { + logger.error("Can't create temp files for XSLT Transformer", e); + throw new UncheckedIOException(e); } } diff --git a/src/main/java/fr/insee/rmes/utils/HttpUtils.java b/src/main/java/fr/insee/rmes/utils/HttpUtils.java index 72b49d85..2783eb25 100644 --- a/src/main/java/fr/insee/rmes/utils/HttpUtils.java +++ b/src/main/java/fr/insee/rmes/utils/HttpUtils.java @@ -28,7 +28,6 @@ public static HttpHeaders generateHttpHeaders(String fileName, FileExtension ext HttpHeaders responseHeaders = new HttpHeaders(); - responseHeaders.setAccept(List.of(MediaType.ALL)); responseHeaders.setContentDisposition(content); responseHeaders.setContentType(contentType); responseHeaders.setAccessControlExposeHeaders(allowHeaders); diff --git a/src/main/java/fr/insee/rmes/utils/XsltUtils.java b/src/main/java/fr/insee/rmes/utils/XsltUtils.java index 067914fb..cf96f076 100644 --- a/src/main/java/fr/insee/rmes/utils/XsltUtils.java +++ b/src/main/java/fr/insee/rmes/utils/XsltUtils.java @@ -15,7 +15,13 @@ import java.io.*; import java.nio.charset.StandardCharsets; import java.nio.file.*; +import java.util.Arrays; import java.util.Map; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; public class XsltUtils { @@ -25,69 +31,38 @@ private XsltUtils() { throw new IllegalStateException("Utility class"); } - - public static void xsltTransform(Map xmlContent, InputStream odtFileIS, InputStream xslFileIS, - PrintStream printStream, Path tempDir) throws TransformerException { - // prepare transformer - StreamSource xsrc = new StreamSource(xslFileIS); - Transformer xsltTransformer = XMLUtils.newTransformer(xsrc); - - // Pass parameters in a file to the transformer - xmlContent.forEach((paramName, xmlData) -> { - try { - addParameter(xsltTransformer, paramName, xmlData, tempDir); - } catch (RmesException e) { - logger.error(e.getMessageAndDetails()); - } - }); - - // transformation - xsltTransformer.transform(new StreamSource(odtFileIS), new StreamResult(printStream)); + static byte[] createOdtFromXml(byte[] contentXml, byte[] zipBase) throws IOException { + var byteArrayOutputStream = new ByteArrayOutputStream(zipBase.length); + byteArrayOutputStream.writeBytes(zipBase); + var zip = new ZipOutputStream(byteArrayOutputStream); + zip.putNextEntry(new ZipEntry("content.xml")); + zip.write(contentXml); + zip.close(); + return byteArrayOutputStream.toByteArray(); } + public static byte[] transformerInputStreamWithXsl(byte[] input,InputStream xslStyleSheet) throws TransformerException { + return doTransform(new StreamSource(xslStyleSheet),new StreamSource(new ByteArrayInputStream(input)), Optional.empty() ); + } - private static void addParameter(Transformer xsltTransformer, String paramName, String paramData, Path tempDir) throws RmesException { - CopyOption[] options = { StandardCopyOption.REPLACE_EXISTING }; - try { - Path tempFile = Files.createTempFile(tempDir, paramName, FileExtension.XML_EXTENSION.extension()); - InputStream is = IOUtils.toInputStream(paramData, StandardCharsets.UTF_8); - Files.copy(is, tempFile, options); - - // Convert to file URI - String fileUri = tempFile.toUri().toString(); - logger.debug("Setting XSLT parameter '{}' to '{}'", paramName, fileUri); - xsltTransformer.setParameter(paramName, fileUri); - } catch (IOException e) { - throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage(), "IOException - Can't create temp files for XSLT Transformer"); - } - } - - public static void createOdtFromXml(Path outputPath, Path finalPath, InputStream zipToCompleteIS, Path tempDir) - throws IOException { - Path contentPath = Paths.get(tempDir + "/content.xml"); - Files.copy(outputPath, contentPath, StandardCopyOption.REPLACE_EXISTING); - Path zipPath = Paths.get(tempDir + "/export.zip"); - Files.copy(zipToCompleteIS, zipPath, StandardCopyOption.REPLACE_EXISTING); - FilesUtils.addFileToZipFolder(contentPath, zipPath); - Files.copy(zipPath, finalPath, StandardCopyOption.REPLACE_EXISTING); + public static byte[] transformerStringWithXsl(String ddi, InputStream xslStyleSheet) throws TransformerException { + return doTransform(new StreamSource(xslStyleSheet), new StreamSource(new StringReader(ddi)), Optional.empty()); } - public static byte[] transformerInputStreamWithXsl(byte[] input,InputStream xslCheckReference) throws TransformerException { - return doTransform(new StreamSource(xslCheckReference),new StreamSource(new ByteArrayInputStream(input)) ); - } - - public static byte[] transformerStringWithXsl(String ddi, InputStream xslRemoveNameSpaces) throws TransformerException { - return doTransform(new StreamSource(xslRemoveNameSpaces), new StreamSource(new StringReader(ddi))); + public static byte[] transformerInputStreamWithXslWithParameters(InputStream input, InputStream xslStyleSheet, Consumer parameterSetter) throws TransformerException { + return doTransform(new StreamSource(xslStyleSheet),new StreamSource(input), Optional.of(parameterSetter )); } public static byte[] transformerFileWithXsl(File input, InputStream xslCheckReference) throws TransformerException { - return doTransform(new StreamSource(xslCheckReference), new StreamSource(input)); + return doTransform(new StreamSource(xslCheckReference), new StreamSource(input), Optional.empty()); } - private static byte[] doTransform(Source stylesheetSource, Source inputSource) throws TransformerException { + private static byte[] doTransform(Source stylesheetSource, Source inputSource, Optional> parameterSetter) throws TransformerException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); Result outputResult = new StreamResult(baos); - XMLUtils.newTransformer(stylesheetSource).transform(inputSource, outputResult); + Transformer transformer = XMLUtils.newTransformer(stylesheetSource); + parameterSetter.ifPresent(p -> p.accept(transformer)); + transformer.transform(inputSource, outputResult); return baos.toByteArray(); } } diff --git a/src/main/java/fr/insee/rmes/utils/export/XDocReport.java b/src/main/java/fr/insee/rmes/utils/export/XDocReport.java index 7c9d3f85..dbf18db2 100644 --- a/src/main/java/fr/insee/rmes/utils/export/XDocReport.java +++ b/src/main/java/fr/insee/rmes/utils/export/XDocReport.java @@ -2,6 +2,8 @@ import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.tocolecticaapi.service.VarBookExportBuilder; +import fr.insee.rmes.utils.DocumentBuilderUtils; +import fr.insee.rmes.utils.ExportUtils; import fr.insee.rmes.utils.XsltUtils; import fr.opensagres.xdocreport.core.XDocReportException; import fr.opensagres.xdocreport.document.IXDocReport; @@ -11,91 +13,62 @@ import freemarker.ext.dom.NodeModel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.core.io.ByteArrayResource; -import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; import org.w3c.dom.Document; import org.xml.sax.InputSource; import org.xml.sax.SAXException; -import javax.xml.XMLConstants; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; import java.io.*; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; - -import static fr.insee.rmes.transfoxsl.service.XsltTransformationService.*; +import java.util.Map; @Component -public record XDocReport(VarBookExportBuilder varBookExport) { +public record XDocReport(VarBookExportBuilder varBookExport, ExportUtils exportUtils) { static final Logger logger = LoggerFactory.getLogger(XDocReport.class); - public byte[] getCodeBookExportV2(String ddi, String xslPatternFile) { - - InputStream xslRemoveNameSpaces = getClass().getResourceAsStream("/xslTransformerFiles/remove-namespaces.xsl"); - InputStream xslCheckReference = getClass().getResourceAsStream("/xslTransformerFiles/check-references.xsl"); - String dicoCode = "/xslTransformerFiles/dico-codes.xsl"; - String zipRmes = "/xslTransformerFiles/dicoCodes/toZipForDicoCodes.zip"; - - byte[] ddiRemoveNameSpace = XsltUtils.transformerStringWithXsl(ddi, xslRemoveNameSpaces); - byte[] control = XsltUtils.transformerInputStreamWithXsl(ddiRemoveNameSpace, xslCheckReference); + public static final byte[] XSL_REMOVE_NAMESPACE = loadFileAsBytes("/xslTransformerFiles/remove-namespaces.xsl"); + public static final byte[] XSL_CHECK_REFERENCE = loadFileAsBytes("/xslTransformerFiles/check-references.xsl"); - // Créer un DocumentBuilderFactory sécurisé - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + private static byte[] loadFileAsBytes(String resourcePath) { + try (InputStream resourceAsStream = XDocReport.class.getResourceAsStream(resourcePath)){ + return resourceAsStream.readAllBytes(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } - // Désactiver l'accès aux entités externes pour des raisons de sécurité (prévention des attaques XXE) - dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - dbf.setFeature(DISALLOW_DOCTYPE_DECL1, true); - dbf.setFeature(EXTERNAL_GENERAL_ENTITIES, false); - dbf.setFeature(EXTERNAL_PARAMETER_ENTITIES, false); + public byte[] getCodeBookExportV2(String ddi, String xslPatternFile) throws RmesException, TransformerException, ParserConfigurationException, IOException, SAXException { - DocumentBuilder db = dbf.newDocumentBuilder(); + String dicoCode = "/xslTransformerFiles/dico-codes.xsl"; + String zipRmes = "/xslTransformerFiles/dicoCodes/toZipForDicoCodes.zip"; + byte[] ddiRemoveNameSpace = XsltUtils.transformerStringWithXsl(ddi, new ByteArrayInputStream(XSL_REMOVE_NAMESPACE)); + byte[] control = XsltUtils.transformerInputStreamWithXsl(ddiRemoveNameSpace, new ByteArrayInputStream(XSL_CHECK_REFERENCE)); - Document doc = db.parse(new ByteArrayInputStream(control)); + Document doc = DocumentBuilderUtils.getDocument(new ByteArrayInputStream(control)); String checkResult = doc.getDocumentElement().getNodeName(); if (!"OK".equals(checkResult)) { - HttpHeaders headers = new HttpHeaders(); - headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + control.getName()); - headers.add("Cache-Control", "no-cache, no-store, must-revalidate"); - headers.add("Pragma", "no-cache"); - headers.add("Expires", "0"); - - ByteArrayResource resourceByte = new ByteArrayResource(Files.readAllBytes(((InputStream) new ByteArrayInputStream(XsltUtils.transformerInputStreamWithXsl(XsltUtils.transformerStringWithXsl(ddi, xslRemoveNameSpaces), xslCheckReference))).toPath())); - - return ResponseEntity.ok() - .headers(headers) - .contentLength(control.length()) - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .body(resourceByte); + return XsltUtils.transformerInputStreamWithXsl(XsltUtils.transformerStringWithXsl(ddi, new ByteArrayInputStream(XSL_REMOVE_NAMESPACE)), new ByteArrayInputStream(XSL_CHECK_REFERENCE)); } - - HashMap contentXML = new HashMap<>(); - contentXML.put("ddi-file", Files.readString(ddiRemoveNameSpaces.toPath())); - - return exportUtils.exportAsODT(Path.of("export.odt"), contentXML, dicoCode, xslPatternFile, zipRmes, "dicoVariable"); + logger.debug("Begin To export dicoVariable"); + var odt=exportUtils.exportAsODT(Map.of("ddi-file", ddiRemoveNameSpace), dicoCode, xslPatternFile, zipRmes); + logger.debug("End To export dicoVariable"); + return odt; } public byte[] getCodeBookCheck(byte[] ddiContent) throws Exception { - - InputStream xslRemoveNameSpaces = getClass().getResourceAsStream("/xslTransformerFiles/remove-namespaces.xsl"); - File ddiRemoveNameSpaces = File.createTempFile("ddiRemoveNameSpaces", ".xml"); - ddiRemoveNameSpaces.deleteOnExit(); - XsltUtils.transformerInputStreamWithXsl(ddiContent, xslRemoveNameSpaces, ddiRemoveNameSpaces); - InputStream xslCodeBookCheck = getClass().getResourceAsStream("/xslTransformerFiles/dico-codes-test-ddi-content.xsl"); - return XsltUtils.transformerFileWithXsl(ddiRemoveNameSpaces, xslCodeBookCheck); + return XsltUtils.transformerInputStreamWithXsl( + XsltUtils.transformerInputStreamWithXsl(ddiContent, new ByteArrayInputStream(XSL_REMOVE_NAMESPACE)), + xslCodeBookCheck + ); } public byte[] exportVariableBookInOdt(String xml, byte[] odtTemplate) throws RmesException { diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 4861ed60..69c1c575 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -66,5 +66,4 @@ fr.insee.rmes.api.host = localhost:8080 fr.insee.rmes.api.name= /ddi-access-services/api fr.insee.rmes.api.scheme= http spring.servlet.multipart.enabled=true -spring.servlet.multipart.max-request-size=50MB -fr.insee.rmes.ddias.filenames.maxlength=50 \ No newline at end of file +spring.servlet.multipart.max-request-size=50MB \ No newline at end of file diff --git a/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java b/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java index bfc64503..a166f22b 100644 --- a/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java +++ b/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java @@ -1,6 +1,6 @@ package fr.insee.rmes.tocolecticaapi.controller; -import fr.insee.rmes.tocolecticaapi.fragments.DdiFragment; +import fr.insee.rmes.tocolecticaapi.fragments.DdiFragmentService; import fr.insee.rmes.tocolecticaapi.service.ColecticaService; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -50,8 +50,8 @@ private String read(String fileName) throws Exception { @TestConfiguration static class ConfigurationForTest{ @Bean - DdiFragment ddiFragment(){ - return new DdiFragment() { + DdiFragmentService ddiFragment(){ + return new DdiFragmentService() { @Override public String extractDataRelationship(String uuid) { return ""; From 5f1c17aa2171a0a75efc7c8c485d258f86a762b2 Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Tue, 14 Jan 2025 15:49:49 +0100 Subject: [PATCH 09/25] feat (get ddi relationship to json) #93 --- .../java/fr/insee/rmes/model/DicoVar.java | 19 ++++ .../fragments/DdiFragmentServiceImpl.java | 28 +++++ .../ddiDatarelationship2Json.xsl | 102 ++++++++++++++++++ 3 files changed, 149 insertions(+) create mode 100644 src/main/java/fr/insee/rmes/model/DicoVar.java create mode 100644 src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java create mode 100644 src/main/resources/xslTransformerFiles/ddiDatarelationship2Json.xsl diff --git a/src/main/java/fr/insee/rmes/model/DicoVar.java b/src/main/java/fr/insee/rmes/model/DicoVar.java new file mode 100644 index 00000000..09deed5d --- /dev/null +++ b/src/main/java/fr/insee/rmes/model/DicoVar.java @@ -0,0 +1,19 @@ +package fr.insee.rmes.model; + +import lombok.Getter; + +@Getter +public enum DicoVar { + + CONCIS("/xslTransformerFiles/dicoCodes/dicoConcisPatternContent.xml"), + CONCIS_AVEC_EXPRESSION("/xslTransformerFiles/dicoCodes/dicoConcisDescrPatternContent.xml"), + SCINDABLE("/xslTransformerFiles/dicoCodes/dicoScindablePatternContent.xml"), + NON_SCINDABLE("/xslTransformerFiles/dicoCodes/dicoNonScindablePatternContent.xml"); + + private final String transformerFilePath; + + DicoVar(String transformerFilePath) { + this.transformerFilePath = transformerFilePath; + } + +} diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java b/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java new file mode 100644 index 00000000..73269b77 --- /dev/null +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java @@ -0,0 +1,28 @@ +package fr.insee.rmes.tocolecticaapi.fragments; + +import fr.insee.rmes.exceptions.RmesException; +import fr.insee.rmes.tocolecticaapi.service.ColecticaService; +import fr.insee.rmes.transfoxsl.service.XsltTransformationService; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +@Service +public record DdiFragmentServiceImpl(XsltTransformationService xsltTransformationService, ColecticaService colecticaService) implements DdiFragmentService { + + static final String ddiRelationshipToJsonXsl = "/xslTransformerFiles/ddiDatarelationship2Json.xsl"; + + @Override + public String extractDataRelationship(String uuid) throws RmesException { + try { + return xsltTransformationService.transformToXmlString( + new ByteArrayInputStream(colecticaService.searchColecticaInstanceByUuid(uuid).getBytes()), + ddiRelationshipToJsonXsl + ); + } catch (IOException e) { + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Error while processing DDI to json", e.getMessage()); + } + } +} diff --git a/src/main/resources/xslTransformerFiles/ddiDatarelationship2Json.xsl b/src/main/resources/xslTransformerFiles/ddiDatarelationship2Json.xsl new file mode 100644 index 00000000..d370f13c --- /dev/null +++ b/src/main/resources/xslTransformerFiles/ddiDatarelationship2Json.xsl @@ -0,0 +1,102 @@ + + + + + + + + + + + + + + [ + + + + + + + + , + + + { "id": " + + ", "nom": " + + ", "label": [ { "contenu": " + + ", "langue": "fr" }, { "contenu": "", "langue": "en" } ], "variables": [ + + + + + + + + + + + , + + { "id": " + + ", "nom": " + + ", "label": [ { "contenu": " + + ", "langue": "fr" }, { "contenu": "", "langue": "en" } ], "ordre": " + + ", "representation": + + + + "codes" + + + "numerique" + + + "texte" + + + "date" + + + "null" + + + + + + , "controles": { "codes": [ + + + + , + + { "code": " + + ", "label": [ { "langue": "fr", "contenu": " + + + " }, { "langue": "en", "contenu": "" } ] } + + ] } + + + } + + ] } + + ] + + + + From 5ffcf38201ac4855a814631b9d8d90d18eb53f0a Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Wed, 15 Jan 2025 16:31:15 +0100 Subject: [PATCH 10/25] refactor (mutulualize, separate concerns, remove useless, remove redondant, remove elastic search) : compile and TESTS GREEN --- pom.xml | 82 +- .../java/fr/insee/rmes/RmesApplication.java | 3 + .../config/InseeSecurityTokenProperties.java | 18 +- .../fr/insee/rmes/config/SecurityConfig.java | 106 +- .../config/keycloak/KeycloakServices.java | 8 +- .../exceptions/GenericExceptionMapper.java | 28 - .../rmes/exceptions/RMeSEnvironment.java | 21 +- .../rmes/exceptions/RMeSExceptionMapper.java | 17 - .../rmes/exceptions/RmesExceptionIO.java | 5 +- .../VtlTransformationException.java | 11 - .../XsltTransformationException.java | 2 +- .../controller/MetadataController.java | 21 +- .../tocolecticaapi/controller/DeleteItem.java | 7 +- .../tocolecticaapi/controller/GetItem.java | 6 +- .../tocolecticaapi/controller/PostItem.java | 3 +- .../controller/RmesExceptionHandler.java | 2 +- .../fragments/DdiFragmentServiceImpl.java | 7 +- .../service/ColecticaService.java | 10 +- .../service/ColecticaServiceImpl.java | 43 +- .../service/ElasticService.java | 16 +- .../controller/TransformationController.java | 37 +- .../service/XsltTransformationService.java | 12 +- .../service/internal/DDIDerefencer.java | 7 +- .../service/internal/DDITransformerToVtl.java | 6 +- .../rmes/utils/DocumentBuilderUtils.java | 8 - .../java/fr/insee/rmes/utils/FilesUtils.java | 26 - .../java/fr/insee/rmes/utils/XsltUtils.java | 15 +- .../utils/jersey/ObjectMapperProvider.java | 33 - src/main/resources/application.properties | 32 +- .../controller/GetItemTest.java | 35 +- .../TransformationControllerTest.java | 53 +- .../XsltTransformationServiceTest.java | 85 +- .../utils/MultipartFileUtilsTest.java | 10 +- .../rmes/utils/DocumentBuilderUtilsTest.java | 21 + .../service/ColecticaServiceImplTest.java | 186 +- ...-f0bb-47df-b3d2-42ff7f8f5874_expected.json | 46 +- .../getItemTest/physicalInstance.xml | 2352 +++++++++++++++++ 37 files changed, 2728 insertions(+), 652 deletions(-) delete mode 100644 src/main/java/fr/insee/rmes/exceptions/GenericExceptionMapper.java delete mode 100644 src/main/java/fr/insee/rmes/exceptions/RMeSExceptionMapper.java delete mode 100644 src/main/java/fr/insee/rmes/exceptions/VtlTransformationException.java delete mode 100644 src/main/java/fr/insee/rmes/utils/jersey/ObjectMapperProvider.java create mode 100644 src/test/java/fr/insee/rmes/utils/DocumentBuilderUtilsTest.java create mode 100644 src/test/resources/getItemTest/physicalInstance.xml diff --git a/pom.xml b/pom.xml index 04c5d7e0..3d6d28df 100644 --- a/pom.xml +++ b/pom.xml @@ -6,23 +6,22 @@ org.springframework.boot spring-boot-starter-parent - 3.4.0 + 3.4.1 fr.insee.rmes - rmes - 2.0.5 - rmes - DDI-Access Services + ddi-access-services + 2.1.0 + DDI Access Services + Rest API to access to Colectica repository services 21 UTF-8 - 2.7.0 - 2.0.6 - 12.4 - 1.1.1 - 2.17.0 - 1.6.14 + 2.8.3 + 2.1.0 + 12.5 + 4.4.0 + 20250107 @@ -30,41 +29,26 @@ com.auth0 java-jwt - 4.4.0 - - - org.apache.commons - commons-lang3 + ${java-jwt.version} + com.fasterxml.jackson.dataformat jackson-dataformat-xml - - - org.freemarker - freemarker - + fr.opensagres.xdocreport fr.opensagres.xdocreport.document.odt ${xdocreport.version} + fr.opensagres.xdocreport fr.opensagres.xdocreport.template.freemarker ${xdocreport.version} - - - commons-io - commons-io - ${commons-io.version} - - - org.springframework.boot - spring-boot-starter-jersey - + org.springframework.boot spring-boot-starter-security @@ -91,47 +75,26 @@ org.json json - 20240303 + ${json.version} org.springdoc springdoc-openapi-starter-webmvc-ui ${springdoc.version} - - org.springframework.security - spring-security-config - + org.springframework.boot spring-boot-starter-oauth2-resource-server - - com.googlecode.json-simple - json-simple - ${json-simple.version} - + net.sf.saxon Saxon-HE ${saxon.version} - - - xml-apis - xml-apis - - - - - jakarta.json - jakarta.json-api - - - io.swagger - swagger-annotations - ${io.swagger.version} + @@ -150,11 +113,4 @@ - - - maven_central - Maven Central - https://repo.maven.apache.org/maven2/ - - diff --git a/src/main/java/fr/insee/rmes/RmesApplication.java b/src/main/java/fr/insee/rmes/RmesApplication.java index 0a830597..6b3e5f99 100644 --- a/src/main/java/fr/insee/rmes/RmesApplication.java +++ b/src/main/java/fr/insee/rmes/RmesApplication.java @@ -1,10 +1,13 @@ package fr.insee.rmes; +import fr.insee.rmes.config.InseeSecurityTokenProperties; import fr.insee.rmes.config.PropertiesLogger; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.boot.context.properties.EnableConfigurationProperties; @SpringBootApplication +@EnableConfigurationProperties(InseeSecurityTokenProperties.class) public class RmesApplication { public static void main(String[] args) { diff --git a/src/main/java/fr/insee/rmes/config/InseeSecurityTokenProperties.java b/src/main/java/fr/insee/rmes/config/InseeSecurityTokenProperties.java index 4f37b219..908d7fb8 100644 --- a/src/main/java/fr/insee/rmes/config/InseeSecurityTokenProperties.java +++ b/src/main/java/fr/insee/rmes/config/InseeSecurityTokenProperties.java @@ -1,21 +1,13 @@ package fr.insee.rmes.config; -import lombok.Data; -import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication; import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; -@Configuration @ConfigurationProperties( prefix = "fr.insee.rmes.security.token" ) -@ConditionalOnWebApplication -@Data -public class InseeSecurityTokenProperties { - - //Chemin pour récupérer la liste des rôles dans le jwt (token) - private String oidcClaimRole; - //Chemin pour récupérer le username dans le jwt (token) - private String oidcClaimUsername; - +public record InseeSecurityTokenProperties( + //Chemin pour récupérer la liste des rôles dans le jwt (token) + String oidcClaimRole, + //Chemin pour récupérer le username dans le jwt (token) + String oidcClaimUsername) { } diff --git a/src/main/java/fr/insee/rmes/config/SecurityConfig.java b/src/main/java/fr/insee/rmes/config/SecurityConfig.java index d817618f..9c45c446 100644 --- a/src/main/java/fr/insee/rmes/config/SecurityConfig.java +++ b/src/main/java/fr/insee/rmes/config/SecurityConfig.java @@ -1,55 +1,58 @@ package fr.insee.rmes.config; import lombok.extern.slf4j.Slf4j; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; import org.springframework.core.convert.converter.Converter; -import org.springframework.core.env.Environment; +import org.springframework.http.HttpMethod; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configurers.AbstractAuthenticationFilterConfigurer; import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer; +import org.springframework.security.config.annotation.web.configurers.HeadersConfigurer; import org.springframework.security.config.http.SessionCreationPolicy; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.oauth2.jwt.Jwt; import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationConverter; import org.springframework.security.web.SecurityFilterChain; import org.springframework.security.web.util.matcher.AntPathRequestMatcher; -import org.springframework.http.HttpMethod; -import java.util.*; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; @Configuration @EnableWebSecurity @Slf4j public class SecurityConfig { - @Autowired - private Environment environment; - @Autowired - InseeSecurityTokenProperties inseeSecurityTokenProperties; - @Value("${fr.insee.rmes.role.administrateur}") - private String administrateurRole; - - @Value("${fr.insee.rmes.role.gestionnaire.colectica}") - private String gestionnaireColectica; - @Value("${fr.insee.rmes.metadata.cors.allowedOrigin}") - private Optional allowedOrigin; - @Value("#{'${fr.insee.rmes.security.whitelist-matchers}'.split(',')}") - private String[] whiteList; + private final InseeSecurityTokenProperties inseeSecurityTokenProperties; + + private final String administrateurRole; + + private final String gestionnaireColectica; + + private final String[] whiteList; + private static final String ROLE_PREFIX = "ROLE_"; - public static final String PROFILES_ACTIVE="spring.profiles.active" ; - final static Logger logger = LogManager.getLogger(SecurityConfig.class); + + public SecurityConfig(InseeSecurityTokenProperties inseeSecurityTokenProperties, @Value("${fr.insee.rmes.role.administrateur}") String administrateurRole, + @Value("${fr.insee.rmes.role.gestionnaire.colectica}") String gestionnaireColectica, + @Value("#{'${fr.insee.rmes.security.whitelist-matchers}'.split(',')}")String[] whiteList) { + this.inseeSecurityTokenProperties = inseeSecurityTokenProperties; + this.administrateurRole = administrateurRole; + this.gestionnaireColectica = gestionnaireColectica; + this.whiteList = whiteList; + } + @Bean - @ConditionalOnProperty(name = PROFILES_ACTIVE, havingValue = "prod") + @Profile("prod") public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { - //Webservice/RestAPI : disable CSRF & no sesion http - .csrf(csrf -> csrf.disable()) + .csrf(AbstractHttpConfigurer::disable) //disable sessions (stateless) .sessionManagement(session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS)); for (var pattern : whiteList) { @@ -75,11 +78,11 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { return http.build(); } @Bean - @ConditionalOnProperty(name = PROFILES_ACTIVE, havingValue = "dev") - public SecurityFilterChain filterChain_noSecurity(HttpSecurity http) throws Exception { + @Profile("dev") + public SecurityFilterChain filterChainNoSecurity(HttpSecurity http) throws Exception { //Allow frames to be able tu use the H2 web console http.headers(headers -> headers - .frameOptions(frameOptions -> frameOptions.sameOrigin() + .frameOptions(HeadersConfigurer.FrameOptionsConfig::sameOrigin ) ); @@ -93,39 +96,38 @@ public SecurityFilterChain filterChain_noSecurity(HttpSecurity http) throws Exce @Bean JwtAuthenticationConverter jwtAuthenticationConverter() { JwtAuthenticationConverter jwtAuthenticationConverter = new JwtAuthenticationConverter(); - jwtAuthenticationConverter.setPrincipalClaimName(inseeSecurityTokenProperties.getOidcClaimUsername()); + jwtAuthenticationConverter.setPrincipalClaimName(inseeSecurityTokenProperties.oidcClaimUsername()); jwtAuthenticationConverter.setJwtGrantedAuthoritiesConverter(jwtGrantedAuthoritiesConverter()); return jwtAuthenticationConverter; } Converter> jwtGrantedAuthoritiesConverter() { - return new Converter>() { - @Override @SuppressWarnings({ "unchecked" }) public Collection convert(Jwt source) { + return source -> { - String[] claimPath = inseeSecurityTokenProperties.getOidcClaimRole().split("\\."); - Map claims = source.getClaims(); - try { + String[] claimPath = inseeSecurityTokenProperties.oidcClaimRole().split("\\."); + Map claims = source.getClaims(); + try { + + for (int i = 0; i < claimPath.length - 1; i++) { + claims = (Map) claims.get(claimPath[i]); + } - for (int i = 0; i < claimPath.length - 1; i++) { - claims = (Map) claims.get(claimPath[i]); + List roles = (List) claims.getOrDefault(claimPath[claimPath.length - 1], List.of()); + //if we need to add customs roles to every connected user we could define this variable (static or from properties) + //roles.addAll(defaultRolesForUsers); + return Collections.unmodifiableCollection(roles.stream().map(s -> new GrantedAuthority() { + @Override + public String getAuthority() { + return ROLE_PREFIX + s; } - List roles = (List) claims.getOrDefault(claimPath[claimPath.length - 1], List.of()); - //if we need to add customs roles to every connected user we could define this variable (static or from properties) - //roles.addAll(defaultRolesForUsers); - return Collections.unmodifiableCollection(roles.stream().map(s -> new GrantedAuthority() { - @Override public String getAuthority() { - return ROLE_PREFIX + s; - } - - @Override public String toString() { - return getAuthority(); - } - }).toList()); - } - catch (ClassCastException e) { - return List.of(); - } + @Override + public String toString() { + return getAuthority(); + } + }).toList()); + } catch (ClassCastException e) { + return List.of(); } }; } diff --git a/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java b/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java index 86a60cea..d669772a 100644 --- a/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java +++ b/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java @@ -74,17 +74,15 @@ private boolean isCurrentTokenValid() { return expiration.get() != null && Instant.now().isBefore(expiration.get().minus(1, ChronoUnit.SECONDS)); } - public String getFreshToken() { + public synchronized String getFreshToken() { if (!this.isCurrentTokenValid()) { - synchronized (this.expiration) { token = getKeycloakAccessToken(); - this.expiration.set(decodeToken(token)); - } + this.expiration.set(expirationFrom(token)); } return token; } - private Instant decodeToken(String token) { + private Instant expirationFrom(String token) { return JWT.decode(token).getExpiresAtAsInstant(); } diff --git a/src/main/java/fr/insee/rmes/exceptions/GenericExceptionMapper.java b/src/main/java/fr/insee/rmes/exceptions/GenericExceptionMapper.java deleted file mode 100644 index 20bdbcb9..00000000 --- a/src/main/java/fr/insee/rmes/exceptions/GenericExceptionMapper.java +++ /dev/null @@ -1,28 +0,0 @@ -package fr.insee.rmes.exceptions; - -import jakarta.ws.rs.NotFoundException; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.ext.ExceptionMapper; -import jakarta.ws.rs.ext.Provider; - - -/** - * Created by acordier on 04/07/17. - */ -@Provider -public class GenericExceptionMapper implements ExceptionMapper { - - private Response.Status STATUS = Response.Status.INTERNAL_SERVER_ERROR; - - public Response toResponse(Throwable error) { - RestMessage message = new RestMessage(STATUS.getStatusCode(), "An unexpected error occured", - error.getMessage()); - if (error instanceof NotFoundException) { - STATUS = Response.Status.NOT_FOUND; - message.setMessage("Not Found"); - message.setDetails("No JAX-RS resource found for this path"); - } - return Response.status(STATUS).entity(message).type(MediaType.APPLICATION_JSON).build(); - } -} diff --git a/src/main/java/fr/insee/rmes/exceptions/RMeSEnvironment.java b/src/main/java/fr/insee/rmes/exceptions/RMeSEnvironment.java index 8f13aedb..0cbb2263 100644 --- a/src/main/java/fr/insee/rmes/exceptions/RMeSEnvironment.java +++ b/src/main/java/fr/insee/rmes/exceptions/RMeSEnvironment.java @@ -5,13 +5,10 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.MediaType; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.extern.slf4j.Slf4j; import org.json.JSONObject; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; +import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -20,17 +17,17 @@ @RestController @RequestMapping("/env") @Tag(name = "RMeS Environment") +@Slf4j public class RMeSEnvironment { - private static final Logger log = LogManager.getLogger(fr.insee.rmes.exceptions.RMeSEnvironment.class); - - @Autowired - Environment env; + private final Environment env; + public RMeSEnvironment(Environment env) { + this.env = env; + } - @GetMapping - @Produces(MediaType.APPLICATION_JSON) + @GetMapping(produces = MediaType.APPLICATION_JSON_VALUE) @Operation( summary = "Get DDI Access Services environment", description = "This will return a safe (no secrets) projection of the current backend environment" @@ -39,7 +36,7 @@ public class RMeSEnvironment { @ApiResponse(responseCode = "200", description = "Success"), @ApiResponse(responseCode = "404", description = "Not found") }) - public String getEnvironment() throws Exception { + public String getEnvironment() { try { JSONObject envRep = new JSONObject(); envRep.put("Swagger Host", env.getProperty("fr.insee.rmes.api.host")); diff --git a/src/main/java/fr/insee/rmes/exceptions/RMeSExceptionMapper.java b/src/main/java/fr/insee/rmes/exceptions/RMeSExceptionMapper.java deleted file mode 100644 index e759c870..00000000 --- a/src/main/java/fr/insee/rmes/exceptions/RMeSExceptionMapper.java +++ /dev/null @@ -1,17 +0,0 @@ -package fr.insee.rmes.exceptions; - -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.ext.ExceptionMapper; -import jakarta.ws.rs.ext.Provider; - -/** - * Created by acordier on 04/07/17. - */ -@Provider -public class RMeSExceptionMapper implements ExceptionMapper { - public Response toResponse(RmesExceptionIO ex) { - RestMessage message = ex.toRestMessage(); - return Response.status(message.getStatus()).entity(message).type(MediaType.APPLICATION_JSON).build(); - } -} diff --git a/src/main/java/fr/insee/rmes/exceptions/RmesExceptionIO.java b/src/main/java/fr/insee/rmes/exceptions/RmesExceptionIO.java index a9566cb1..467c5fa8 100644 --- a/src/main/java/fr/insee/rmes/exceptions/RmesExceptionIO.java +++ b/src/main/java/fr/insee/rmes/exceptions/RmesExceptionIO.java @@ -1,8 +1,9 @@ package fr.insee.rmes.exceptions; -import java.io.IOException; +import lombok.Getter; -public class RmesExceptionIO extends IOException { +@Getter +public class RmesExceptionIO extends RuntimeException { private final int status; private final String details; diff --git a/src/main/java/fr/insee/rmes/exceptions/VtlTransformationException.java b/src/main/java/fr/insee/rmes/exceptions/VtlTransformationException.java deleted file mode 100644 index 6cbadff3..00000000 --- a/src/main/java/fr/insee/rmes/exceptions/VtlTransformationException.java +++ /dev/null @@ -1,11 +0,0 @@ -package fr.insee.rmes.exceptions; - -public class VtlTransformationException extends RuntimeException { - public VtlTransformationException(String message) { - super(message); - } - - public VtlTransformationException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/src/main/java/fr/insee/rmes/exceptions/XsltTransformationException.java b/src/main/java/fr/insee/rmes/exceptions/XsltTransformationException.java index 8a4fc0f2..b8086c62 100644 --- a/src/main/java/fr/insee/rmes/exceptions/XsltTransformationException.java +++ b/src/main/java/fr/insee/rmes/exceptions/XsltTransformationException.java @@ -4,7 +4,7 @@ import java.util.Optional; -public class XsltTransformationException extends RuntimeException { +public class XsltTransformationException extends Exception { public XsltTransformationException(String message, SaxonApiException cause) { super(message, cause); diff --git a/src/main/java/fr/insee/rmes/metadata/controller/MetadataController.java b/src/main/java/fr/insee/rmes/metadata/controller/MetadataController.java index 298f1e7a..6237629b 100644 --- a/src/main/java/fr/insee/rmes/metadata/controller/MetadataController.java +++ b/src/main/java/fr/insee/rmes/metadata/controller/MetadataController.java @@ -6,11 +6,8 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.MediaType; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -31,13 +28,17 @@ @ApiResponse(responseCode = "404", description = "Not found"), @ApiResponse(responseCode = "406", description = "Not Acceptable"), @ApiResponse(responseCode = "500", description = "Internal server error") }) +@Slf4j public class MetadataController { - static final Logger log = LogManager.getLogger(MetadataController.class); - @Autowired - private MetadataService metadataService; - @GetMapping("/units") - @Produces(MediaType.APPLICATION_JSON) + private final MetadataService metadataService; + + public MetadataController(MetadataService metadataService) { + this.metadataService = metadataService; + } + + + @GetMapping(value = "/units", produces = MediaType.APPLICATION_JSON_VALUE) public List getUnits() throws Exception{ try { return metadataService.getUnits(); diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/DeleteItem.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/DeleteItem.java index 05ed1ad1..21ba40bd 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/DeleteItem.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/DeleteItem.java @@ -9,9 +9,6 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.json.simple.parser.ParseException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; @@ -39,8 +36,6 @@ }) public class DeleteItem { - final static Logger logger = LogManager.getLogger(DeleteItem.class); - private final ColecticaService colecticaService; @Autowired @@ -53,7 +48,7 @@ public DeleteItem(ColecticaService colecticaService) { @Operation(summary = "Delete a CodeList via Colectica API", description = "Delete a CodeList and all their children.") public ResponseEntity deleteCodeList(@RequestParam("uuid") String uuid) - throws IOException, ExceptionColecticaUnreachable, ParseException, RmesException { + throws IOException, ExceptionColecticaUnreachable, RmesException { colecticaService.sendDeleteColectica(uuid, TransactionType.COPYCOMMIT); return ResponseEntity.noContent().build(); } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java index d3ec8dc2..e5d6ae0f 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import fr.insee.rmes.exceptions.ExceptionColecticaUnreachable; import fr.insee.rmes.exceptions.RmesException; -import fr.insee.rmes.exceptions.RmesExceptionIO; import fr.insee.rmes.model.DDIItemType; import fr.insee.rmes.tocolecticaapi.fragments.DdiFragmentService; import fr.insee.rmes.tocolecticaapi.service.ColecticaService; @@ -13,7 +12,6 @@ import io.swagger.v3.oas.annotations.tags.Tag; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.json.simple.parser.ParseException; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -158,13 +156,13 @@ public Object getJsonWithChild( schema = @Schema( type = "string", example="d6c08ec1-c4d2-4b9a-b358-b23aa4e0af93")) String identifier, @RequestParam(value = "fieldIdName",defaultValue = "id") String outputField, - @RequestParam(value="fieldLabelName",defaultValue = "label") String fieldLabelName) throws Exception { + @RequestParam(value="fieldLabelName",defaultValue = "label") String fieldLabelName) throws JsonProcessingException { return colecticaService.getJsonWithChild(identifier, outputField, fieldLabelName); } @GetMapping(value = "/RessourcePackageToJson", produces = MediaType.APPLICATION_JSON_VALUE) public String getRessourcePackage( - @RequestParam(name = "uuid", required = true) String uuid) throws ExceptionColecticaUnreachable, JsonProcessingException, RmesExceptionIO, ParseException { + @RequestParam(name = "uuid", required = true) String uuid) throws ExceptionColecticaUnreachable, JsonProcessingException { return colecticaService.getRessourcePackage(uuid); } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java index bc1a4e1c..bc1c0c48 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/PostItem.java @@ -12,7 +12,6 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; -import org.json.simple.parser.ParseException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -77,7 +76,7 @@ public ResponseEntity uploadItem(@RequestParam("file") MultipartFile file) @PreAuthorize("hasRole('Administrateur_RMESGOPS')") public ResponseEntity byType( @PathVariable("type") DDIItemType type) - throws IOException, ExceptionColecticaUnreachable, ParseException { + throws IOException, ExceptionColecticaUnreachable{ return ResponseEntity.ok(colecticaService.getByType(type)); } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/RmesExceptionHandler.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/RmesExceptionHandler.java index 6afa8bf2..b48a54c6 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/RmesExceptionHandler.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/RmesExceptionHandler.java @@ -11,7 +11,7 @@ @ControllerAdvice(assignableTypes = {DeleteItem.class, GetItem.class, PostItem.class, TransformationController.class}) public class RmesExceptionHandler extends ResponseEntityExceptionHandler { - @ExceptionHandler({RmesException.class, RmesExceptionIO.class}) + @ExceptionHandler({RmesException.class}) public final ResponseEntity handleRmesException(RmesException exception){ logger.error(exception.getMessageAndDetails(), exception); return ResponseEntity.status(exception.getStatus()).body(exception.getMessage()); diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java b/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java index 73269b77..198e1b84 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java @@ -1,6 +1,7 @@ package fr.insee.rmes.tocolecticaapi.fragments; import fr.insee.rmes.exceptions.RmesException; +import fr.insee.rmes.exceptions.XsltTransformationException; import fr.insee.rmes.tocolecticaapi.service.ColecticaService; import fr.insee.rmes.transfoxsl.service.XsltTransformationService; import org.springframework.http.HttpStatus; @@ -17,11 +18,11 @@ public record DdiFragmentServiceImpl(XsltTransformationService xsltTransformatio @Override public String extractDataRelationship(String uuid) throws RmesException { try { - return xsltTransformationService.transformToXmlString( + return new String(xsltTransformationService.transformToRawText( new ByteArrayInputStream(colecticaService.searchColecticaInstanceByUuid(uuid).getBytes()), ddiRelationshipToJsonXsl - ); - } catch (IOException e) { + )); + } catch (IOException | XsltTransformationException e) { throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Error while processing DDI to json", e.getMessage()); } } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java index 239a9fdf..4ccc9320 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaService.java @@ -3,11 +3,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import fr.insee.rmes.exceptions.ExceptionColecticaUnreachable; import fr.insee.rmes.exceptions.RmesException; -import fr.insee.rmes.exceptions.RmesExceptionIO; import fr.insee.rmes.model.DDIItemType; import fr.insee.rmes.tocolecticaapi.models.TransactionType; import org.json.JSONArray; -import org.json.simple.parser.ParseException; import java.io.IOException; import java.util.List; @@ -16,7 +14,7 @@ public interface ColecticaService { String findFragmentByUuid(String uuid); - void sendDeleteColectica(String uuid, TransactionType transactionType) throws JsonProcessingException, ExceptionColecticaUnreachable, RmesExceptionIO, ParseException, RmesException; + void sendDeleteColectica(String uuid, TransactionType transactionType) throws JsonProcessingException, ExceptionColecticaUnreachable, RmesException; String searchColecticaInstanceByUuid(String uuid) throws RmesException; JSONArray findFragmentByUuidWithChildren(String uuid) throws RmesException; String filteredSearchText(String index, String texte) throws RmesException; @@ -24,11 +22,11 @@ public interface ColecticaService { String searchTexteByType(String index, String texte, DDIItemType type) throws RmesException; String searchByType(String index, DDIItemType type) throws RmesException; - List> getJsonWithChild(String identifier, String outputField, String fieldLabelName) throws Exception; + List> getJsonWithChild(String identifier, String outputField, String fieldLabelName) throws JsonProcessingException; - String getRessourcePackage(String uuid) throws ExceptionColecticaUnreachable, JsonProcessingException, RmesExceptionIO, ParseException; + String getRessourcePackage(String uuid) throws ExceptionColecticaUnreachable, JsonProcessingException; - String getByType(DDIItemType type) throws IOException, ExceptionColecticaUnreachable, ParseException; + String getByType(DDIItemType type) throws IOException, ExceptionColecticaUnreachable; void sendUpdateColectica(String ddiUpdatingInJson, TransactionType transactionType) throws RmesException; diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java index 35fd8299..b03b92e1 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java @@ -10,16 +10,17 @@ import fr.insee.rmes.exceptions.RmesBadRequestException; import fr.insee.rmes.exceptions.RmesException; import fr.insee.rmes.exceptions.RmesExceptionIO; +import fr.insee.rmes.exceptions.XsltTransformationException; import fr.insee.rmes.model.DDIItemType; import fr.insee.rmes.tocolecticaapi.models.NamespaceContextMap; import fr.insee.rmes.tocolecticaapi.models.RessourcePackage; import fr.insee.rmes.tocolecticaapi.models.TransactionType; import fr.insee.rmes.transfoxsl.service.internal.DDIDerefencer; import fr.insee.rmes.utils.ExportUtils; -import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.json.JSONArray; import org.json.JSONObject; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; @@ -53,26 +54,29 @@ @Service @Slf4j -public record ColecticaServiceImpl(@NonNull KeycloakServices kc, - ElasticService elasticService, +public record ColecticaServiceImpl(ElasticService elasticService, RestClient restClient, ExportUtils exportUtils, DDIDerefencer ddiDerefencer, - @Value("${fr.insee.rmes.api.remote.metadata.url}") - String serviceUrl, - @Value("${fr.insee.rmes.api.remote.metadata.agency}") String agency ) implements ColecticaService { - private static final String BEARER = "Bearer "; private static final String TRANSACTIONID = "{\"TransactionId\":"; - public ColecticaServiceImpl{ - restClient = initRestClient(); + @Autowired + public ColecticaServiceImpl(KeycloakServices kc, + ElasticService elasticService, + ExportUtils exportUtils, + DDIDerefencer ddiDerefencer, + @Value("${fr.insee.rmes.api.remote.metadata.url}") + String serviceUrl, + @Value("${fr.insee.rmes.api.remote.metadata.agency}") + String agency){ + this(elasticService, initRestClient(kc, serviceUrl), exportUtils, ddiDerefencer, agency); } - private RestClient initRestClient() { + private static RestClient initRestClient(KeycloakServices kc, String serviceUrl) { SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); factory.setConnectTimeout(100_000); // Temps de connexion en millisecondes factory.setReadTimeout(100_000); // Temps de lecture en millisecondes @@ -81,7 +85,6 @@ private RestClient initRestClient() { .requestFactory(factory) .baseUrl(URI.create(serviceUrl + "/").resolve("api/v1/")) .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_XML_VALUE) - .defaultHeader(HttpHeaders.AUTHORIZATION, BEARER) .defaultHeaders(headers -> headers.setBearerAuth(kc.getFreshToken())) .build(); } @@ -95,14 +98,15 @@ public String findFragmentByUuid(String uuid) { private String getWithRestClient(URI relativeUri, MediaType acceptedMediaType) { - return restClient.get().uri(relativeUri) + RestClient.ResponseSpec response = restClient.get().uri(relativeUri) .accept(acceptedMediaType) - .retrieve() - .onStatus(HttpStatusCode::is4xxClientError, (request, response) -> { - throw new RmesExceptionIO(response.getStatusCode().value(), "Bad request or inexisting resource", readBodySafely(response)); + .retrieve(); + return response + .onStatus(HttpStatusCode::is4xxClientError, (request, httpResponse) -> { + throw new RmesExceptionIO(httpResponse.getStatusCode().value(), "Bad request or inexisting resource", readBodySafely(httpResponse)); }) - .onStatus(HttpStatusCode::is5xxServerError, (request, response) -> { - throw new RmesExceptionIO(response.getStatusCode().value(), "Error while calling Colectica", readBodySafely(response)); + .onStatus(HttpStatusCode::is5xxServerError, (request, httpResponse) -> { + throw new RmesExceptionIO(httpResponse.getStatusCode().value(), "Error while calling Colectica", readBodySafely(httpResponse)); }) .body(String.class); } @@ -129,7 +133,8 @@ public JSONArray findFragmentByUuidWithChildren(String uuid) throws RmesExceptio String instance = searchColecticaInstanceByUuid(uuid); try { return XmlProcessing.parseDereferencedXmlWithEnum(this.ddiDerefencer.intermediateDereference(instance)); - } catch (ParserConfigurationException | IOException | SAXException | XPathExpressionException e) { + } catch (IOException | XsltTransformationException | ParserConfigurationException | SAXException | + XPathExpressionException e) { throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Error while processing XML from Colectica", e.getMessage()); } } @@ -270,7 +275,7 @@ private static String filterAndTransformResponse(String json) throws RmesExcepti } @Override - public List> getJsonWithChild(String identifier, String outputField, String fieldLabelName) throws Exception { + public List> getJsonWithChild(String identifier, String outputField, String fieldLabelName) throws JsonProcessingException { ObjectMapper objectMapper = new ObjectMapper(); objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); JsonNode jsonNode = objectMapper.readTree(findJsonsetByIdentifier(identifier)); diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ElasticService.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ElasticService.java index a168fcc9..8840c5ec 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ElasticService.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ElasticService.java @@ -3,6 +3,7 @@ import fr.insee.rmes.exceptions.RmesExceptionIO; import fr.insee.rmes.model.DDIItemType; import lombok.NonNull; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatusCode; @@ -15,23 +16,22 @@ import static fr.insee.rmes.transfoxsl.utils.RestClientUtils.readBodySafely; @Component -public record ElasticService(@Value("${fr.insee.rmes.elasticsearch.url}") String elasticUrl, - @Value("${fr.insee.rmes.elasticsearch.apikey}") String elasticApiKey, - RestClient elasticClient - ) { +public record ElasticService(RestClient elasticClient) { private static final String APIKEYHEADER = "apiKey "; private static final String SEARCH = "/_search"; - public ElasticService{ - elasticClient = initRestClient(); + @Autowired + public ElasticService(@Value("${fr.insee.rmes.elasticsearch.url}") String elasticUrl, + @Value("${fr.insee.rmes.elasticsearch.apikey}") String elasticApiKey){ + this(initRestClient(elasticApiKey, elasticUrl)); } - private RestClient initRestClient() { + private static RestClient initRestClient(String elasticApiKey, String elasticUrl) { return RestClient.builder() .defaultHeader(HttpHeaders.AUTHORIZATION, APIKEYHEADER+elasticApiKey) .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) - .baseUrl(URI.create(this.elasticUrl+"/")) + .baseUrl(URI.create(elasticUrl+"/")) .build(); } diff --git a/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java b/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java index 2466e30d..1fa0383a 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/controller/TransformationController.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import fr.insee.rmes.exceptions.RmesException; -import fr.insee.rmes.exceptions.VtlTransformationException; import fr.insee.rmes.exceptions.XsltTransformationException; import fr.insee.rmes.model.DDIItemType; import fr.insee.rmes.model.DicoVar; @@ -30,7 +29,10 @@ import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.util.UriUtils; +import org.xml.sax.SAXException; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -49,7 +51,6 @@ @Slf4j public class TransformationController { - public static final String FAILED_TO_PROCESS_THE_INPUT_FILE = "Failed to process the input file"; public static final String TRANSFORMATION_FAILED_DURING_THE_XSLT_PROCESSING = "Transformation failed during the XSLT processing"; public static final MediaType APPLICATION_ODT = MediaType.valueOf("application/vnd.oasis.opendocument.text"); public static final String REL_TO_JSON_XSL = "dataRelToJson.xsl"; @@ -87,7 +88,7 @@ public ResponseEntity dereferenceDDI(@RequestParam("file") MultipartFile @Operation(summary = "Générer un fichier texte contenant les règles VTL à partir d'une physicalInstance") @PostMapping(value = "/ddi2vtl", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public ResponseEntity ddi2vtl(@RequestParam("file") MultipartFile file) { + public ResponseEntity ddi2vtl(@RequestParam("file") MultipartFile file) throws RmesException { DDITransformerToVtl ddiTransformerToVtl = new DDITransformerToVtl(xsltTransformationService); @@ -110,18 +111,14 @@ public ResponseEntity ddi2vtl(@RequestParam("file") MultipartFile file .contentType(MediaType.TEXT_PLAIN) .body(resource); - } catch (IOException e) { - throw new VtlTransformationException(FAILED_TO_PROCESS_THE_INPUT_FILE + ".", e); - } catch (XsltTransformationException e) { - throw new VtlTransformationException(TRANSFORMATION_FAILED_DURING_THE_XSLT_PROCESSING + ".", e); - } catch (VtlTransformationException e) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(null); + } catch (IOException | XsltTransformationException e) { + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Failure while processing input file or while xslt transfo", e.getMessage(), e); } } @Operation(summary = "Transformer un objet dataRelationShip en JSON") @PostMapping(value = "/dataRelationShiptoJson", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public ResponseEntity ddiDataRelationShiptoJson(@RequestParam("file") MultipartFile file) { + public ResponseEntity ddiDataRelationShiptoJson(@RequestParam("file") MultipartFile file) throws RmesException { try { // Conversion du MultipartFile en InputStream InputStream inputStream = multipartFileUtils.convertToInputStream(file); @@ -140,18 +137,14 @@ public ResponseEntity ddiDataRelationShiptoJson(@RequestParam("file") Mu return ResponseEntity.ok() .contentType(MediaType.APPLICATION_JSON) .body(cleanJson); - } catch (IOException e) { - throw new VtlTransformationException(FAILED_TO_PROCESS_THE_INPUT_FILE + ".", e); - } catch (XsltTransformationException e) { - throw new VtlTransformationException(TRANSFORMATION_FAILED_DURING_THE_XSLT_PROCESSING + ".", e); - } catch (VtlTransformationException e) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(null); + } catch (IOException | XsltTransformationException e) { + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Failure while processing input file or while xslt transfo", e.getMessage(), e); } } @Operation(summary = "Générer les règles VTL à partir d'une physicalInstance et renvoyer sous forme de texte brut") @PostMapping(value = "/ddi2vtlBrut", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public ResponseEntity ddi2vtlBrut(@RequestParam("file") MultipartFile file) { + public ResponseEntity ddi2vtlBrut(@RequestParam("file") MultipartFile file) throws RmesException { try { // Conversion du MultipartFile en InputStream InputStream inputStream = multipartFileUtils.convertToInputStream(file); @@ -164,12 +157,8 @@ public ResponseEntity ddi2vtlBrut(@RequestParam("file") MultipartFile fi return ResponseEntity.ok() .contentType(MediaType.TEXT_PLAIN) .body(new String(output)); - } catch (IOException e) { - throw new VtlTransformationException(FAILED_TO_PROCESS_THE_INPUT_FILE + ".", e); - } catch (XsltTransformationException e) { - throw new VtlTransformationException(TRANSFORMATION_FAILED_DURING_THE_XSLT_PROCESSING + ".", e); - } catch (VtlTransformationException e) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(null); + } catch (IOException | XsltTransformationException e) { + throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Failure while processing input file or while xslt transfo", e.getMessage(), e); } } @@ -274,7 +263,7 @@ public ResponseEntity ddiToCodeBook( public ResponseEntity getCodeBookV2( @RequestParam(value = "dicoVar") DicoVar dicoVar, @RequestParam(value = "ddiFile") MultipartFile ddiFile - ) throws Exception { + ) throws IOException, RmesException, ParserConfigurationException, TransformerException, SAXException { String ddi = new String(ddiFile.getBytes(), StandardCharsets.UTF_8); return ResponseEntity.ok() diff --git a/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java b/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java index d7f3628c..c1486ae5 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/service/XsltTransformationService.java @@ -46,7 +46,7 @@ @Service @Slf4j -public class XsltTransformationService { +public record XsltTransformationService(Processor processor) { public static final String DISALLOW_DOCTYPE_DECL1 = "http://apache.org/xml/features/disallow-doctype-decl"; public static final String EXTERNAL_GENERAL_ENTITIES = "http://xml.org/sax/features/external-general-entities"; @@ -55,10 +55,8 @@ public class XsltTransformationService { static final byte[] DATA_TAG_BEGIN = "".getBytes(); static final byte[] DATA_TAG_END = "".getBytes(); - private final Processor processor; - public XsltTransformationService() { - this.processor = new Processor(false); + this(new Processor(false)); } @@ -225,11 +223,11 @@ static byte[] transformWithParameters(byte[] source, InputStream xsltFileJson, M return outputStream.toByteArray(); } - public String transformToXmlString(InputStream inputStream, String xslFileName) throws XsltTransformationException, IOException { + public String transformToXmlString(InputStream inputStream, String xslFileName) throws IOException, XsltTransformationException { return toXmlString(transformToXml(inputStream, xslFileName)); } - public byte[] transformToXml(InputStream inputStream, String xslFileName) throws IOException { + public byte[] transformToXml(InputStream inputStream, String xslFileName) throws IOException, XsltTransformationException { return transform(inputStream, xslFileName, SerializerConfigurer.forXmlString(processor)); } @@ -237,7 +235,7 @@ private String toXmlString(byte[] result) { return new String(result); } - public byte[] transformToRawText(InputStream stream, String xslFilename) throws IOException { + public byte[] transformToRawText(InputStream stream, String xslFilename) throws IOException, XsltTransformationException { return transform(stream, xslFilename, SerializerConfigurer.forRawText(processor)); } diff --git a/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDIDerefencer.java b/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDIDerefencer.java index 9fbed08c..45d646b5 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDIDerefencer.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDIDerefencer.java @@ -1,5 +1,6 @@ package fr.insee.rmes.transfoxsl.service.internal; +import fr.insee.rmes.exceptions.XsltTransformationException; import fr.insee.rmes.transfoxsl.service.XsltTransformationService; import lombok.NonNull; import org.springframework.stereotype.Service; @@ -13,16 +14,16 @@ public record DDIDerefencer(XsltTransformationService xsltTransformationService) public static final String DEREFERENCE_XSL = "dereference.xsl"; - public String dereferenceToString(@NonNull InputStream inputStream) throws IOException { + public String dereferenceToString(@NonNull InputStream inputStream) throws IOException, XsltTransformationException { // Première transformation - XML en sortie return xsltTransformationService.transformToXmlString(inputStream, DEREFERENCE_XSL); } - public byte[] intermediateDereference(@NonNull InputStream inputStream) throws IOException { + public byte[] intermediateDereference(@NonNull InputStream inputStream) throws IOException, XsltTransformationException { return xsltTransformationService.transformToXml(inputStream, DEREFERENCE_XSL); } - public byte[] intermediateDereference(@NonNull String xmlString) throws IOException { + public byte[] intermediateDereference(@NonNull String xmlString) throws IOException, XsltTransformationException { return intermediateDereference(new ByteArrayInputStream(xmlString.getBytes())); } diff --git a/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDITransformerToVtl.java b/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDITransformerToVtl.java index 30687766..2cbaefdc 100644 --- a/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDITransformerToVtl.java +++ b/src/main/java/fr/insee/rmes/transfoxsl/service/internal/DDITransformerToVtl.java @@ -1,5 +1,6 @@ package fr.insee.rmes.transfoxsl.service.internal; +import fr.insee.rmes.exceptions.XsltTransformationException; import fr.insee.rmes.transfoxsl.service.XsltTransformationService; import java.io.ByteArrayInputStream; @@ -14,11 +15,8 @@ public DDITransformerToVtl(XsltTransformationService xsltTransformationService){ this(xsltTransformationService, new DDIDerefencer(xsltTransformationService)); } - public byte[] transform(InputStream inputStream) throws IOException { - - + public byte[] transform(InputStream inputStream) throws IOException, XsltTransformationException { // Deuxième transformation - Texte en sortie (on récupère directement une liste de lignes de texte) return xsltTransformationService.transformToRawText(new ByteArrayInputStream(ddiDerefencer.intermediateDereference(inputStream)), DDI_2_VTL_XSL); - } } diff --git a/src/main/java/fr/insee/rmes/utils/DocumentBuilderUtils.java b/src/main/java/fr/insee/rmes/utils/DocumentBuilderUtils.java index f447033f..8b282d49 100644 --- a/src/main/java/fr/insee/rmes/utils/DocumentBuilderUtils.java +++ b/src/main/java/fr/insee/rmes/utils/DocumentBuilderUtils.java @@ -5,14 +5,10 @@ import org.xml.sax.InputSource; import org.xml.sax.SAXException; -import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.StringReader; import java.util.Optional; @Service @@ -21,10 +17,6 @@ public class DocumentBuilderUtils { private DocumentBuilderUtils(){} - public static Document getDocument(String fragment) throws ParserConfigurationException, IOException, SAXException { - return getDocument(new ByteArrayInputStream(fragment.getBytes())); - } - public static Document getDocument(InputStream fragment) throws ParserConfigurationException, IOException, SAXException { DocumentBuilder builder = DocumentBuilders.createSaferDocumentBuilder(Optional.empty()); if (null == fragment) { diff --git a/src/main/java/fr/insee/rmes/utils/FilesUtils.java b/src/main/java/fr/insee/rmes/utils/FilesUtils.java index c7d71ad7..07c9d9da 100644 --- a/src/main/java/fr/insee/rmes/utils/FilesUtils.java +++ b/src/main/java/fr/insee/rmes/utils/FilesUtils.java @@ -1,15 +1,5 @@ package fr.insee.rmes.utils; -import org.apache.commons.io.IOUtils; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; import java.text.Normalizer; public class FilesUtils { @@ -19,22 +9,6 @@ public static String reduceFileNameSize(String fileName, int maxLength) { return fileName.substring(0, Math.min(fileName.length(), maxLength)); } - //todo : besoin - public static File streamToFile(InputStream in, String fileName, String fileExtension) throws IOException { - final File tempFile = File.createTempFile(fileName, fileExtension); - tempFile.deleteOnExit(); - try (FileOutputStream out = new FileOutputStream(tempFile)) { - IOUtils.copy(in, out); - } - return tempFile; - } - - public static void addFileToZipFolder(Path fileToAdd, Path zipArchive) throws IOException { - try(FileSystem fs = FileSystems.newFileSystem(zipArchive)){ - Files.write(fs.getPath(fileToAdd.getFileName().toString()), Files.readAllBytes(fileToAdd)); - } - } - private FilesUtils() { throw new IllegalStateException("Utility class"); } diff --git a/src/main/java/fr/insee/rmes/utils/XsltUtils.java b/src/main/java/fr/insee/rmes/utils/XsltUtils.java index cf96f076..665edb6f 100644 --- a/src/main/java/fr/insee/rmes/utils/XsltUtils.java +++ b/src/main/java/fr/insee/rmes/utils/XsltUtils.java @@ -1,11 +1,5 @@ package fr.insee.rmes.utils; -import fr.insee.rmes.exceptions.RmesException; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpStatus; - import javax.xml.transform.Result; import javax.xml.transform.Source; import javax.xml.transform.Transformer; @@ -13,21 +7,14 @@ import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; import java.io.*; -import java.nio.charset.StandardCharsets; -import java.nio.file.*; -import java.util.Arrays; -import java.util.Map; import java.util.Optional; import java.util.function.Consumer; import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; public class XsltUtils { - private static final Logger logger = LoggerFactory.getLogger(XsltUtils.class); - - private XsltUtils() { + private XsltUtils() { throw new IllegalStateException("Utility class"); } diff --git a/src/main/java/fr/insee/rmes/utils/jersey/ObjectMapperProvider.java b/src/main/java/fr/insee/rmes/utils/jersey/ObjectMapperProvider.java deleted file mode 100644 index a908b2d8..00000000 --- a/src/main/java/fr/insee/rmes/utils/jersey/ObjectMapperProvider.java +++ /dev/null @@ -1,33 +0,0 @@ -package fr.insee.rmes.utils.jersey; - -import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.ANY; - - -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import jakarta.ws.rs.ext.ContextResolver; -import jakarta.ws.rs.ext.Provider; - -/** - * Created by acordier on 13/07/17. - */ -@Provider -public class ObjectMapperProvider implements ContextResolver { - - @Override - public ObjectMapper getContext(Class aClass) { - return createObjectMapper(); - } - - private ObjectMapper createObjectMapper() { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - objectMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); - objectMapper.setVisibility(PropertyAccessor.FIELD, ANY); - - return objectMapper; - } - -} \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 69c1c575..9e2fda4f 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -4,20 +4,11 @@ spring.servlet.multipart.max-file-size=50MB springdoc.swagger-ui.path= / springdoc.swagger-ui.enabled=true springdoc.swagger-ui.syntax-highlight.activated=false -fr.insee.rmes.security.enabled=false -logging.logback.rollingpolicy.max-file-size=1MB -logging.file.name= XXXX + spring.profiles.active= dev ##SWAGGER springdoc.swagger-ui.oauth.clientId= localhost-frontend -fr.insee.rmes.springdoc.issuer-url-authorization=${fr.insee.rmes.metadata.keycloak.server}/.../protocol/openid-connect/auth -fr.insee.rmes.springdoc.issuer-url-refresh=${fr.insee.rmes.metadata.keycloak.server}/.../protocol/openid-connect/token -fr.insee.rmes.springdoc.issuer-url-token=${fr.insee.rmes.metadata.keycloak.server}/.../protocol/openid-connect/token -fr.insee.rmes.springdoc.issuer-description=Keycloak -fr.insee.rmes.springdoc.contact-name= xxxxx -fr.insee.rmes.springdoc.contact-email=un-email-fonctionnel@xxx.fr -fr.insee.rmes.springdoc.contact-url=https://www.insee.fr/fr/accueil spring.security.oauth2.resourceserver.jwt.jwk-set-uri=${fr.insee.rmes.metadata.keycloak.server}/.../protocol/openid-connect/certs fr.insee.rmes.security.whitelist-matchers=/swagger-ui/index.html,/v3/api-docs/** @@ -27,30 +18,14 @@ fr.insee.rmes.role.gestionnaire.colectica= XXX fr.insee.rmes.security.token.oidc-claim-role= XXX fr.insee.rmes.security.token.oidc-claim-username= XXX -auth.api.url= XXX -item.api.url= ${fr.insee.rmes.api.remote.metadata.url}/api/v1/item/ - -#For colectica on kube demo -auth.username= XXX -auth.password= XXX - #search -fr.insee.rmes.search.root.sub-group.ids = 6baffbbc-86d5-4982-b6a4-7b035f7043ad, 2bfe02de-0044-4a94-b516-b166d9499971 -fr.insee.rmes.search.root.resource-package.ids = d574ed7e-2a7c-491f-b25c-f84141d4e96e fr.insee.rmes.elasticsearch.url= https://urlElastic -fr.insee.rmes.elasticsearch.index.name = xxxx -fr.insee.rmes.elasticsearch.host = urlElastic -fr.insee.rmes.elasticsearch.port = 80 -fr.insee.rmes.elasticsearch.apiId= id fr.insee.rmes.elasticsearch.apikey= cle # Metadata service -fr.insee.rmes.api.remote.metadata.MetadataService.impl = MetadataServiceImpl ## Colectica Repository API fr.insee.rmes.api.remote.metadata.url = http://ddiApi -fr.insee.rmes.api.remote.metadata.agency = xxxx -fr.insee.rmes.api.remote.metadata.key = xxxx -fr.insee.rmes.api.remote.metadata.user = xxxxx +fr.insee.rmes.api.remote.metadata.agency = xxx #Keycloack Services fr.insee.rmes.metadata.keycloak.secret = xxxxx @@ -58,9 +33,6 @@ fr.insee.rmes.metadata.keycloak.resource = xxxxx fr.insee.rmes.metadata.keycloak.server = https://keycloak.xxx.fr/ fr.insee.rmes.metadata.keycloak.realm = xxxx -#Cors -fr.insee.rmes.metadata.cors.allowedOrigin= * - # Swagger host fr.insee.rmes.api.host = localhost:8080 fr.insee.rmes.api.name= /ddi-access-services/api diff --git a/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java b/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java index a166f22b..511891dd 100644 --- a/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java +++ b/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java @@ -1,12 +1,15 @@ package fr.insee.rmes.tocolecticaapi.controller; -import fr.insee.rmes.tocolecticaapi.fragments.DdiFragmentService; +import fr.insee.rmes.config.InseeSecurityTokenProperties; +import fr.insee.rmes.config.SecurityConfig; +import fr.insee.rmes.tocolecticaapi.fragments.DdiFragmentServiceImpl; import fr.insee.rmes.tocolecticaapi.service.ColecticaService; +import fr.insee.rmes.transfoxsl.service.XsltTransformationService; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; import org.springframework.http.MediaType; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.bean.override.mockito.MockitoBean; @@ -17,11 +20,14 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @WebMvcTest(GetItem.class) +@Import(value = {DdiFragmentServiceImpl.class, SecurityConfig.class, XsltTransformationService.class}) +@EnableConfigurationProperties(InseeSecurityTokenProperties.class) @ActiveProfiles("dev") class GetItemTest { @@ -34,30 +40,19 @@ class GetItemTest { @Test void whenGetDataRelationship_shouldReturnRightJson() throws Exception { String uuid="34abf2d5-f0bb-47df-b3d2-42ff7f8f5874"; - var dataRelationShipEndpoint="ddiFragment/"+uuid+"/dataRelationship"; + var dataRelationShipEndpoint="/Item/ddiFragment/"+uuid+"/dataRelationship"; + when(colecticaService.searchColecticaInstanceByUuid(uuid)).thenReturn(read("/getItemTest/physicalInstance.xml")); mockMvc.perform(get(dataRelationShipEndpoint).accept(MediaType.APPLICATION_JSON)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) - .andExpect(content().json(read("34abf2d5-f0bb-47df-b3d2-42ff7f8f5874_expected.json"))); + .andExpect(content().json(read("/getItemTest/34abf2d5-f0bb-47df-b3d2-42ff7f8f5874_expected.json"))); } - private String read(String fileName) throws Exception { - try (Stream lines = Files.lines(Path.of(fileName))){ + private static String read(String fileName) throws Exception { + Path path = Path.of(GetItemTest.class.getResource(fileName).toURI()); + try (Stream lines = Files.lines(path)){ return lines.collect(Collectors.joining()); } } - @TestConfiguration - static class ConfigurationForTest{ - @Bean - DdiFragmentService ddiFragment(){ - return new DdiFragmentService() { - @Override - public String extractDataRelationship(String uuid) { - return ""; - } - }; - } - } - } \ No newline at end of file diff --git a/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java b/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java index e9cf262a..a283e152 100644 --- a/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java +++ b/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java @@ -1,11 +1,16 @@ package fr.insee.rmes.transfoxsl.controller; -import fr.insee.rmes.exceptions.VtlTransformationException; +import fr.insee.rmes.config.InseeSecurityTokenProperties; +import fr.insee.rmes.config.SecurityConfig; +import fr.insee.rmes.exceptions.XsltTransformationException; import fr.insee.rmes.transfoxsl.service.XsltTransformationService; import fr.insee.rmes.transfoxsl.utils.MultipartFileUtils; +import fr.insee.rmes.utils.export.XDocReport; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.context.annotation.Import; import org.springframework.http.MediaType; import org.springframework.mock.web.MockMultipartFile; import org.springframework.security.test.context.support.WithMockUser; @@ -16,14 +21,16 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; -import java.util.List; -import static org.mockito.ArgumentMatchers.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @WebMvcTest(TransformationController.class) +@Import(SecurityConfig.class) +@EnableConfigurationProperties(InseeSecurityTokenProperties.class) @ActiveProfiles("dev") class TransformationControllerTest { @@ -36,6 +43,9 @@ class TransformationControllerTest { @MockitoBean private MultipartFileUtils multipartFileUtils; + @MockitoBean + private XDocReport xDocReport; + @Test @WithMockUser // Simule un utilisateur authentifié void ddi2vtl_ShouldReturnPlainText_WhenTransformationIsSuccessful() throws Exception { @@ -46,11 +56,11 @@ void ddi2vtl_ShouldReturnPlainText_WhenTransformationIsSuccessful() throws Excep when(multipartFileUtils.convertToInputStream(mockFile)).thenReturn(new ByteArrayInputStream("".getBytes())); // Mocking transformation steps - List intermediateOutput = List.of("XML"); - List finalOutput = List.of("VTL rule 1", "VTL rule 2"); + byte[] intermediateOutput = "XML".getBytes(); + byte[] finalOutput = "VTL rule 1\nVTL rule 2".getBytes(); - when(xsltTransformationService.transform(any(), anyString(), anyBoolean())).thenReturn(intermediateOutput); - when(xsltTransformationService.transform(any(), anyString(), anyBoolean())).thenReturn(finalOutput); + when(xsltTransformationService.transformToXml(any(), anyString())).thenReturn(intermediateOutput); + when(xsltTransformationService.transformToRawText(any(), anyString())).thenReturn(finalOutput); // Perform the request and verify the response mockMvc.perform(MockMvcRequestBuilders.multipart("/xsl/ddi2vtl") @@ -70,8 +80,8 @@ void ddi2vtl_ShouldReturnServerError_WhenTransformationFails() throws Exception when(multipartFileUtils.convertToInputStream(any())).thenReturn(new ByteArrayInputStream("".getBytes())); // Simulate transformation failure with the correct exception - when(xsltTransformationService.transform(any(), anyString(), anyBoolean())) - .thenThrow(new VtlTransformationException("Transformation failed during the XSLT processing.")); + when(xsltTransformationService.transformToXml(any(), anyString())) + .thenThrow(new XsltTransformationException("Transformation failed during the XSLT processing.", null)); // Perform the request and expect the server error mockMvc.perform(MockMvcRequestBuilders.multipart("/xsl/ddi2vtl") @@ -81,7 +91,7 @@ void ddi2vtl_ShouldReturnServerError_WhenTransformationFails() throws Exception } @Test - @WithMockUser // Simule un utilisateur authentifié + @WithMockUser() // Simule un utilisateur authentifié void dataRelationShiptoJson_ShouldReturnJson_WhenTransformationIsSuccessful() throws Exception { // Mocking MultipartFile and transformation service MockMultipartFile mockFile = new MockMultipartFile("file", "test.xml", "application/xml", "".getBytes()); @@ -90,21 +100,20 @@ void dataRelationShiptoJson_ShouldReturnJson_WhenTransformationIsSuccessful() th when(multipartFileUtils.convertToInputStream(mockFile)).thenReturn(new ByteArrayInputStream("".getBytes())); // Mocking transformation result - String intermediateOutput = "[{\"VTL\":\"rule 1\", \"VTL2\":\"rule 2\"}]"; + byte[] intermediateOutput = "[{\"VTL\":\"rule 1\", \"VTL2\":\"rule 2\"}]".getBytes(); - when(xsltTransformationService.transform(any(InputStream.class), anyString(), anyBoolean())) - .thenReturn(List.of(intermediateOutput)); + when(xsltTransformationService.transformToRawText(any(InputStream.class), anyString())) + .thenReturn(intermediateOutput); // Expected formatted JSON output (as an object) String expectedJson = "[{\n \"VTL\" : \"rule 1\",\n \"VTL2\" : \"rule 2\"\n}]"; // Perform the request with CSRF token and verify the JSON response - mockMvc.perform(MockMvcRequestBuilders.multipart("/xsl/dataRelationShiptoJson") // Correct URL - .file(mockFile) - .with(csrf())) // Add CSRF token to the request + mockMvc.perform(MockMvcRequestBuilders.multipart("/xsl/dataRelationShiptoJson") + .file(mockFile)) .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) // Verify JSON content type - .andExpect(content().json(expectedJson)); // Verify formatted JSON response + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andExpect(content().json(expectedJson)); } @@ -119,12 +128,12 @@ void ddi2vtlBrut_ShouldReturnPlainText_WhenTransformationIsSuccessful() throws E when(multipartFileUtils.convertToInputStream(mockFile)).thenReturn(new ByteArrayInputStream("".getBytes())); // Mocking transformation steps - List intermediateOutput = List.of("XML"); - List finalOutput = List.of("VTL rule 1", "VTL rule 2"); + byte[] intermediateOutput = "XML".getBytes(); + byte[] finalOutput = "VTL rule 1\nVTL rule 2".getBytes(); - when(xsltTransformationService.transform(any(InputStream.class), anyString(), anyBoolean())) + when(xsltTransformationService.transformToXml(any(InputStream.class), anyString())) .thenReturn(intermediateOutput); - when(xsltTransformationService.transform(any(InputStream.class), anyString(), anyBoolean())) + when(xsltTransformationService.transformToRawText(any(InputStream.class), anyString())) .thenReturn(finalOutput); // Perform the request and verify the plain text response diff --git a/src/test/java/fr/insee/rmes/transfoxsl/service/XsltTransformationServiceTest.java b/src/test/java/fr/insee/rmes/transfoxsl/service/XsltTransformationServiceTest.java index e6b762e1..83f4a4e6 100644 --- a/src/test/java/fr/insee/rmes/transfoxsl/service/XsltTransformationServiceTest.java +++ b/src/test/java/fr/insee/rmes/transfoxsl/service/XsltTransformationServiceTest.java @@ -1,48 +1,93 @@ package fr.insee.rmes.transfoxsl.service; -import org.junit.Before; -import org.junit.Test; -import org.mockito.InjectMocks; -import org.mockito.MockitoAnnotations; +import fr.insee.rmes.exceptions.RmesException; +import fr.insee.rmes.model.DDIItemType; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.core.io.Resource; +import org.springframework.mock.web.MockMultipartFile; +import java.io.ByteArrayInputStream; import java.io.InputStream; -import java.util.List; import static org.junit.jupiter.api.Assertions.*; -public class XsltTransformationServiceTest { +class XsltTransformationServiceTest { - @InjectMocks - private XsltTransformationService xsltTransformationService; - - @Before - public void setUp() { - MockitoAnnotations.openMocks(this); // Initialiser les mocks - } + private final XsltTransformationService xsltTransformationService = new XsltTransformationService(); @Test - public void transform_ShouldReturnTransformedText_WhenXslTransformationIsSuccessful() throws Exception { + void transform_ShouldReturnTransformedText_WhenXslTransformationIsSuccessful() throws Exception { // Utilisation d'un vrai fichier XML en tant qu'InputStream InputStream inputStream = getClass().getClassLoader().getResourceAsStream("physicalInstance-test.xml"); assertNotNull(inputStream, "The XML input stream should not be null"); // Exécution de la transformation avec un fichier XSL réel - List result = xsltTransformationService.transform(inputStream, "testTransformation.xsl", true); + byte[] result = xsltTransformationService.transformToRawText(inputStream, "testTransformation.xsl"); - // Vérification des résultats assertNotNull(result); - assertTrue(result.size() > 0); - assertEquals("Transformation result: fr.insee", result.get(0)); // Personnalisez cette valeur selon vos attentes + assertTrue(result.length > 0); + assertEquals("Transformation result: fr.insee", new String(result)); } @Test - public void transform_ShouldThrowException_WhenXslFileIsInvalid() { + void transform_ShouldThrowException_WhenXslFileIsInvalid() { InputStream inputStream = getClass().getClassLoader().getResourceAsStream("physicalInstance-test.xml"); assertNotNull(inputStream, "The XML input stream should not be null"); // Simuler un fichier XSL non valide en passant un nom de fichier inexistant assertThrows(Exception.class, () -> { - xsltTransformationService.transform(inputStream, "invalid.xsl", true); + xsltTransformationService.transformToRawText(inputStream, "invalid.xsl"); }); } + + @Test + void testReplaceXmlParameters() { + String inputXml = "1Old NameOld Labelurn:example:1"; + DDIItemType type = DDIItemType.CODE_LIST; + String label = "New Label"; + int version = 2; + String name = "New Name"; + String idepUtilisateur = "user"; + String result = xsltTransformationService.replaceXmlParameters(inputXml, type, label, version, name, idepUtilisateur); + assertNotNull(result); + assertFalse(result.isEmpty()); + } + + @Test + void testTransformFile() throws RmesException { + MockMultipartFile file = new MockMultipartFile("file", "test.json", "application/json", "[{\"id\": \"value\",\"label\": \"value\" }]".getBytes()); + String result = xsltTransformationService.transformFile(file, "idValue", "nomenclatureName", "suggesterDescription", "version", "idepUtilisateur", "timbre"); + assertNotNull(result); + } + + @Test + void testTransformToJson() throws Exception { + // Mock the Resource to simulate the XML input + Resource mockResource = Mockito.mock(Resource.class); + Mockito.when(mockResource.getInputStream()).thenReturn(new ByteArrayInputStream("Test".getBytes())); + + // Provide a valid XSLT as InputStream + String xsltContent = """ + + + + + + + + + """; + InputStream xsltStream = new ByteArrayInputStream(xsltContent.getBytes()); + + // Call the method to be tested + String result = new String(XsltTransformationService.transformToJson(mockResource, xsltStream, "user")); + + // Assertions + assertNotNull(result); + assertFalse(result.isEmpty()); + assertTrue(result.contains("")); + assertTrue(result.contains("Test")); + } + } diff --git a/src/test/java/fr/insee/rmes/transfoxsl/utils/MultipartFileUtilsTest.java b/src/test/java/fr/insee/rmes/transfoxsl/utils/MultipartFileUtilsTest.java index a338ad5b..ad53e52e 100644 --- a/src/test/java/fr/insee/rmes/transfoxsl/utils/MultipartFileUtilsTest.java +++ b/src/test/java/fr/insee/rmes/transfoxsl/utils/MultipartFileUtilsTest.java @@ -1,7 +1,7 @@ package fr.insee.rmes.transfoxsl.utils; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.springframework.mock.web.MockMultipartFile; @@ -17,19 +17,19 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; -public class MultipartFileUtilsTest { +class MultipartFileUtilsTest { @Mock private MultipartFileUtils multipartFileUtils; // Mock class - @Before + @BeforeEach public void setUp() { // Initialiser les mocks MockitoAnnotations.openMocks(this); } @Test - public void convertToInputStream_ShouldReturnValidInputStream_WhenFileIsProvided() throws Exception { + void convertToInputStream_ShouldReturnValidInputStream_WhenFileIsProvided() throws Exception { // Mocking MultipartFile MockMultipartFile mockFile = new MockMultipartFile("file", "test.xml", "application/xml", "".getBytes()); diff --git a/src/test/java/fr/insee/rmes/utils/DocumentBuilderUtilsTest.java b/src/test/java/fr/insee/rmes/utils/DocumentBuilderUtilsTest.java new file mode 100644 index 00000000..319709e2 --- /dev/null +++ b/src/test/java/fr/insee/rmes/utils/DocumentBuilderUtilsTest.java @@ -0,0 +1,21 @@ +package fr.insee.rmes.utils; + +import org.junit.jupiter.api.Test; +import org.w3c.dom.Document; + +import java.io.ByteArrayInputStream; + +import static org.junit.jupiter.api.Assertions.*; + +class DocumentBuilderUtilsTest { + + @Test + void testGetDocument() throws Exception { + String xml = "value"; + Document doc = DocumentBuilderUtils.getDocument(new ByteArrayInputStream(xml.getBytes())); + + assertNotNull(doc); + assertEquals("root", doc.getDocumentElement().getNodeName()); + } + +} \ No newline at end of file diff --git a/src/test/java/service/ColecticaServiceImplTest.java b/src/test/java/service/ColecticaServiceImplTest.java index b3e1eddb..31bd404a 100644 --- a/src/test/java/service/ColecticaServiceImplTest.java +++ b/src/test/java/service/ColecticaServiceImplTest.java @@ -1,173 +1,59 @@ package service; -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import fr.insee.rmes.config.keycloak.KeycloakServices; -import fr.insee.rmes.model.DDIItemType; +import fr.insee.rmes.exceptions.RmesException; +import fr.insee.rmes.exceptions.RmesExceptionIO; import fr.insee.rmes.tocolecticaapi.service.ColecticaServiceImpl; -import fr.insee.rmes.transfoxsl.service.XsltTransformationService; -import fr.insee.rmes.utils.DocumentBuilderUtils; -import org.apache.http.ProtocolVersion; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.message.BasicStatusLine; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.io.Resource; +import org.springframework.boot.test.web.client.MockServerRestClientCustomizer; import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.mock.web.MockMultipartFile; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.test.util.ReflectionTestUtils; -import org.springframework.web.client.RestTemplate; -import org.w3c.dom.Document; +import org.springframework.http.MediaType; +import org.springframework.web.client.RestClient; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withResourceNotFound; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; -@SpringBootTest -@TestPropertySource(properties = {"fr.insee.rmes.api.remote.metadata.url=http://mock-url"}) -@RunWith(SpringRunner.class) class ColecticaServiceImplTest { - @Configuration - static class TestConfig { - // Configuration beans can be defined here if needed - } - static class TestableColecticaServiceImpl extends ColecticaServiceImpl { - public static ResponseEntity callProtectedMethod(CloseableHttpResponse response) throws IOException { - return getResponseEntitySearchColecticaFragmentByUuid(response); - } - } - - @Mock - private CloseableHttpClient mockHttpClient; - - @Mock - private RestTemplate restTemplate; - - @Mock - private ElasticsearchClient elasticsearchClient; - - @Mock - private KeycloakServices keycloakServices; - - @InjectMocks - private ColecticaServiceImpl colecticaServiceImpl; - - @BeforeEach - void setUp() throws Exception { - MockitoAnnotations.openMocks(this); - when(keycloakServices.isTokenValid(anyString())).thenReturn(true); - - CloseableHttpResponse mockResponse = mock(CloseableHttpResponse.class); - when(mockResponse.getStatusLine()).thenReturn(new BasicStatusLine(new ProtocolVersion("HTTP", 1, 1), 404, "Not Found")); - StringEntity entity = new StringEntity(""); - when(mockResponse.getEntity()).thenReturn(entity); - when(mockHttpClient.execute(any(HttpGet.class))).thenReturn(mockResponse); - - colecticaServiceImpl = new ColecticaServiceImpl(mockHttpClient, elasticsearchClient, restTemplate); - when(keycloakServices.getKeycloakAccessToken()).thenReturn("un_token_valide"); - ReflectionTestUtils.setField(colecticaServiceImpl, "serviceUrl", "http://mock-url"); - } - - @Test - void getResponseEntity_NotOkStatus() throws IOException { - CloseableHttpResponse mockResponse = mock(CloseableHttpResponse.class); - when(mockResponse.getStatusLine()).thenReturn(new BasicStatusLine(new ProtocolVersion("HTTP", 1, 1), 404, "Not Found")); - ResponseEntity result = TestableColecticaServiceImpl.callProtectedMethod(mockResponse); - assertEquals(HttpStatus.NOT_FOUND, result.getStatusCode()); - assertTrue(result.getBody().contains("Erreur lors de la requête vers Colectica.")); - } - @Test - void getResponseEntity_OkStatus() throws IOException { - String jsonContent = "{\"Item\": \"Contenu\"}"; - CloseableHttpResponse mockResponse = mock(CloseableHttpResponse.class); - when(mockResponse.getStatusLine()).thenReturn(new BasicStatusLine(new ProtocolVersion("HTTP", 1, 1), 200, "OK")); - when(mockResponse.getEntity()).thenReturn(new StringEntity(jsonContent)); - ResponseEntity result = TestableColecticaServiceImpl.callProtectedMethod(mockResponse); - assertEquals(HttpStatus.OK, result.getStatusCode()); - assertTrue(result.getBody().contains("Contenu")); - } - - @Test - void testReplaceXmlParameters() { - String inputXml = "1Old NameOld Labelurn:example:1"; - DDIItemType type = DDIItemType.CODE_LIST; - String label = "New Label"; - int version = 2; - String name = "New Name"; - String idepUtilisateur = "user"; - ColecticaServiceImpl colecticaService = new ColecticaServiceImpl(elasticsearchClient, restTemplate); - String result = colecticaService.replaceXmlParameters(inputXml, type, label, version, name, idepUtilisateur); - assertNotNull(result); - assertFalse(result.isEmpty()); - } + void getResponseEntity_NotOkStatus() { + var uuid="1"; + var agency="insee"; + MockServerRestClientCustomizer customizer = new MockServerRestClientCustomizer(); + String baseUrl = "http://keycloak/"; + RestClient.Builder builder = RestClient.builder() + .baseUrl(baseUrl); + customizer.customize(builder); + customizer.getServer().expect(requestTo(baseUrl+"item/"+agency+"/"+ uuid)).andRespond(withResourceNotFound()); - @Test - void testTransformFile() { - ColecticaServiceImpl colecticaService = new ColecticaServiceImpl(elasticsearchClient, restTemplate); - MockMultipartFile file = new MockMultipartFile("file", "test.json", "application/json", "[{\"id\": \"value\",\"label\": \"value\" }]".getBytes()); - ResponseEntity result = colecticaService.transformFile(file, "idValue", "nomenclatureName", "suggesterDescription", "version", "idepUtilisateur", "timbre"); - assertNotNull(result); - assertEquals(HttpStatus.OK, result.getStatusCode()); - assertNotNull(result.getBody()); + ColecticaServiceImpl colecticaService = new ColecticaServiceImpl(null, builder.build(), null, null, agency); + var expectedException=assertThrows(RmesExceptionIO.class, ()-> colecticaService.findFragmentByUuid(uuid)); + assertThat(expectedException.getStatus()).isEqualTo(HttpStatus.NOT_FOUND.value()); + assertThat(expectedException.getMessage()).isEqualTo("Bad request or inexisting resource"); } @Test - void testTransformToJson() throws Exception { - ColecticaServiceImpl colecticaService = new ColecticaServiceImpl(); - - // Mock the Resource to simulate the XML input - Resource mockResource = Mockito.mock(Resource.class); - Mockito.when(mockResource.getInputStream()).thenReturn(new ByteArrayInputStream("Test".getBytes())); - - // Provide a valid XSLT as InputStream - String xsltContent = "\n" + - "\n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - ""; - InputStream xsltStream = new ByteArrayInputStream(xsltContent.getBytes()); - - // Call the method to be tested - String result = XsltTransformationService.transformToJson(mockResource, xsltStream, "user"); - - // Assertions - assertNotNull(result); - assertTrue(!result.isEmpty()); - assertTrue(result.contains("")); - assertTrue(result.contains("Test")); - } + void getResponseEntity_OkStatus() throws IOException { + String xmlContent = "Contenu"; + String jsonContent = "{\"Item\": \""+xmlContent+"\"}"; + var uuid="1"; + var agency="insee"; + String baseUrl = "http://keycloak/"; + MockServerRestClientCustomizer customizer = new MockServerRestClientCustomizer(); + RestClient.Builder builder = RestClient.builder() + .baseUrl(baseUrl); + customizer.customize(builder); + customizer.getServer().expect(requestTo(baseUrl+"item/"+agency+"/"+ uuid)).andRespond(withSuccess(jsonContent, MediaType.APPLICATION_JSON)); - @Test - void testGetDocument() throws Exception { - String xml = "value"; - Document doc = DocumentBuilderUtils.getDocument(xml); + ColecticaServiceImpl colecticaService = new ColecticaServiceImpl(null, builder.build(), null, null, agency); - assertNotNull(doc); - assertEquals("root", doc.getDocumentElement().getNodeName()); + assertThat(colecticaService.findFragmentByUuid(uuid)).hasToString(xmlContent); } diff --git a/src/test/resources/getItemTest/34abf2d5-f0bb-47df-b3d2-42ff7f8f5874_expected.json b/src/test/resources/getItemTest/34abf2d5-f0bb-47df-b3d2-42ff7f8f5874_expected.json index a8c39682..0d46b857 100644 --- a/src/test/resources/getItemTest/34abf2d5-f0bb-47df-b3d2-42ff7f8f5874_expected.json +++ b/src/test/resources/getItemTest/34abf2d5-f0bb-47df-b3d2-42ff7f8f5874_expected.json @@ -47,7 +47,7 @@ ], "ordre": "4", "representation": "codes", "controles": { "codes": [ - { "code": "1 " , "label": [{ + { "code": "1" , "label": [{ "langue": "fr", "contenu": "Agriculteurs exploitants" }, @@ -58,7 +58,7 @@ ] } - ,{ "code": "2 " , "label": [{ + ,{ "code": "2" , "label": [{ "langue": "fr", "contenu": "Artisans, commerçants et chefs d'entreprise" }, @@ -69,7 +69,7 @@ ] } - ,{ "code": "3 " , "label": [{ + ,{ "code": "3" , "label": [{ "langue": "fr", "contenu": "Cadres et professions intellectuelles supérieures" }, @@ -80,7 +80,7 @@ ] } - ,{ "code": "4 " , "label": [{ + ,{ "code": "4" , "label": [{ "langue": "fr", "contenu": "Professions intermédiaires" }, @@ -91,7 +91,7 @@ ] } - ,{ "code": "5 " , "label": [{ + ,{ "code": "5" , "label": [{ "langue": "fr", "contenu": "Employés" }, @@ -102,7 +102,7 @@ ] } - ,{ "code": "6 " , "label": [{ + ,{ "code": "6" , "label": [{ "langue": "fr", "contenu": "Ouvriers" }, @@ -113,7 +113,7 @@ ] } - ,{ "code": "7 " , "label": [{ + ,{ "code": "7" , "label": [{ "langue": "fr", "contenu": "Retraités" }, @@ -124,7 +124,7 @@ ] } - ,{ "code": "8 " , "label": [{ + ,{ "code": "8" , "label": [{ "langue": "fr", "contenu": "Autres inactifs" }, @@ -147,7 +147,7 @@ ], "ordre": "5", "representation": "codes", "controles": { "codes": [ - { "code": "EE&FIP " , "label": [{ + { "code": "EE&FIP" , "label": [{ "langue": "fr", "contenu": "L’individu a été enquêté à l’enquête Emploi, et au moins une déclaration fiscale le concernant a été retrouvée" }, @@ -158,7 +158,7 @@ ] } - ,{ "code": "EE_CAF " , "label": [{ + ,{ "code": "EE_CAF" , "label": [{ "langue": "fr", "contenu": "L’individu d’un ménage considéré comme répondant ERFS ne se retrouve dans aucune déclaration fiscale, mais ses revenus ont pu être estimés à partir d’un appariement réussi avec le fichier CNAF" }, @@ -169,7 +169,7 @@ ] } - ,{ "code": "EE " , "label": [{ + ,{ "code": "EE" , "label": [{ "langue": "fr", "contenu": "L'individu d’un ménage considéré comme répondant ERFS ne se retrouve dans aucune des déclarations fiscales associées au ménage, ses revenus individuels seront donc entièrement imputés" }, @@ -180,7 +180,7 @@ ] } - ,{ "code": "EE_NRT " , "label": [{ + ,{ "code": "EE_NRT" , "label": [{ "langue": "fr", "contenu": "Etudiants de moins de 26 ans formant un ménage à part dans l’Enquête Emploi et dont on n'a retrouvé aucune trace dans les fichiers fiscaux (non-réponse totale), vraisemblablement parce qu’ils étaient rattachés au foyer fiscal de leur(s) parent(s)" }, @@ -203,7 +203,7 @@ ], "ordre": "6", "representation": "codes", "controles": { "codes": [ - { "code": "vous " , "label": [{ + { "code": "vous" , "label": [{ "langue": "fr", "contenu": "L'individu est le déclarant" }, @@ -214,7 +214,7 @@ ] } - ,{ "code": "conj " , "label": [{ + ,{ "code": "conj" , "label": [{ "langue": "fr", "contenu": "Conjoint" }, @@ -225,7 +225,7 @@ ] } - ,{ "code": "pac " , "label": [{ + ,{ "code": "pac" , "label": [{ "langue": "fr", "contenu": "Personne à charge" }, @@ -248,7 +248,7 @@ ], "ordre": "7", "representation": "codes", "controles": { "codes": [ - { "code": "mad " , "label": [{ + { "code": "mad" , "label": [{ "langue": "fr", "contenu": "Madame" }, @@ -259,7 +259,7 @@ ] } - ,{ "code": "mon " , "label": [{ + ,{ "code": "mon" , "label": [{ "langue": "fr", "contenu": "Monsieur" }, @@ -270,7 +270,7 @@ ] } - ,{ "code": "vopa " , "label": [{ + ,{ "code": "vopa" , "label": [{ "langue": "fr", "contenu": "Individus atteignant l'âge de 18 ans l’année de référence, pouvant avoir une double déclaration (seule et sur celle de leurs parents)" }, @@ -752,7 +752,7 @@ ], "ordre": "59", "representation": "codes", "controles": { "codes": [ - { "code": "1 " , "label": [{ + { "code": "1" , "label": [{ "langue": "fr", "contenu": "Salarié" }, @@ -763,7 +763,7 @@ ] } - ,{ "code": "2 " , "label": [{ + ,{ "code": "2" , "label": [{ "langue": "fr", "contenu": "Indépendant" }, @@ -774,7 +774,7 @@ ] } - ,{ "code": "3 " , "label": [{ + ,{ "code": "3" , "label": [{ "langue": "fr", "contenu": "Chômeur" }, @@ -785,7 +785,7 @@ ] } - ,{ "code": "4 " , "label": [{ + ,{ "code": "4" , "label": [{ "langue": "fr", "contenu": "Retraité" }, @@ -796,7 +796,7 @@ ] } - ,{ "code": "5 " , "label": [{ + ,{ "code": "5" , "label": [{ "langue": "fr", "contenu": "Autre inactif" }, diff --git a/src/test/resources/getItemTest/physicalInstance.xml b/src/test/resources/getItemTest/physicalInstance.xml new file mode 100644 index 00000000..8b203835 --- /dev/null +++ b/src/test/resources/getItemTest/physicalInstance.xml @@ -0,0 +1,2352 @@ + + + + fr.insee + 34abf2d5-f0bb-47df-b3d2-42ff7f8f5874 + 1 + PhysicalInstance + + + + urn:ddi:fr.insee:34abf2d5-f0bb-47df-b3d2-42ff7f8f5874:1 + fr.insee + 34abf2d5-f0bb-47df-b3d2-42ff7f8f5874 + 1 + + ddi31:Paths + {"0":"X:\\_ORG_SOURCES\\ERFS\\ERFS\\ERFS_ERFS_2017\\indivi17.sas7bdat"} + + + + indivi17 + + Table contenant tous les individus appartenant à un ménage répondant au quatrième trimestre 2017 de l'Enquête Emploi et apparié, que ceux-ci aient rempli ou non un questionnaire individuel lors de cette enquête + + + fr.insee + fcc416b8-2ea0-4df4-8420-4f7f3d761168 + 1 + DataRelationship + + + file:///X:/_ORG_SOURCES/ERFS/ERFS/ERFS_ERFS_2017/indivi17.sas7bdat + + + urn:ddi:fr.insee:c4a69fba-4564-432e-99bb-48cd7c75d607:1 + fr.insee + c4a69fba-4564-432e-99bb-48cd7c75d607 + 1 + 115357 + + + + + + urn:ddi:fr.insee:fcc416b8-2ea0-4df4-8420-4f7f3d761168:1 + fr.insee + fcc416b8-2ea0-4df4-8420-4f7f3d761168 + 1 + + indivi17 + + + urn:ddi:fr.insee:24f91f23-a83d-447b-983f-bac7baaed2ea:1 + fr.insee + 24f91f23-a83d-447b-983f-bac7baaed2ea + 1 + + indivi17 + + + + fr.insee + 8ec8f1bc-ff67-45ee-a78b-c31951a78cb3 + 1 + Variable + + + fr.insee + 6758fc71-dc86-4edb-8c14-0f14362b3dee + 1 + Variable + + + fr.insee + 4d63151e-3bb5-444b-b3f1-9771507e0ea0 + 1 + Variable + + + fr.insee + ab13bec9-0495-426d-a582-4b087a7b6ea5 + 1 + Variable + + + fr.insee + 0369bc65-682c-4b63-9d09-9b339ec49dd2 + 1 + Variable + + + fr.insee + 5d269749-dd06-4af9-a91e-8972a2b2ba82 + 1 + Variable + + + fr.insee + 8abdcea6-ce62-46a4-b0d8-31a7d9bde324 + 1 + Variable + + + fr.insee + f76704e8-1df8-45fb-9c0c-3dac8b29b372 + 1 + Variable + + + fr.insee + 63056208-ea5d-40b4-afba-4e4439f13b9f + 1 + Variable + + + fr.insee + 19e06933-0e02-44e8-9a9b-875c1b7581b2 + 1 + Variable + + + fr.insee + 3783b748-2f6e-4143-88a9-5d454f432d9a + 1 + Variable + + + fr.insee + 87e85c5d-6380-4423-be51-b4c9f3736e55 + 1 + Variable + + + fr.insee + 00b3f378-535f-4770-961d-4822aebf36b0 + 1 + Variable + + + fr.insee + 42ae6bd5-804a-4182-b68e-70ff83193ba7 + 1 + Variable + + + fr.insee + 96d279f5-cef3-40e6-b519-b9dc7bfd42f7 + 1 + Variable + + + fr.insee + b399dde0-790b-4a57-b530-354664ba7ba1 + 1 + Variable + + + fr.insee + f08de465-fb1b-42c3-be4f-34e0390e216e + 1 + Variable + + + fr.insee + dd2b743d-82a6-450f-8274-e5dbccc02946 + 1 + Variable + + + fr.insee + d1fd0fe7-458d-43f0-b336-3fd24e12cb70 + 1 + Variable + + + fr.insee + 22d6b5f9-ded9-4da1-a4bf-38c4e6ea6f3f + 1 + Variable + + + fr.insee + 397b2d4d-19e7-4e79-bad8-7c229988e887 + 1 + Variable + + + fr.insee + a36675ec-ba11-4425-b74e-172fed36be6b + 1 + Variable + + + fr.insee + d2ce4d15-6e37-4ea6-a180-01db98beb530 + 1 + Variable + + + fr.insee + 4f2e87c9-c9b5-4637-95d3-94641b141e48 + 1 + Variable + + + fr.insee + 7991a4e1-bfe1-4c83-a6d9-57dd6e455ab3 + 1 + Variable + + + fr.insee + d8f3410e-e828-4454-9cbd-084b0f2b30e6 + 1 + Variable + + + fr.insee + a5d806bc-a8ee-4d06-a38e-db1ffb603d2c + 1 + Variable + + + fr.insee + f5d573dd-00c0-4d05-a7ed-45d6e4beb41e + 1 + Variable + + + fr.insee + 22d270e6-eb42-4ddf-b0de-55e2652a7de6 + 1 + Variable + + + fr.insee + 63274f17-4fce-4b3d-b065-64e521831b14 + 1 + Variable + + + fr.insee + 649d1170-d501-4e8a-a95c-c37af565bb1f + 1 + Variable + + + fr.insee + 2b168ae2-3391-4688-b1f6-510bb2ce1a38 + 1 + Variable + + + fr.insee + f2952d87-a888-434e-adb1-0ca8fe145b6b + 1 + Variable + + + fr.insee + 44b36f21-f401-4b35-a537-cadc1b526ffe + 1 + Variable + + + fr.insee + 630c67cc-4f16-4148-bb26-94b87d84fa48 + 1 + Variable + + + fr.insee + feba9ab5-8938-4b8b-aba8-ca689e78949f + 1 + Variable + + + fr.insee + 8b357a51-6c7d-4d22-badc-8dd7198f0bbc + 1 + Variable + + + fr.insee + b966cdc3-d1b8-40ab-8e76-3b8d335f0d73 + 1 + Variable + + + fr.insee + 4bc65274-25e0-4a9f-824f-c86762788614 + 1 + Variable + + + fr.insee + 59ccfb28-6023-4384-8e1f-056c56922c47 + 1 + Variable + + + fr.insee + b8f75a18-0856-424e-a6c1-6bafc9ddbf34 + 1 + Variable + + + fr.insee + efe9b759-5926-423a-823d-4a5b9bf84736 + 1 + Variable + + + fr.insee + c4993982-f996-4e3e-8565-3e1f3a79fad8 + 1 + Variable + + + fr.insee + 4a08bed8-1dde-4957-8014-788c1a6436bb + 1 + Variable + + + fr.insee + b6d2773c-67b0-435e-9ffe-13b3c083548a + 1 + Variable + + + fr.insee + f49137c2-8cee-45c2-b3bd-8d3da380dd47 + 1 + Variable + + + fr.insee + 509c63a6-02ad-49f6-bf27-a1b011662d08 + 1 + Variable + + + fr.insee + 6eef0c5d-5ad6-4ac6-a8b9-ddd8c7b082de + 1 + Variable + + + fr.insee + a49541ee-eea0-4f3f-af3d-36d24e90b81f + 1 + Variable + + + fr.insee + c2e92b35-4372-46c6-9931-453c240aeada + 1 + Variable + + + fr.insee + 1d930e75-9aac-4938-8f13-0adbe490f2a3 + 1 + Variable + + + fr.insee + 12e924d4-4d9b-40a1-89af-d1b0520540de + 1 + Variable + + + fr.insee + 2ad09d76-ee1c-4280-9d69-059f3a0324b5 + 1 + Variable + + + fr.insee + d2a61c5b-6e39-471e-816c-33759dd71ae7 + 1 + Variable + + + fr.insee + f68b0a26-6a19-4bc1-b50f-8abad2c5e81b + 1 + Variable + + + fr.insee + 9343c28a-3382-4a68-b3a5-78bb9b460062 + 1 + Variable + + + fr.insee + af48a15d-baf1-4ff2-aee5-ae0602b355ce + 1 + Variable + + + fr.insee + b261c008-ac55-4dd4-8be9-2c3e96c9e02f + 1 + Variable + + + fr.insee + f6895b5d-febe-4a7f-b745-03c2b00ac593 + 1 + Variable + + + + + + + + urn:ddi:fr.insee:8ec8f1bc-ff67-45ee-a78b-c31951a78cb3:1 + fr.insee + 8ec8f1bc-ff67-45ee-a78b-c31951a78cb3 + 1 + + noindiv + + + Identifiant individu + + + + + + + + + urn:ddi:fr.insee:6758fc71-dc86-4edb-8c14-0f14362b3dee:1 + fr.insee + 6758fc71-dc86-4edb-8c14-0f14362b3dee + 1 + + ident17 + + + Identifiant du ménage + + + + + + + + + urn:ddi:fr.insee:4d63151e-3bb5-444b-b3f1-9771507e0ea0:1 + fr.insee + 4d63151e-3bb5-444b-b3f1-9771507e0ea0 + 1 + + noi + + + Numéro individuel Enquête Emploi + + + + + + + + + urn:ddi:fr.insee:ab13bec9-0495-426d-a582-4b087a7b6ea5:1 + fr.insee + ab13bec9-0495-426d-a582-4b087a7b6ea5 + 1 + + cs8cor + + + CS en 8 postes avec correction ERFS + + + + + fr.insee + 3d8cd74a-3678-4dc9-8560-eca56a43f1a4 + 1 + CodeList + + + + + + + + urn:ddi:fr.insee:3d8cd74a-3678-4dc9-8560-eca56a43f1a4:1 + fr.insee + 3d8cd74a-3678-4dc9-8560-eca56a43f1a4 + 1 + + urn:ddi:fr.insee:8da0c54b-cc8f-428e-86f6-da91f22d0bb4:1 + fr.insee + 8da0c54b-cc8f-428e-86f6-da91f22d0bb4 + 1 + + fr.insee + 72e8bc4d-2058-4ce0-8940-829c073a1a17 + 1 + Category + + 1 + + + urn:ddi:fr.insee:373c0b2f-2857-4f97-9ce9-9be5cfe5b6ca:1 + fr.insee + 373c0b2f-2857-4f97-9ce9-9be5cfe5b6ca + 1 + + fr.insee + 55850b76-2120-4020-a3c6-2c0c6fd35cc4 + 1 + Category + + 2 + + + urn:ddi:fr.insee:c3c04cd1-65f2-4ad0-aad1-6a12ee909027:1 + fr.insee + c3c04cd1-65f2-4ad0-aad1-6a12ee909027 + 1 + + fr.insee + 94c8f4bb-59fe-436f-9995-38ed3e56f6e7 + 1 + Category + + 3 + + + urn:ddi:fr.insee:9031e2a0-cb12-4d56-a3bf-682000af07e7:1 + fr.insee + 9031e2a0-cb12-4d56-a3bf-682000af07e7 + 1 + + fr.insee + 7485f4bf-2c8b-4b90-9304-aac4aa358253 + 1 + Category + + 4 + + + urn:ddi:fr.insee:17308f0e-91e6-4733-bd8f-7913b20ea41e:1 + fr.insee + 17308f0e-91e6-4733-bd8f-7913b20ea41e + 1 + + fr.insee + 9f1863b2-2d23-4f43-92d2-35d361aab0f2 + 1 + Category + + 5 + + + urn:ddi:fr.insee:3ef063c7-637f-4208-a908-7b2b5177b40e:1 + fr.insee + 3ef063c7-637f-4208-a908-7b2b5177b40e + 1 + + fr.insee + b3d92835-54ab-42dd-a9c3-a69ccf43b534 + 1 + Category + + 6 + + + urn:ddi:fr.insee:b49a1df5-7ab5-4341-8070-a05e4f4581df:1 + fr.insee + b49a1df5-7ab5-4341-8070-a05e4f4581df + 1 + + fr.insee + 9c1e080c-3d61-4d01-b638-5de22f33cb46 + 1 + Category + + 7 + + + urn:ddi:fr.insee:d45ffee7-9972-4fb8-8baf-c4afed10d41c:1 + fr.insee + d45ffee7-9972-4fb8-8baf-c4afed10d41c + 1 + + fr.insee + 84b02f7e-3f37-4fb0-88f9-6245260c8490 + 1 + Category + + 8 + + + + + + urn:ddi:fr.insee:72e8bc4d-2058-4ce0-8940-829c073a1a17:1 + fr.insee + 72e8bc4d-2058-4ce0-8940-829c073a1a17 + 1 + + Agriculteurs exploitants + + + + + + urn:ddi:fr.insee:55850b76-2120-4020-a3c6-2c0c6fd35cc4:1 + fr.insee + 55850b76-2120-4020-a3c6-2c0c6fd35cc4 + 1 + + Artisans, commerçants et chefs d'entreprise + + + + + + urn:ddi:fr.insee:94c8f4bb-59fe-436f-9995-38ed3e56f6e7:1 + fr.insee + 94c8f4bb-59fe-436f-9995-38ed3e56f6e7 + 1 + + Cadres et professions intellectuelles supérieures + + + + + + urn:ddi:fr.insee:7485f4bf-2c8b-4b90-9304-aac4aa358253:1 + fr.insee + 7485f4bf-2c8b-4b90-9304-aac4aa358253 + 1 + + Professions intermédiaires + + + + + + urn:ddi:fr.insee:9f1863b2-2d23-4f43-92d2-35d361aab0f2:1 + fr.insee + 9f1863b2-2d23-4f43-92d2-35d361aab0f2 + 1 + + Employés + + + + + + urn:ddi:fr.insee:b3d92835-54ab-42dd-a9c3-a69ccf43b534:1 + fr.insee + b3d92835-54ab-42dd-a9c3-a69ccf43b534 + 1 + + Ouvriers + + + + + + urn:ddi:fr.insee:9c1e080c-3d61-4d01-b638-5de22f33cb46:1 + fr.insee + 9c1e080c-3d61-4d01-b638-5de22f33cb46 + 1 + + Retraités + + + + + + urn:ddi:fr.insee:84b02f7e-3f37-4fb0-88f9-6245260c8490:1 + fr.insee + 84b02f7e-3f37-4fb0-88f9-6245260c8490 + 1 + + Autres inactifs + + + + + + urn:ddi:fr.insee:0369bc65-682c-4b63-9d09-9b339ec49dd2:1 + fr.insee + 0369bc65-682c-4b63-9d09-9b339ec49dd2 + 1 + + quelfic + + + Origine de l'observation + + + + + fr.insee + 851c8c35-c7e6-4195-932d-232d6f81c5af + 1 + CodeList + + + + + + + + urn:ddi:fr.insee:851c8c35-c7e6-4195-932d-232d6f81c5af:1 + fr.insee + 851c8c35-c7e6-4195-932d-232d6f81c5af + 1 + + urn:ddi:fr.insee:c1c60d18-694a-4d01-8c80-aa88e225a8d2:1 + fr.insee + c1c60d18-694a-4d01-8c80-aa88e225a8d2 + 1 + + fr.insee + e57dcdf4-f167-4e14-85a9-bd53b47413fe + 1 + Category + + EE&FIP + + + urn:ddi:fr.insee:b41b0519-1a8b-47e5-b731-b10d31903fc1:1 + fr.insee + b41b0519-1a8b-47e5-b731-b10d31903fc1 + 1 + + fr.insee + a4a679f9-eebb-493b-9b18-6c9347b94255 + 1 + Category + + EE_CAF + + + urn:ddi:fr.insee:36fd6fa8-8904-4f8f-bac5-38d5c9d82ab7:1 + fr.insee + 36fd6fa8-8904-4f8f-bac5-38d5c9d82ab7 + 1 + + fr.insee + 20dd50ca-cf11-4588-a797-a673f344f41d + 1 + Category + + EE + + + urn:ddi:fr.insee:e5cae3eb-f6bb-4276-b4f1-e9d94bc2c570:1 + fr.insee + e5cae3eb-f6bb-4276-b4f1-e9d94bc2c570 + 1 + + fr.insee + aa025ea9-da94-4e7f-960b-0e7a62f6828f + 1 + Category + + EE_NRT + + + + + + urn:ddi:fr.insee:e57dcdf4-f167-4e14-85a9-bd53b47413fe:1 + fr.insee + e57dcdf4-f167-4e14-85a9-bd53b47413fe + 1 + + L’individu a été enquêté à l’enquête Emploi, et au moins une + déclaration fiscale le concernant a été retrouvée + + + + + + urn:ddi:fr.insee:a4a679f9-eebb-493b-9b18-6c9347b94255:1 + fr.insee + a4a679f9-eebb-493b-9b18-6c9347b94255 + 1 + + L’individu d’un ménage considéré comme répondant ERFS ne se + retrouve dans aucune déclaration fiscale, mais ses revenus ont pu être estimés à partir + d’un appariement réussi avec le fichier CNAF + + + + + + urn:ddi:fr.insee:20dd50ca-cf11-4588-a797-a673f344f41d:1 + fr.insee + 20dd50ca-cf11-4588-a797-a673f344f41d + 1 + + L'individu d’un ménage considéré comme répondant ERFS ne se + retrouve dans aucune des déclarations fiscales associées au ménage, ses revenus + individuels seront donc entièrement imputés + + + + + + urn:ddi:fr.insee:aa025ea9-da94-4e7f-960b-0e7a62f6828f:1 + fr.insee + aa025ea9-da94-4e7f-960b-0e7a62f6828f + 1 + + Etudiants de moins de 26 ans formant un ménage à part dans + l’Enquête Emploi et dont on n'a retrouvé aucune trace dans les fichiers fiscaux + (non-réponse totale), vraisemblablement parce qu’ils étaient rattachés au foyer fiscal de + leur(s) parent(s) + + + + + + urn:ddi:fr.insee:5d269749-dd06-4af9-a91e-8972a2b2ba82:1 + fr.insee + 5d269749-dd06-4af9-a91e-8972a2b2ba82 + 1 + + persfip + + + Identifiant individuel (2042) : indique si l'individu est le déclarant ou quel est son lien avec le déclarant + + + + + fr.insee + d4579b04-e17a-4477-a6c3-44d7ec9aeacf + 1 + CodeList + + + + + + + + urn:ddi:fr.insee:d4579b04-e17a-4477-a6c3-44d7ec9aeacf:1 + fr.insee + d4579b04-e17a-4477-a6c3-44d7ec9aeacf + 1 + + persfip (ERFS) + + + urn:ddi:fr.insee:fa703500-ef18-49c3-8ca8-7d6e6278beaa:1 + fr.insee + fa703500-ef18-49c3-8ca8-7d6e6278beaa + 1 + + fr.insee + 6a5f9cd9-b228-441b-98be-6c52c6cbd250 + 1 + Category + + vous + + + urn:ddi:fr.insee:26006d20-7bdb-4dc6-a0b5-546569e69e41:1 + fr.insee + 26006d20-7bdb-4dc6-a0b5-546569e69e41 + 1 + + fr.insee + 9a03c0d0-5c68-4a4d-b922-a83e4b775696 + 1 + Category + + conj + + + urn:ddi:fr.insee:6aed20c9-b81e-4552-8d66-bfdc47266e15:1 + fr.insee + 6aed20c9-b81e-4552-8d66-bfdc47266e15 + 1 + + fr.insee + 84bd612d-00f4-42bd-86d7-dc25b4dbfb9d + 1 + Category + + pac + + + + + + urn:ddi:fr.insee:6a5f9cd9-b228-441b-98be-6c52c6cbd250:1 + fr.insee + 6a5f9cd9-b228-441b-98be-6c52c6cbd250 + 1 + + L'individu est le déclarant + + + + + + urn:ddi:fr.insee:9a03c0d0-5c68-4a4d-b922-a83e4b775696:1 + fr.insee + 9a03c0d0-5c68-4a4d-b922-a83e4b775696 + 1 + + Conjoint + + + + + + urn:ddi:fr.insee:84bd612d-00f4-42bd-86d7-dc25b4dbfb9d:1 + fr.insee + 84bd612d-00f4-42bd-86d7-dc25b4dbfb9d + 1 + + Personne à charge + + + + + + urn:ddi:fr.insee:8abdcea6-ce62-46a4-b0d8-31a7d9bde324:1 + fr.insee + 8abdcea6-ce62-46a4-b0d8-31a7d9bde324 + 1 + + persfipd + + + Identifiant individuel détaillé (2042) : indique où rechercher les revenus de l’individu sur la déclaration fiscale + + + + + fr.insee + 89f93f10-55c7-49d3-994f-8667f0249427 + 1 + CodeList + + + + + + + + urn:ddi:fr.insee:89f93f10-55c7-49d3-994f-8667f0249427:1 + fr.insee + 89f93f10-55c7-49d3-994f-8667f0249427 + 1 + + persfipd (ERFS) + + + urn:ddi:fr.insee:80864bb5-9f16-405d-8abf-ed7dcde0ab2f:1 + fr.insee + 80864bb5-9f16-405d-8abf-ed7dcde0ab2f + 1 + + fr.insee + 00e3c1e5-7174-48c2-9675-831e708fbf2c + 1 + Category + + mad + + + urn:ddi:fr.insee:be7755e2-c4cb-4332-964e-8c5f61de97d0:1 + fr.insee + be7755e2-c4cb-4332-964e-8c5f61de97d0 + 1 + + fr.insee + 5e3bcebb-c435-447f-9949-734e490114d0 + 1 + Category + + mon + + + urn:ddi:fr.insee:223774f1-2aca-48b0-a994-a9b9704be748:1 + fr.insee + 223774f1-2aca-48b0-a994-a9b9704be748 + 1 + + fr.insee + 6157479b-49bd-4290-afff-03e930eba9d1 + 1 + Category + + vopa + + + + + + urn:ddi:fr.insee:00e3c1e5-7174-48c2-9675-831e708fbf2c:1 + fr.insee + 00e3c1e5-7174-48c2-9675-831e708fbf2c + 1 + + Madame + + + + + + urn:ddi:fr.insee:5e3bcebb-c435-447f-9949-734e490114d0:1 + fr.insee + 5e3bcebb-c435-447f-9949-734e490114d0 + 1 + + Monsieur + + + + + + urn:ddi:fr.insee:6157479b-49bd-4290-afff-03e930eba9d1:1 + fr.insee + 6157479b-49bd-4290-afff-03e930eba9d1 + 1 + + Individus atteignant l'âge de 18 ans l’année de référence, + pouvant avoir une double déclaration (seule et sur celle de leurs parents) + + + + + + urn:ddi:fr.insee:f76704e8-1df8-45fb-9c0c-3dac8b29b372:1 + fr.insee + f76704e8-1df8-45fb-9c0c-3dac8b29b372 + 1 + + sas:StorageWidth + 8 + + + zsali + + + Salaires au sens strict (individu) + + + + Double + + + + + + + urn:ddi:fr.insee:63056208-ea5d-40b4-afba-4e4439f13b9f:1 + fr.insee + 63056208-ea5d-40b4-afba-4e4439f13b9f + 1 + + sas:StorageWidth + 8 + + + zchoi + + + Allocation chômage et préretraite (individu) + + + + Double + + + + + + + urn:ddi:fr.insee:19e06933-0e02-44e8-9a9b-875c1b7581b2:1 + fr.insee + 19e06933-0e02-44e8-9a9b-875c1b7581b2 + 1 + + sas:StorageWidth + 8 + + + ztsai + + + Traitements et salaires (individu) + + + + Double + + + + + + + urn:ddi:fr.insee:3783b748-2f6e-4143-88a9-5d454f432d9a:1 + fr.insee + 3783b748-2f6e-4143-88a9-5d454f432d9a + 1 + + sas:StorageWidth + 8 + + + zrsti + + + Retraites au sens strict (individu) + + + + Double + + + + + + + urn:ddi:fr.insee:87e85c5d-6380-4423-be51-b4c9f3736e55:1 + fr.insee + 87e85c5d-6380-4423-be51-b4c9f3736e55 + 1 + + sas:StorageWidth + 8 + + + zpii + + + Pensions d'invalidité (individu) + + + + Double + + + + + + + urn:ddi:fr.insee:00b3f378-535f-4770-961d-4822aebf36b0:1 + fr.insee + 00b3f378-535f-4770-961d-4822aebf36b0 + 1 + + sas:StorageWidth + 8 + + + zalri + + + Pensions alimentaires reçues (individu) + + + + Double + + + + + + + urn:ddi:fr.insee:42ae6bd5-804a-4182-b68e-70ff83193ba7:1 + fr.insee + 42ae6bd5-804a-4182-b68e-70ff83193ba7 + 1 + + sas:StorageWidth + 8 + + + zreti + + + Retraites et pensions (individu) + + + + Double + + + + + + + urn:ddi:fr.insee:96d279f5-cef3-40e6-b519-b9dc7bfd42f7:1 + fr.insee + 96d279f5-cef3-40e6-b519-b9dc7bfd42f7 + 1 + + sas:StorageWidth + 8 + + + zrtoi + + + Rentes viagères à titre onéreux (individu) + + + + Double + + + + + + + urn:ddi:fr.insee:b399dde0-790b-4a57-b530-354664ba7ba1:1 + fr.insee + b399dde0-790b-4a57-b530-354664ba7ba1 + 1 + + sas:StorageWidth + 8 + + + zperi + + + Pensions, retraites, rentes (individu) + + + + Double + + + + + + + urn:ddi:fr.insee:f08de465-fb1b-42c3-be4f-34e0390e216e:1 + fr.insee + f08de465-fb1b-42c3-be4f-34e0390e216e + 1 + + sas:StorageWidth + 8 + + + zragi + + + Revenus agricoles (individu) + + + + Double + + + + + + + urn:ddi:fr.insee:dd2b743d-82a6-450f-8274-e5dbccc02946:1 + fr.insee + dd2b743d-82a6-450f-8274-e5dbccc02946 + 1 + + sas:StorageWidth + 8 + + + zrici + + + Revenus industriels et commerciaux (individu) + + + + Double + + + + + + + urn:ddi:fr.insee:d1fd0fe7-458d-43f0-b336-3fd24e12cb70:1 + fr.insee + d1fd0fe7-458d-43f0-b336-3fd24e12cb70 + 1 + + sas:StorageWidth + 8 + + + zrnci + + + Revenus non commerciaux (individu) + + + + Double + + + + + + + urn:ddi:fr.insee:22d6b5f9-ded9-4da1-a4bf-38c4e6ea6f3f:1 + fr.insee + 22d6b5f9-ded9-4da1-a4bf-38c4e6ea6f3f + 1 + + sas:StorageWidth + 8 + + + zsalo + + + Salaires au sens strict (origine) + + + + Double + + + + + + + urn:ddi:fr.insee:397b2d4d-19e7-4e79-bad8-7c229988e887:1 + fr.insee + 397b2d4d-19e7-4e79-bad8-7c229988e887 + 1 + + sas:StorageWidth + 8 + + + zchoo + + + Allocation chômage et préretraite (origine) + + + + Double + + + + + + + urn:ddi:fr.insee:a36675ec-ba11-4425-b74e-172fed36be6b:1 + fr.insee + a36675ec-ba11-4425-b74e-172fed36be6b + 1 + + sas:StorageWidth + 8 + + + ztsao + + + Traitements et salaires (origine) + + + + Double + + + + + + + urn:ddi:fr.insee:d2ce4d15-6e37-4ea6-a180-01db98beb530:1 + fr.insee + d2ce4d15-6e37-4ea6-a180-01db98beb530 + 1 + + sas:StorageWidth + 8 + + + zrsto + + + Retraites au sens strict (origine) + + + + Double + + + + + + + urn:ddi:fr.insee:4f2e87c9-c9b5-4637-95d3-94641b141e48:1 + fr.insee + 4f2e87c9-c9b5-4637-95d3-94641b141e48 + 1 + + sas:StorageWidth + 8 + + + zpio + + + Pensions d'invalidité (origine) + + + + Double + + + + + + + urn:ddi:fr.insee:7991a4e1-bfe1-4c83-a6d9-57dd6e455ab3:1 + fr.insee + 7991a4e1-bfe1-4c83-a6d9-57dd6e455ab3 + 1 + + sas:StorageWidth + 8 + + + zalro + + + Pensions alimentaires reçues (origine) + + + + Double + + + + + + + urn:ddi:fr.insee:d8f3410e-e828-4454-9cbd-084b0f2b30e6:1 + fr.insee + d8f3410e-e828-4454-9cbd-084b0f2b30e6 + 1 + + sas:StorageWidth + 8 + + + zreto + + + Retraites et pensions (origine) + + + + Double + + + + + + + urn:ddi:fr.insee:a5d806bc-a8ee-4d06-a38e-db1ffb603d2c:1 + fr.insee + a5d806bc-a8ee-4d06-a38e-db1ffb603d2c + 1 + + sas:StorageWidth + 8 + + + zrtoo + + + Rentes viagères à titre onéreux (origine) + + + + Double + + + + + + + urn:ddi:fr.insee:f5d573dd-00c0-4d05-a7ed-45d6e4beb41e:1 + fr.insee + f5d573dd-00c0-4d05-a7ed-45d6e4beb41e + 1 + + sas:StorageWidth + 8 + + + zpero + + + Pensions, retraites, rentes (origine) + + + + Double + + + + + + + urn:ddi:fr.insee:22d270e6-eb42-4ddf-b0de-55e2652a7de6:1 + fr.insee + 22d270e6-eb42-4ddf-b0de-55e2652a7de6 + 1 + + sas:StorageWidth + 8 + + + zrago + + + Revenus agricoles (origine) + + + + Double + + + + + + + urn:ddi:fr.insee:63274f17-4fce-4b3d-b065-64e521831b14:1 + fr.insee + 63274f17-4fce-4b3d-b065-64e521831b14 + 1 + + sas:StorageWidth + 8 + + + zrico + + + Revenus industriels et commerciaux (origine) + + + + Double + + + + + + + urn:ddi:fr.insee:649d1170-d501-4e8a-a95c-c37af565bb1f:1 + fr.insee + 649d1170-d501-4e8a-a95c-c37af565bb1f + 1 + + sas:StorageWidth + 8 + + + zrnco + + + Revenus non commerciaux (origine) + + + + Double + + + + + + + urn:ddi:fr.insee:2b168ae2-3391-4688-b1f6-510bb2ce1a38:1 + fr.insee + 2b168ae2-3391-4688-b1f6-510bb2ce1a38 + 1 + + declar1 + + + Description résumée : déclaration n°1 + + + + + + + + + urn:ddi:fr.insee:f2952d87-a888-434e-adb1-0ca8fe145b6b:1 + fr.insee + f2952d87-a888-434e-adb1-0ca8fe145b6b + 1 + + declar2 + + + Description résumée : déclaration n°2 + + + + + + + + + urn:ddi:fr.insee:44b36f21-f401-4b35-a537-cadc1b526ffe:1 + fr.insee + 44b36f21-f401-4b35-a537-cadc1b526ffe + 1 + + sas:StorageWidth + 8 + + + csgsald + + + CSG déductible sur salaires + + + + Double + + + + + + + urn:ddi:fr.insee:630c67cc-4f16-4148-bb26-94b87d84fa48:1 + fr.insee + 630c67cc-4f16-4148-bb26-94b87d84fa48 + 1 + + sas:StorageWidth + 8 + + + csgsali + + + CSG imposable sur salaires + + + + Double + + + + + + + urn:ddi:fr.insee:feba9ab5-8938-4b8b-aba8-ca689e78949f:1 + fr.insee + feba9ab5-8938-4b8b-aba8-ca689e78949f + 1 + + sas:StorageWidth + 8 + + + crdssal + + + CRDS sur salaires + + + + Double + + + + + + + urn:ddi:fr.insee:8b357a51-6c7d-4d22-badc-8dd7198f0bbc:1 + fr.insee + 8b357a51-6c7d-4d22-badc-8dd7198f0bbc + 1 + + sas:StorageWidth + 8 + + + csgchod + + + CSG déductible sur allocations chômage + + + + Double + + + + + + + urn:ddi:fr.insee:b966cdc3-d1b8-40ab-8e76-3b8d335f0d73:1 + fr.insee + b966cdc3-d1b8-40ab-8e76-3b8d335f0d73 + 1 + + sas:StorageWidth + 8 + + + csgchoi + + + CSG imposable sur allocations chômage + + + + Double + + + + + + + urn:ddi:fr.insee:4bc65274-25e0-4a9f-824f-c86762788614:1 + fr.insee + 4bc65274-25e0-4a9f-824f-c86762788614 + 1 + + sas:StorageWidth + 8 + + + crdscho + + + CRDS sur allocations chômage + + + + Double + + + + + + + urn:ddi:fr.insee:59ccfb28-6023-4384-8e1f-056c56922c47:1 + fr.insee + 59ccfb28-6023-4384-8e1f-056c56922c47 + 1 + + sas:StorageWidth + 8 + + + csgrstd + + + CSG déductible sur retraites + + + + Double + + + + + + + urn:ddi:fr.insee:b8f75a18-0856-424e-a6c1-6bafc9ddbf34:1 + fr.insee + b8f75a18-0856-424e-a6c1-6bafc9ddbf34 + 1 + + sas:StorageWidth + 8 + + + csgrsti + + + CSG imposable sur retraites + + + + Double + + + + + + + urn:ddi:fr.insee:efe9b759-5926-423a-823d-4a5b9bf84736:1 + fr.insee + efe9b759-5926-423a-823d-4a5b9bf84736 + 1 + + sas:StorageWidth + 8 + + + crdsrst + + + CRDS sur retraites + + + + Double + + + + + + + urn:ddi:fr.insee:c4993982-f996-4e3e-8565-3e1f3a79fad8:1 + fr.insee + c4993982-f996-4e3e-8565-3e1f3a79fad8 + 1 + + sas:StorageWidth + 8 + + + csgpid + + + CSG imposable sur pensions d'invalidité + + + + Double + + + + + + + urn:ddi:fr.insee:4a08bed8-1dde-4957-8014-788c1a6436bb:1 + fr.insee + 4a08bed8-1dde-4957-8014-788c1a6436bb + 1 + + sas:StorageWidth + 8 + + + csgpii + + + CSG déductible sur pensions d'invalidité + + + + Double + + + + + + + urn:ddi:fr.insee:b6d2773c-67b0-435e-9ffe-13b3c083548a:1 + fr.insee + b6d2773c-67b0-435e-9ffe-13b3c083548a + 1 + + sas:StorageWidth + 8 + + + crdspi + + + CRDS sur pensions d'invalidité + + + + Double + + + + + + + urn:ddi:fr.insee:f49137c2-8cee-45c2-b3bd-8d3da380dd47:1 + fr.insee + f49137c2-8cee-45c2-b3bd-8d3da380dd47 + 1 + + sas:StorageWidth + 8 + + + csgragd + + + CSG déductible sur RAG + + + + Double + + + + + + + urn:ddi:fr.insee:509c63a6-02ad-49f6-bf27-a1b011662d08:1 + fr.insee + 509c63a6-02ad-49f6-bf27-a1b011662d08 + 1 + + sas:StorageWidth + 8 + + + csgragi + + + CSG imposable sur RAG + + + + Double + + + + + + + urn:ddi:fr.insee:6eef0c5d-5ad6-4ac6-a8b9-ddd8c7b082de:1 + fr.insee + 6eef0c5d-5ad6-4ac6-a8b9-ddd8c7b082de + 1 + + sas:StorageWidth + 8 + + + crdsrag + + + CRDS sur RAG + + + + Double + + + + + + + urn:ddi:fr.insee:a49541ee-eea0-4f3f-af3d-36d24e90b81f:1 + fr.insee + a49541ee-eea0-4f3f-af3d-36d24e90b81f + 1 + + sas:StorageWidth + 8 + + + csgricd + + + CSG déductible sur RIC + + + + Double + + + + + + + urn:ddi:fr.insee:c2e92b35-4372-46c6-9931-453c240aeada:1 + fr.insee + c2e92b35-4372-46c6-9931-453c240aeada + 1 + + sas:StorageWidth + 8 + + + csgrici + + + CSG imposable sur RIC + + + + Double + + + + + + + urn:ddi:fr.insee:1d930e75-9aac-4938-8f13-0adbe490f2a3:1 + fr.insee + 1d930e75-9aac-4938-8f13-0adbe490f2a3 + 1 + + sas:StorageWidth + 8 + + + crdsric + + + CRDS sur RIC + + + + Double + + + + + + + urn:ddi:fr.insee:12e924d4-4d9b-40a1-89af-d1b0520540de:1 + fr.insee + 12e924d4-4d9b-40a1-89af-d1b0520540de + 1 + + sas:StorageWidth + 8 + + + csgrncd + + + CSG déductible sur RNC + + + + Double + + + + + + + urn:ddi:fr.insee:2ad09d76-ee1c-4280-9d69-059f3a0324b5:1 + fr.insee + 2ad09d76-ee1c-4280-9d69-059f3a0324b5 + 1 + + sas:StorageWidth + 8 + + + csgrnci + + + CSG imposable sur RNC + + + + Double + + + + + + + urn:ddi:fr.insee:d2a61c5b-6e39-471e-816c-33759dd71ae7:1 + fr.insee + d2a61c5b-6e39-471e-816c-33759dd71ae7 + 1 + + sas:StorageWidth + 8 + + + crdsrnc + + + CRDS sur RNC + + + + Double + + + + + + + urn:ddi:fr.insee:f68b0a26-6a19-4bc1-b50f-8abad2c5e81b:1 + fr.insee + f68b0a26-6a19-4bc1-b50f-8abad2c5e81b + 1 + + sas:StorageWidth + 8 + + + wprm + + + Poids final ERFS + + + + Double + + + + + + + urn:ddi:fr.insee:9343c28a-3382-4a68-b3a5-78bb9b460062:1 + fr.insee + 9343c28a-3382-4a68-b3a5-78bb9b460062 + 1 + + logt + + + Statut d'occupation corrigé + + + + + + + + + urn:ddi:fr.insee:af48a15d-baf1-4ff2-aee5-ae0602b355ce:1 + fr.insee + af48a15d-baf1-4ff2-aee5-ae0602b355ce + 1 + + sas:StorageWidth + 8 + + + nb_UCI + + + Nombre d'Unités de Consommation du ménage + + + + Double + + + + + + + urn:ddi:fr.insee:b261c008-ac55-4dd4-8be9-2c3e96c9e02f:1 + fr.insee + b261c008-ac55-4dd4-8be9-2c3e96c9e02f + 1 + + sas:StorageWidth + 8 + + + m_minvm + + + Minimum vieillesse du ménage + + + + Double + + + + + + + urn:ddi:fr.insee:f6895b5d-febe-4a7f-b745-03c2b00ac593:1 + fr.insee + f6895b5d-febe-4a7f-b745-03c2b00ac593 + 1 + + acteu5 + + + Activité au sens BIT recalculée (5 postes) + + + + + fr.insee + f66eab41-a868-4ce7-904c-2c19d4aebb7d + 1 + CodeList + + + + + + + + urn:ddi:fr.insee:f66eab41-a868-4ce7-904c-2c19d4aebb7d:1 + fr.insee + f66eab41-a868-4ce7-904c-2c19d4aebb7d + 1 + + urn:ddi:fr.insee:997b7269-b5e9-4f97-846e-91964a58ba91:1 + fr.insee + 997b7269-b5e9-4f97-846e-91964a58ba91 + 1 + + fr.insee + 2c0b3d18-7923-403d-b250-eca26aa78181 + 1 + Category + + 1 + + + urn:ddi:fr.insee:3e298682-32ba-4337-b174-b86cecaff617:1 + fr.insee + 3e298682-32ba-4337-b174-b86cecaff617 + 1 + + fr.insee + 04e9cc7c-547b-4647-acce-728dbf1decd8 + 1 + Category + + 2 + + + urn:ddi:fr.insee:a58353a4-838f-4706-8710-434a2d7f4c27:1 + fr.insee + a58353a4-838f-4706-8710-434a2d7f4c27 + 1 + + fr.insee + 362d2a60-0776-4fc7-89fd-3629b5cd227e + 1 + Category + + 3 + + + urn:ddi:fr.insee:8eabfae1-1d81-4f9e-885c-12febdcbde9a:1 + fr.insee + 8eabfae1-1d81-4f9e-885c-12febdcbde9a + 1 + + fr.insee + 9e53152f-4fd3-43a3-bb93-1af2a8a394d4 + 1 + Category + + 4 + + + urn:ddi:fr.insee:c1c43297-6783-48cb-80ff-da49d9dffb3d:1 + fr.insee + c1c43297-6783-48cb-80ff-da49d9dffb3d + 1 + + fr.insee + 39baf1b6-089a-4c7d-b3b7-1045a07cd5f8 + 1 + Category + + 5 + + + + + + urn:ddi:fr.insee:2c0b3d18-7923-403d-b250-eca26aa78181:1 + fr.insee + 2c0b3d18-7923-403d-b250-eca26aa78181 + 1 + + Salarié + + + + + + urn:ddi:fr.insee:04e9cc7c-547b-4647-acce-728dbf1decd8:1 + fr.insee + 04e9cc7c-547b-4647-acce-728dbf1decd8 + 1 + + Indépendant + + + + + + urn:ddi:fr.insee:362d2a60-0776-4fc7-89fd-3629b5cd227e:1 + fr.insee + 362d2a60-0776-4fc7-89fd-3629b5cd227e + 1 + + Chômeur + + + + + + urn:ddi:fr.insee:9e53152f-4fd3-43a3-bb93-1af2a8a394d4:1 + fr.insee + 9e53152f-4fd3-43a3-bb93-1af2a8a394d4 + 1 + + Retraité + + + + + + urn:ddi:fr.insee:39baf1b6-089a-4c7d-b3b7-1045a07cd5f8:1 + fr.insee + 39baf1b6-089a-4c7d-b3b7-1045a07cd5f8 + 1 + + Autre inactif + + + + \ No newline at end of file From 42eee50773d05d9d2480a99e4a873c612f2447e6 Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Wed, 22 Jan 2025 14:24:10 +0100 Subject: [PATCH 11/25] fix (fail to process xml returned by colectica when there is a BOM) --- .../fragments/DdiFragmentServiceImpl.java | 8 +- .../service/ColecticaServiceImpl.java | 10 +-- .../java/fr/insee/rmes/utils/HttpUtils.java | 16 ++++ .../service/ColecticaServiceImplTest.java | 81 +++++++++++++++++++ .../fr/insee/rmes/utils/HttpUtilsTest.java | 23 ++++++ .../service/ColecticaServiceImplTest.java | 60 -------------- .../resources/utf8-bom/fichierAvecBom.xml | 2 + .../resources/utf8-bom/fichierSansBom.xml | 4 + 8 files changed, 138 insertions(+), 66 deletions(-) create mode 100644 src/test/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImplTest.java create mode 100644 src/test/java/fr/insee/rmes/utils/HttpUtilsTest.java delete mode 100644 src/test/java/service/ColecticaServiceImplTest.java create mode 100644 src/test/resources/utf8-bom/fichierAvecBom.xml create mode 100644 src/test/resources/utf8-bom/fichierSansBom.xml diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java b/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java index 198e1b84..81558e16 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java @@ -4,12 +4,15 @@ import fr.insee.rmes.exceptions.XsltTransformationException; import fr.insee.rmes.tocolecticaapi.service.ColecticaService; import fr.insee.rmes.transfoxsl.service.XsltTransformationService; +import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; +@Slf4j @Service public record DdiFragmentServiceImpl(XsltTransformationService xsltTransformationService, ColecticaService colecticaService) implements DdiFragmentService { @@ -18,11 +21,14 @@ public record DdiFragmentServiceImpl(XsltTransformationService xsltTransformatio @Override public String extractDataRelationship(String uuid) throws RmesException { try { + String ddiInstance = colecticaService.searchColecticaInstanceByUuid(uuid); + log.trace("Ddi Instance returned by colectica :\n{}", ddiInstance); return new String(xsltTransformationService.transformToRawText( - new ByteArrayInputStream(colecticaService.searchColecticaInstanceByUuid(uuid).getBytes()), + new ByteArrayInputStream(ddiInstance.getBytes()), ddiRelationshipToJsonXsl )); } catch (IOException | XsltTransformationException e) { + log.debug("Error while processing DDI Instance : "+e.getMessage(), e); throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Error while processing DDI to json", e.getMessage()); } } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java index b03b92e1..df2989b3 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java @@ -17,6 +17,7 @@ import fr.insee.rmes.tocolecticaapi.models.TransactionType; import fr.insee.rmes.transfoxsl.service.internal.DDIDerefencer; import fr.insee.rmes.utils.ExportUtils; +import fr.insee.rmes.utils.HttpUtils; import lombok.extern.slf4j.Slf4j; import org.json.JSONArray; import org.json.JSONObject; @@ -96,22 +97,21 @@ public String findFragmentByUuid(String uuid) { } - - private String getWithRestClient(URI relativeUri, MediaType acceptedMediaType) { + String getWithRestClient(URI relativeUri, MediaType acceptedMediaType) { RestClient.ResponseSpec response = restClient.get().uri(relativeUri) .accept(acceptedMediaType) .retrieve(); - return response + byte[] rawResponseContent = response .onStatus(HttpStatusCode::is4xxClientError, (request, httpResponse) -> { throw new RmesExceptionIO(httpResponse.getStatusCode().value(), "Bad request or inexisting resource", readBodySafely(httpResponse)); }) .onStatus(HttpStatusCode::is5xxServerError, (request, httpResponse) -> { throw new RmesExceptionIO(httpResponse.getStatusCode().value(), "Error while calling Colectica", readBodySafely(httpResponse)); }) - .body(String.class); + .body(byte[].class); + return HttpUtils.filterBOM(rawResponseContent); } - private String postWithRestClient(URI relativeUri, Optional requestBody, MediaType contentType) { RestClient.RequestBodySpec postRequest = restClient.post() .uri(relativeUri) diff --git a/src/main/java/fr/insee/rmes/utils/HttpUtils.java b/src/main/java/fr/insee/rmes/utils/HttpUtils.java index 2783eb25..a5cdb322 100644 --- a/src/main/java/fr/insee/rmes/utils/HttpUtils.java +++ b/src/main/java/fr/insee/rmes/utils/HttpUtils.java @@ -4,6 +4,8 @@ import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.List; public final class HttpUtils { @@ -34,4 +36,18 @@ public static HttpHeaders generateHttpHeaders(String fileName, FileExtension ext return responseHeaders; } + public static String filterBOM(byte[] bytes) { + if (hasBom(bytes)){ + return new String(Arrays.copyOfRange(bytes, 3, bytes.length), StandardCharsets.UTF_8); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + private static boolean hasBom(byte[] bytes) { + return bytes.length > 2 + && bytes[0] == (byte) 0xEF + && bytes[1] == (byte) 0xBB + && bytes[2] == (byte) 0xBF; + } + } diff --git a/src/test/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImplTest.java b/src/test/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImplTest.java new file mode 100644 index 00000000..fab12b6d --- /dev/null +++ b/src/test/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImplTest.java @@ -0,0 +1,81 @@ +package fr.insee.rmes.tocolecticaapi.service; + +import fr.insee.rmes.exceptions.RmesExceptionIO; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.web.client.MockServerRestClientCustomizer; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.test.web.client.match.MockRestRequestMatchers; +import org.springframework.test.web.client.response.MockRestResponseCreators; +import org.springframework.web.client.RestClient; +import org.xmlunit.builder.DiffBuilder; +import org.xmlunit.builder.Input; + +import java.io.IOException; +import java.net.URI; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +class ColecticaServiceImplTest { + + @Test + void getResponseEntity_NotOkStatus() { + var uuid="1"; + var agency="insee"; + + MockServerRestClientCustomizer customizer = new MockServerRestClientCustomizer(); + String baseUrl = "http://keycloak/"; + RestClient.Builder builder = RestClient.builder() + .baseUrl(baseUrl); + customizer.customize(builder); + customizer.getServer().expect(MockRestRequestMatchers.requestTo(baseUrl+"item/"+agency+"/"+ uuid)).andRespond(MockRestResponseCreators.withResourceNotFound()); + + ColecticaServiceImpl colecticaService = new ColecticaServiceImpl(null, builder.build(), null, null, agency); + var expectedException= Assertions.assertThrows(RmesExceptionIO.class, ()-> colecticaService.findFragmentByUuid(uuid)); + assertThat(expectedException.getStatus()).isEqualTo(HttpStatus.NOT_FOUND.value()); + assertThat(expectedException.getMessage()).isEqualTo("Bad request or inexisting resource"); + } + + @Test + void getResponseEntity_OkStatus() throws IOException { + String xmlContent = "Contenu"; + String jsonContent = "{\"Item\": \""+xmlContent+"\"}"; + var uuid="1"; + var agency="insee"; + String baseUrl = "http://keycloak/"; + + MockServerRestClientCustomizer customizer = new MockServerRestClientCustomizer(); + RestClient.Builder builder = RestClient.builder() + .baseUrl(baseUrl); + customizer.customize(builder); + customizer.getServer().expect(MockRestRequestMatchers.requestTo(baseUrl+"item/"+agency+"/"+ uuid)).andRespond(withSuccess(jsonContent, MediaType.APPLICATION_JSON)); + + ColecticaServiceImpl colecticaService = new ColecticaServiceImpl(null, builder.build(), null, null, agency); + + assertThat(colecticaService.findFragmentByUuid(uuid)).hasToString(xmlContent); + } + + @Test + void getHttpResponseWithBom_shouldNotReturnUglyChars() throws IOException { + String baseUrl = "http://collectica"; + byte[] xmlContentWithBom = ColecticaServiceImplTest.class.getResourceAsStream("/utf8-bom/fichierAvecBom.xml").readAllBytes(); + var expectedXml= new String(ColecticaServiceImplTest.class.getResourceAsStream("/utf8-bom/fichierSansBom.xml").readAllBytes()); + MockServerRestClientCustomizer customizer = new MockServerRestClientCustomizer(); + RestClient.Builder builder = RestClient.builder() + .baseUrl(baseUrl); + customizer.customize(builder); + customizer.getServer().expect(MockRestRequestMatchers.requestTo(baseUrl+"/ddi")).andRespond(withSuccess(xmlContentWithBom, MediaType.APPLICATION_XML)); + ColecticaServiceImpl colecticaService = new ColecticaServiceImpl(null, builder.build(), null, null, null); + var diff= DiffBuilder.compare(Input.fromString(colecticaService.getWithRestClient(URI.create("ddi"), MediaType.APPLICATION_XML))) + .withTest(Input.fromString(expectedXml)) + .ignoreWhitespace() + .build(); + assertFalse(diff.hasDifferences(), diff.toString()); + + } + + +} diff --git a/src/test/java/fr/insee/rmes/utils/HttpUtilsTest.java b/src/test/java/fr/insee/rmes/utils/HttpUtilsTest.java new file mode 100644 index 00000000..edde3c6b --- /dev/null +++ b/src/test/java/fr/insee/rmes/utils/HttpUtilsTest.java @@ -0,0 +1,23 @@ +package fr.insee.rmes.utils; + +import org.junit.jupiter.api.Test; +import org.xmlunit.builder.DiffBuilder; +import org.xmlunit.builder.Input; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.*; + +class HttpUtilsTest { + + @Test + void filterBOM() throws IOException { + var filteredContentWithBom=HttpUtils.filterBOM(HttpUtilsTest.class.getResourceAsStream("/utf8-bom/fichierAvecBom.xml").readAllBytes()); + var filteredContentWithoutBom=HttpUtils.filterBOM(HttpUtilsTest.class.getResourceAsStream("/utf8-bom/fichierSansBom.xml").readAllBytes()); + var diff=DiffBuilder.compare(Input.fromString(filteredContentWithBom)) + .withTest(Input.fromString(filteredContentWithoutBom)) + .ignoreWhitespace() + .build(); + assertFalse(diff.hasDifferences(), diff.toString()); + } +} \ No newline at end of file diff --git a/src/test/java/service/ColecticaServiceImplTest.java b/src/test/java/service/ColecticaServiceImplTest.java deleted file mode 100644 index 31bd404a..00000000 --- a/src/test/java/service/ColecticaServiceImplTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package service; - -import fr.insee.rmes.exceptions.RmesException; -import fr.insee.rmes.exceptions.RmesExceptionIO; -import fr.insee.rmes.tocolecticaapi.service.ColecticaServiceImpl; -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.web.client.MockServerRestClientCustomizer; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.web.client.RestClient; - -import java.io.IOException; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; -import static org.springframework.test.web.client.response.MockRestResponseCreators.withResourceNotFound; -import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; - -class ColecticaServiceImplTest { - - @Test - void getResponseEntity_NotOkStatus() { - var uuid="1"; - var agency="insee"; - - MockServerRestClientCustomizer customizer = new MockServerRestClientCustomizer(); - String baseUrl = "http://keycloak/"; - RestClient.Builder builder = RestClient.builder() - .baseUrl(baseUrl); - customizer.customize(builder); - customizer.getServer().expect(requestTo(baseUrl+"item/"+agency+"/"+ uuid)).andRespond(withResourceNotFound()); - - ColecticaServiceImpl colecticaService = new ColecticaServiceImpl(null, builder.build(), null, null, agency); - var expectedException=assertThrows(RmesExceptionIO.class, ()-> colecticaService.findFragmentByUuid(uuid)); - assertThat(expectedException.getStatus()).isEqualTo(HttpStatus.NOT_FOUND.value()); - assertThat(expectedException.getMessage()).isEqualTo("Bad request or inexisting resource"); - } - - @Test - void getResponseEntity_OkStatus() throws IOException { - String xmlContent = "Contenu"; - String jsonContent = "{\"Item\": \""+xmlContent+"\"}"; - var uuid="1"; - var agency="insee"; - String baseUrl = "http://keycloak/"; - - MockServerRestClientCustomizer customizer = new MockServerRestClientCustomizer(); - RestClient.Builder builder = RestClient.builder() - .baseUrl(baseUrl); - customizer.customize(builder); - customizer.getServer().expect(requestTo(baseUrl+"item/"+agency+"/"+ uuid)).andRespond(withSuccess(jsonContent, MediaType.APPLICATION_JSON)); - - ColecticaServiceImpl colecticaService = new ColecticaServiceImpl(null, builder.build(), null, null, agency); - - assertThat(colecticaService.findFragmentByUuid(uuid)).hasToString(xmlContent); - } - - -} diff --git a/src/test/resources/utf8-bom/fichierAvecBom.xml b/src/test/resources/utf8-bom/fichierAvecBom.xml new file mode 100644 index 00000000..8624bad1 --- /dev/null +++ b/src/test/resources/utf8-bom/fichierAvecBom.xml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/src/test/resources/utf8-bom/fichierSansBom.xml b/src/test/resources/utf8-bom/fichierSansBom.xml new file mode 100644 index 00000000..08d244a2 --- /dev/null +++ b/src/test/resources/utf8-bom/fichierSansBom.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file From a15232c852a899f22ccbf99c19d5065c372b45a2 Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Wed, 22 Jan 2025 16:40:38 +0100 Subject: [PATCH 12/25] fix (fail to refresh bearer token for colectica) --- pom.xml | 2 +- .../rmes/config/keycloak/KeycloakServices.java | 14 ++++++++++++-- .../service/ColecticaServiceImpl.java | 9 +++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 4827e1f9..5938785a 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ fr.insee.rmes ddi-access-services - 2.1.0 + 2.1.1 DDI Access Services Rest API to access to Colectica repository services diff --git a/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java b/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java index d669772a..502d36bb 100644 --- a/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java +++ b/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java @@ -12,6 +12,8 @@ import org.springframework.web.client.RestTemplate; import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; import java.time.temporal.ChronoUnit; import java.util.concurrent.atomic.AtomicReference; @@ -75,13 +77,21 @@ private boolean isCurrentTokenValid() { } public synchronized String getFreshToken() { + log.atTrace().log(() -> "Check if token is valid with expiration at " + expirationAsString()); if (!this.isCurrentTokenValid()) { - token = getKeycloakAccessToken(); - this.expiration.set(expirationFrom(token)); + log.debug("Start refreshing token"); + token = getKeycloakAccessToken(); + this.expiration.set(expirationFrom(token)); + log.atTrace().log(() -> "New token valid until " + expirationAsString()); } return token; } + private String expirationAsString() { + Instant expirationInstant = expiration.get(); + return expirationInstant == null ? null : LocalDateTime.ofInstant(expirationInstant, ZoneId.systemDefault()).toString(); + } + private Instant expirationFrom(String token) { return JWT.decode(token).getExpiresAtAsInstant(); } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java index df2989b3..54538c2f 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java @@ -86,7 +86,9 @@ private static RestClient initRestClient(KeycloakServices kc, String serviceUrl) .requestFactory(factory) .baseUrl(URI.create(serviceUrl + "/").resolve("api/v1/")) .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_XML_VALUE) - .defaultHeaders(headers -> headers.setBearerAuth(kc.getFreshToken())) + .defaultRequest(requestHeadersSpec -> requestHeadersSpec.header(HttpHeaders.AUTHORIZATION, + "Bearer " + kc.getFreshToken()) + ) .build(); } @@ -98,6 +100,7 @@ public String findFragmentByUuid(String uuid) { } String getWithRestClient(URI relativeUri, MediaType acceptedMediaType) { + log.debug("Start processing GET {} as {}" ,relativeUri, acceptedMediaType); RestClient.ResponseSpec response = restClient.get().uri(relativeUri) .accept(acceptedMediaType) .retrieve(); @@ -109,7 +112,9 @@ String getWithRestClient(URI relativeUri, MediaType acceptedMediaType) { throw new RmesExceptionIO(httpResponse.getStatusCode().value(), "Error while calling Colectica", readBodySafely(httpResponse)); }) .body(byte[].class); - return HttpUtils.filterBOM(rawResponseContent); + String filteredResponse = HttpUtils.filterBOM(rawResponseContent); + log.trace("Filtered response: {}", filteredResponse); + return filteredResponse; } private String postWithRestClient(URI relativeUri, Optional requestBody, MediaType contentType) { From 4197c07b5229e28e7d97b9e6b69cc38096abbe3e Mon Sep 17 00:00:00 2001 From: Fabrice Bibonne Date: Mon, 3 Feb 2025 09:32:43 +0100 Subject: [PATCH 13/25] fix (uuid example for GET /Item/ddiFragment/{uuid}/dataRelationship) #93 See https://github.com/InseeFr/DDI-Access-Services/issues/93#issuecomment-2612795232 --- .../java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java index e5d6ae0f..709e8da1 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/controller/GetItem.java @@ -47,7 +47,7 @@ public ResponseEntity findInstanceByUuidColectica( public ResponseEntity extractDataRelationshipFromFragment(@Parameter( description = "id du fragment DDI sous la forme uuid", required = true, - schema = @Schema(type = "string", example="d6c08ec1-c4d2-4b9a-b358-b23aa4e0af93")) + schema = @Schema(type = "string", example="16a35b68-4479-4282-95ed-ff7d151746e4")) @PathVariable String uuid) throws RmesException { return ResponseEntity.ok(this.ddiFragmentService.extractDataRelationship(uuid)); } From ba74f83bdbe461e47029f9f8102a6fe2929ca3da Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Thu, 27 Feb 2025 18:23:23 +0100 Subject: [PATCH 14/25] refactor (sonar issues and unused code) --- .../config/keycloak/KeycloakServices.java | 15 +- .../models/RessourcePackage.java | 635 +++--------------- .../fr/insee/rmes/utils/ddi/DDIFunctions.java | 126 ---- 3 files changed, 116 insertions(+), 660 deletions(-) delete mode 100644 src/main/java/fr/insee/rmes/utils/ddi/DDIFunctions.java diff --git a/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java b/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java index 502d36bb..48bc558c 100644 --- a/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java +++ b/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java @@ -15,7 +15,6 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.temporal.ChronoUnit; -import java.util.concurrent.atomic.AtomicReference; @Service @Slf4j @@ -30,15 +29,15 @@ public class KeycloakServices { final String realm; private String token; - //TODO is it the right way to synchronize ? - private final AtomicReference expiration; + + private Instant expiration; public KeycloakServices(@Value("${fr.insee.rmes.metadata.keycloak.secret}") String secret, @Value("${fr.insee.rmes.metadata.keycloak.resource}") String resource, @Value("${fr.insee.rmes.metadata.keycloak.server}") String server, @Value("${fr.insee.rmes.metadata.keycloak.realm}") String realm) { this.secret = secret; this.resource = resource; this.server = server; this.realm = realm; - this.expiration = new AtomicReference<>(); + this.expiration = null; } /** @@ -73,22 +72,22 @@ private String getKeycloakAccessToken() { * @return boolean */ private boolean isCurrentTokenValid() { - return expiration.get() != null && Instant.now().isBefore(expiration.get().minus(1, ChronoUnit.SECONDS)); + return expiration != null && Instant.now().isBefore(expiration.minus(1, ChronoUnit.SECONDS)); } - public synchronized String getFreshToken() { + public String getFreshToken() { log.atTrace().log(() -> "Check if token is valid with expiration at " + expirationAsString()); if (!this.isCurrentTokenValid()) { log.debug("Start refreshing token"); token = getKeycloakAccessToken(); - this.expiration.set(expirationFrom(token)); + this.expiration = expirationFrom(token); log.atTrace().log(() -> "New token valid until " + expirationAsString()); } return token; } private String expirationAsString() { - Instant expirationInstant = expiration.get(); + Instant expirationInstant = expiration; return expirationInstant == null ? null : LocalDateTime.ofInstant(expirationInstant, ZoneId.systemDefault()).toString(); } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/models/RessourcePackage.java b/src/main/java/fr/insee/rmes/tocolecticaapi/models/RessourcePackage.java index 6484769a..97c48d67 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/models/RessourcePackage.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/models/RessourcePackage.java @@ -1,592 +1,175 @@ package fr.insee.rmes.tocolecticaapi.models; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.*; -import java.util.stream.Collectors; @JsonIgnoreProperties(ignoreUnknown = true) -public class RessourcePackage { - - @JsonProperty("CodeListSchemes") - private List codeListSchemes; - - @JsonProperty("ItemName") - private Map itemName; - - @JsonProperty("IsPublished") - private boolean isPublished; - - @JsonProperty("IsPopulated") - private boolean isPopulated; - - @JsonProperty("Version") - private int version; - - @JsonProperty("VersionDate") - private Date versionDate; - - @JsonProperty("CompositeId") - private CodeListScheme.CodeList.CompositeId compositeId; - - @JsonProperty("AgencyId") - private String agencyId; - - @JsonProperty("Identifier") - private String identifier; - - @JsonProperty("ItemType") - private String itemType; - // Getters and Setters - - - public Map getItemName() { - return itemName; - } - - public void setItemName(Map itemName) { - this.itemName = itemName; - } - - public boolean isPublished() { - return isPublished; - } - - public void setPublished(boolean published) { - isPublished = published; - } +public record RessourcePackage(@JsonProperty("CodeListSchemes") + List codeListSchemes, - public boolean isPopulated() { - return isPopulated; - } + @JsonProperty("ItemName") + Map itemName, - public void setPopulated(boolean populated) { - isPopulated = populated; - } + @JsonProperty("IsPublished") + boolean isPublished, - public int getVersion() { - return version; - } - - public void setVersion(int version) { - this.version = version; - } + @JsonProperty("IsPopulated") + boolean isPopulated, - public Date getVersionDate() { - return versionDate; - } + @JsonProperty("Version") + int version, - public void setVersionDate(Date versionDate) { - this.versionDate = versionDate; - } + @JsonProperty("VersionDate") + Date versionDate, + @JsonProperty("CompositeId") + CompositeId compositeId, - public String getItemType() { - return itemType; - } + @JsonProperty("AgencyId") + String agencyId, - public void setItemType(String itemType) { - this.itemType = itemType; - } + @JsonProperty("Identifier") + String identifier, + @JsonProperty("ItemType") + String itemType) { - public String getAgencyId() { - return agencyId; - } - public void setAgencyId(String agencyId) { - this.agencyId = agencyId; - } + public static final String LABEL_FR = "fr-FR"; - public String getIdentifier() { - return identifier; - } + @JsonIgnoreProperties(ignoreUnknown = true) + public record CodeListScheme(@JsonProperty("CodeLists") List codeLists) { - public void setIdentifier(String identifier) { - this.identifier = identifier; - } + private List> mapCodesToLabels() { + List> listOfMappings = codeLists().getFirst().initMapCodeList(); - public List getCodeListSchemes() { - return codeListSchemes; - } + codeLists().forEach(codeList -> codeList.addMapCodeListTo(listOfMappings)); - public void setCodeListSchemes(List codeListSchemes) { - this.codeListSchemes = codeListSchemes; + return listOfMappings; + } } @JsonIgnoreProperties(ignoreUnknown = true) - public static class CodeListScheme { - @JsonProperty("CodeLists") - private List codeLists; - - public List getCodeLists() { - return codeLists; - } - - public void setCodeLists(List codeLists) { - this.codeLists = codeLists; - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class CodeList { - - @JsonProperty("ItemType") - private String itemType; - @JsonProperty("Codes") - private List codes; - - @JsonProperty("ItemName") - private Map itemName; - - @JsonProperty("Label") - private Map label; - - @JsonProperty("DisplayLabel") - private String displayLabel; - - @JsonProperty("IsPublished") - private boolean isPublished; - - @JsonProperty("IsPopulated") - private boolean isPopulated; - - @JsonProperty("Version") - private int version; - - @JsonProperty("VersionDate") - private Date versionDate; - - @JsonProperty("CompositeId") - private CompositeId compositeId; - - @JsonProperty("AgencyId") - private String agencyId; - - @JsonProperty("Identifier") - private String identifier; - - - public List getCodes() { - return codes; - } - - public void setCodes(List codes) { - this.codes = codes; - } - - public Map getItemName() { - return itemName; - } - - public void setItemName(Map itemName) { - this.itemName = itemName; - } - - public Map getLabel() { - return label; - } - - public void setLabel(Map label) { - this.label = label; - } - - public String getDisplayLabel() { - return displayLabel; - } - - public void setDisplayLabel(String displayLabel) { - this.displayLabel = displayLabel; - } - - public boolean isPublished() { - return isPublished; - } - - public void setPublished(boolean published) { - isPublished = published; - } - - public boolean isPopulated() { - return isPopulated; - } - - public void setPopulated(boolean populated) { - isPopulated = populated; - } - - public int getVersion() { - return version; - } - - public void setVersion(int version) { - this.version = version; - } - - public Date getVersionDate() { - return versionDate; - } - - public void setVersionDate(Date versionDate) { - this.versionDate = versionDate; - } - - public CompositeId getCompositeId() { - return compositeId; - } - - public void setCompositeId(CompositeId compositeId) { - this.compositeId = compositeId; - } - - public String getAgencyId() { - return agencyId; - } - - public void setAgencyId(String agencyId) { - this.agencyId = agencyId; - } - - public String getIdentifier() { - return identifier; - } - - public void setIdentifier(String identifier) { - this.identifier = identifier; - } - - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class Code { - - @JsonProperty("Value") - private String value; - - @JsonProperty("Category") - private Category category; - - @JsonProperty("AgencyId") - private String agencyId; - - @JsonProperty("Identifier") - private String identifier; - - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - public Category getCategory() { - return category; - } - - public void setCategory(Category category) { - this.category = category; - } - - public String getAgencyId() { - return agencyId; - } - - public void setAgencyId(String agencyId) { - this.agencyId = agencyId; - } - - public String getIdentifier() { - return identifier; - } - - public void setIdentifier(String identifier) { - this.identifier = identifier; - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class Category { - - @JsonProperty("ItemName") - private Map itemName; - - @JsonProperty("Label") - private Map label; - - @JsonProperty("DisplayLabel") - private String displayLabel; - - @JsonProperty("IsPublished") - private boolean isPublished; - - @JsonProperty("IsPopulated") - private boolean isPopulated; - - @JsonProperty("Version") - private int version; - - @JsonProperty("VersionDate") - private Date versionDate; + public record CodeList(@JsonProperty("ItemType") + String itemType, + @JsonProperty("Codes") + List codes, - @JsonProperty("CompositeId") - private CompositeId compositeId; + @JsonProperty("ItemName") + Map itemName, - @JsonProperty("AgencyId") - private String agencyId; + @JsonProperty("Label") + Map label, - @JsonProperty("Identifier") - private String identifier; + @JsonProperty("DisplayLabel") + String displayLabel, + @JsonProperty("IsPublished") + boolean isPublished, - public Map getItemName() { - return itemName; - } + @JsonProperty("IsPopulated") + boolean isPopulated, - public void setItemName(Map itemName) { - this.itemName = itemName; - } + @JsonProperty("Version") + int version, - public Map getLabel() { - return label; - } + @JsonProperty("VersionDate") + Date versionDate, - public void setLabel(Map label) { - this.label = label; - } + @JsonProperty("CompositeId") + CompositeId compositeId, - public String getDisplayLabel() { - return displayLabel; - } + @JsonProperty("AgencyId") + String agencyId, - public void setDisplayLabel(String displayLabel) { - this.displayLabel = displayLabel; - } + @JsonProperty("Identifier") + String identifier) { - public boolean isPublished() { - return isPublished; - } - - public void setPublished(boolean published) { - isPublished = published; - } - - public boolean isPopulated() { - return isPopulated; - } - - public void setPopulated(boolean populated) { - isPopulated = populated; - } - - public int getVersion() { - return version; - } - - public void setVersion(int version) { - this.version = version; - } - - public Date getVersionDate() { - return versionDate; - } - - public void setVersionDate(Date versionDate) { - this.versionDate = versionDate; - } - - public CompositeId getCompositeId() { - return compositeId; - } - - public void setCompositeId(CompositeId compositeId) { - this.compositeId = compositeId; - } - - public String getAgencyId() { - return agencyId; - } - - public void setAgencyId(String agencyId) { - this.agencyId = agencyId; - } - - public String getIdentifier() { - return identifier; - } - - public void setIdentifier(String identifier) { - this.identifier = identifier; - } - } + private List> initMapCodeList() { + return codes().stream() + .map(code -> { + Map map = new HashMap<>(); + map.put(itemName().get(LABEL_FR), code.value()); + map.put("Label", code.category().label().get(LABEL_FR)); + return map; + }) + .toList(); + } + private void addMapCodeListTo(List> listOfMappings) { + Iterator codeIterator = codes().iterator(); + for (Map mapping : listOfMappings) { + mapping.put(itemName().get(LABEL_FR), codeIterator.next().value()); } + } + } - @JsonIgnoreProperties(ignoreUnknown = true) - public static class CompositeId { + @JsonIgnoreProperties(ignoreUnknown = true) + public record Code(@JsonProperty("Value") + String value, - @JsonProperty("Item1") - private String item1; + @JsonProperty("Category") + Category category, - @JsonProperty("Item2") - private int item2; + @JsonProperty("AgencyId") + String agencyId, - @JsonProperty("Item3") - private String item3; + @JsonProperty("Identifier") + String identifier) { + } + @JsonIgnoreProperties(ignoreUnknown = true) + public record Category(@JsonProperty("ItemName") + Map itemName, - public String getItem1() { - return item1; - } + @JsonProperty("Label") + Map label, - public void setItem1(String item1) { - this.item1 = item1; - } + @JsonProperty("DisplayLabel") + String displayLabel, - public int getItem2() { - return item2; - } + @JsonProperty("IsPublished") + boolean isPublished, - public void setItem2(int item2) { - this.item2 = item2; - } + @JsonProperty("IsPopulated") + boolean isPopulated, - public String getItem3() { - return item3; - } + @JsonProperty("Version") + int version, - public void setItem3(String item3) { - this.item3 = item3; - } + @JsonProperty("VersionDate") + Date versionDate, - } + @JsonProperty("CompositeId") + CompositeId compositeId, - } + @JsonProperty("AgencyId") + String agencyId, + @JsonProperty("Identifier") + String identifier) { } - public static class ItemNameWithCodes { - private Map itemName; - private List codes; - - public Map getItemName() { - return itemName; - } - public void setItemName(Map itemName) { - this.itemName = itemName; - } + @JsonIgnoreProperties(ignoreUnknown = true) + public record CompositeId(@JsonProperty("Item1") + String item1, - public List getCodes() { - return codes; - } + @JsonProperty("Item2") + int item2, - public void setCodes(List codes) { - this.codes = codes; - } - } - public List getAllValuesFromCodeListScheme(int x) { - List values = new ArrayList<>(); - - // Vérifiez si l'indice x est valide pour la liste codeListSchemes - if (x >= 0 && x < codeListSchemes.size()) { - CodeListScheme scheme = codeListSchemes.get(x); - if (scheme != null && scheme.getCodeLists() != null) { - // Itérer sur toutes les CodeLists - for (CodeListScheme.CodeList codeList : scheme.getCodeLists()) { - if (codeList.getCodes() != null) { - // Itérer sur tous les Codes dans la liste - for (CodeListScheme.CodeList.Code code : codeList.getCodes()) { - values.add(code.getValue()); - } - } - } - } - } - return values; - } + @JsonProperty("Item3") + String item3) { - public List> getAllItemNamesFromCodeListScheme(int x) { - List> itemNamesList = new ArrayList<>(); - - // Vérifiez si l'indice x est valide pour la liste codeListSchemes - if (x >= 0 && x < codeListSchemes.size()) { - CodeListScheme scheme = codeListSchemes.get(x); - if (scheme != null && scheme.getCodeLists() != null) { - // Itérer sur toutes les CodeLists - for (CodeListScheme.CodeList codeList : scheme.getCodeLists()) { - if (codeList.getItemName() != null) { - // Ajoutez la carte itemName à la liste des noms d'items - itemNamesList.add(codeList.getItemName()); - } - } - } - } - return itemNamesList; - } - public List getAllItemNamesWithCodesFromCodeListScheme(int x) { - List itemNamesWithCodesList = new ArrayList<>(); - - // Vérifiez si l'indice x est valide pour la liste codeListSchemes - if (x >= 0 && x < codeListSchemes.size()) { - CodeListScheme scheme = codeListSchemes.get(x); - if (scheme != null && scheme.getCodeLists() != null) { - // Itérer sur toutes les CodeLists - for (CodeListScheme.CodeList codeList : scheme.getCodeLists()) { - if (codeList.getItemName() != null) { - // Créez un nouvel objet pour stocker l'itemName et la liste des codes - ItemNameWithCodes itemNameWithCodes = new ItemNameWithCodes(); - itemNameWithCodes.setItemName(codeList.getItemName()); - List codes = new ArrayList<>(); - for (CodeListScheme.CodeList.Code code : codeList.getCodes()) { - codes.add(code.getValue()); - } - itemNameWithCodes.setCodes(codes); - // Ajoutez l'objet à la liste des noms d'items avec les codes - itemNamesWithCodesList.add(itemNameWithCodes); - } - } - } - } - return itemNamesWithCodesList; } public List> mapCodesToLabels() { - List> listOfMappings = new ArrayList<>(); - CodeListScheme cls = this.codeListSchemes.get(0); // Obtention du premier CodeListScheme et normalement unique pour le moment - - // Nous allons d'abord créer une liste de tous les labels depuis la première CodeList où chaque code est relié à un unique pays - List labels = cls.getCodeLists().get(0).getCodes().stream() - .map(code -> code.getCategory().getLabel().get("fr-FR")) - .collect(Collectors.toList()); - - // Ensuite, nous parcourons toutes les CodeLists pour récuperer les codes - for (int i = 0; i < cls.getCodeLists().size(); i++) { - CodeListScheme.CodeList currentCodeList = cls.getCodeLists().get(i); - List codes = currentCodeList.getCodes(); - - // Itérer sur chaque code dans la CodeList actuelle - for (int j = 0; j < codes.size(); j++) { - Map mapping = new HashMap<>(); - CodeListScheme.CodeList.Code currentCode = codes.get(j); - - // Utiliser le label correspondant de la première CodeList - mapping.put(currentCodeList.getItemName().get("fr-FR"), currentCode.getValue()); - if (i == 0) { // pour la première CodeList, on ajoute aussi le label - mapping.put("Label", labels.get(j)); - } - - // Si c'est la première CodeList, initialiser les éléments de la liste des mappings - if (i == 0) { - listOfMappings.add(mapping); - } else { - // Ajouter les données de code aux mappings existants pour les autres CodeLists - listOfMappings.get(j).putAll(mapping); - } - } - } - - return listOfMappings; + return codeListSchemes.getFirst().mapCodesToLabels(); } + } diff --git a/src/main/java/fr/insee/rmes/utils/ddi/DDIFunctions.java b/src/main/java/fr/insee/rmes/utils/ddi/DDIFunctions.java deleted file mode 100644 index cba1b660..00000000 --- a/src/main/java/fr/insee/rmes/utils/ddi/DDIFunctions.java +++ /dev/null @@ -1,126 +0,0 @@ -package fr.insee.rmes.utils.ddi; - -import lombok.extern.slf4j.Slf4j; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; -import java.io.IOException; -import java.io.StringReader; -import java.util.HashMap; -import java.util.Map; - -@Slf4j -public class DDIFunctions { - - - public static Map getDDIControlConstructReferenceListFromFragment(String fragment) - throws RuntimeException { - - NodeList controlConstructReferenceList = null; - - String expression = "/Fragment/Instrument[1]/ControlConstructReference"; - try { - controlConstructReferenceList = getElementListFromFragmentByXpath(fragment, expression); - log.debug("Occurence of ControlConstructReference : " + controlConstructReferenceList.getLength()); - } catch (XPathExpressionException|SAXException|IOException|ParserConfigurationException ex) { - throw new RuntimeException(ex); - } - return getMapIdTypeOfObjectFromElementList(controlConstructReferenceList); - - } - - public static Map getDDIControlConstructReferenceListFromSequenceFragment(String fragment) - throws RuntimeException { - - NodeList controlConstructReferenceList = null; - - String expression = "/Fragment/Sequence[1]/ControlConstructReference"; - try { - controlConstructReferenceList = getElementListFromFragmentByXpath(fragment, expression); - log.debug("Occurence of ControlConstructReference : " + controlConstructReferenceList.getLength()); - } catch (XPathExpressionException|SAXException|IOException|ParserConfigurationException ex) { - throw new RuntimeException(ex); - } - return getMapIdTypeOfObjectFromElementList(controlConstructReferenceList); - } - - public static Map getDDIQuestionReferenceIDFromFragment(String fragment) throws RuntimeException { - - NodeList questionReferenceList = null; - String expression = "/Fragment/QuestionConstruct/r:QuestionReference"; - try { - questionReferenceList = getElementListFromFragmentByXpath(fragment, expression); - log.debug("Occurence of QuestionReference : " + questionReferenceList.getLength()); - } catch (XPathExpressionException|SAXException|IOException|ParserConfigurationException ex) { - throw new RuntimeException(ex); - } - return getMapIdTypeOfObjectFromElementList(questionReferenceList); - - } - - public static Map getDDISourceParameterReferenceIDFromFragment(String fragment) - throws RuntimeException { - NodeList sourceParameterReferenceList = null; - String expression = "/Fragment/QuestionItem[1]/r:Binding[1]/r:SourceParameterReference[1]"; - try { - sourceParameterReferenceList = getElementListFromFragmentByXpath(fragment, expression); - log.debug("Occurence of SourceParameterReference : " + sourceParameterReferenceList.getLength()); - } catch (XPathExpressionException|SAXException|IOException|ParserConfigurationException ex) { - throw new RuntimeException(ex); - } - return getMapIdTypeOfObjectFromElementList(sourceParameterReferenceList); - } - - public static Map getDDITargetParameterReferenceIDFromFragment(String fragment) - throws RuntimeException { - NodeList targetParameterReferenceList = null; - String expression = "/Fragment/QuestionItem[1]/r:Binding[1]/r:TargetParameterReference[1]"; - try { - targetParameterReferenceList = getElementListFromFragmentByXpath(fragment, expression); - log.debug("Occurence of TargetParameterReference : " + targetParameterReferenceList.getLength()); - } catch (XPathExpressionException|SAXException|IOException|ParserConfigurationException ex) { - throw new RuntimeException(ex); - } - return getMapIdTypeOfObjectFromElementList(targetParameterReferenceList); - } - - public static NodeList getElementListFromFragmentByXpath(String fragment, String xpathExpression) - throws SAXException, IOException, ParserConfigurationException, XPathExpressionException { - - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - InputSource is = new InputSource(new StringReader(fragment)); - Document doc = builder.parse(is); - Element fragmentElement = doc.getDocumentElement(); - XPathFactory xpf = XPathFactory.newInstance(); - XPath path = xpf.newXPath(); - return (NodeList) path.evaluate(xpathExpression, fragmentElement, XPathConstants.NODESET); - - } - - public static Map getMapIdTypeOfObjectFromElementList(NodeList nodeList) { - Map mapIdTypeOfObject = new HashMap(); - String id = ""; - String type = ""; - for (int i = 0; i < nodeList.getLength(); i++) { - Node n = nodeList.item(i); - NodeList list = n.getChildNodes(); - id = list.item(3).getTextContent(); - type = list.item(7).getTextContent(); - mapIdTypeOfObject.put(id, type); - } - return mapIdTypeOfObject; - } - -} From 136e2eb681e9eb468757bdfbc63dad3c0efec7c2 Mon Sep 17 00:00:00 2001 From: Fabrice Bibonne Date: Fri, 28 Feb 2025 07:47:04 +0100 Subject: [PATCH 15/25] refactor (use yaml for properties) --- src/main/resources/application.properties | 41 ---------- src/main/resources/application.yaml | 99 +++++++++++++++++++++++ 2 files changed, 99 insertions(+), 41 deletions(-) delete mode 100644 src/main/resources/application.properties create mode 100644 src/main/resources/application.yaml diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties deleted file mode 100644 index 9e2fda4f..00000000 --- a/src/main/resources/application.properties +++ /dev/null @@ -1,41 +0,0 @@ -# Get properties from pom.xml -fr.insee.rmes.ddias.version=@project.version@ -spring.servlet.multipart.max-file-size=50MB -springdoc.swagger-ui.path= / -springdoc.swagger-ui.enabled=true -springdoc.swagger-ui.syntax-highlight.activated=false - -spring.profiles.active= dev - -##SWAGGER -springdoc.swagger-ui.oauth.clientId= localhost-frontend -spring.security.oauth2.resourceserver.jwt.jwk-set-uri=${fr.insee.rmes.metadata.keycloak.server}/.../protocol/openid-connect/certs -fr.insee.rmes.security.whitelist-matchers=/swagger-ui/index.html,/v3/api-docs/** - -#Configuration du jeton JWT -fr.insee.rmes.role.administrateur= XXX -fr.insee.rmes.role.gestionnaire.colectica= XXX -fr.insee.rmes.security.token.oidc-claim-role= XXX -fr.insee.rmes.security.token.oidc-claim-username= XXX - -#search -fr.insee.rmes.elasticsearch.url= https://urlElastic -fr.insee.rmes.elasticsearch.apikey= cle - -# Metadata service -## Colectica Repository API -fr.insee.rmes.api.remote.metadata.url = http://ddiApi -fr.insee.rmes.api.remote.metadata.agency = xxx - -#Keycloack Services -fr.insee.rmes.metadata.keycloak.secret = xxxxx -fr.insee.rmes.metadata.keycloak.resource = xxxxx -fr.insee.rmes.metadata.keycloak.server = https://keycloak.xxx.fr/ -fr.insee.rmes.metadata.keycloak.realm = xxxx - -# Swagger host -fr.insee.rmes.api.host = localhost:8080 -fr.insee.rmes.api.name= /ddi-access-services/api -fr.insee.rmes.api.scheme= http -spring.servlet.multipart.enabled=true -spring.servlet.multipart.max-request-size=50MB \ No newline at end of file diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml new file mode 100644 index 00000000..3624fc61 --- /dev/null +++ b/src/main/resources/application.yaml @@ -0,0 +1,99 @@ +fr: + insee: + rmes: + ddias: + version: "@project.version@" + filenames: + maxlength: 50 + security: + enabled: false + whitelist-matchers: "/swagger-ui/index.html,/v3/api-docs/**" + token: + oidc-claim-role: XXX + oidc-claim-username: XXX + role: + administrateur: XXX + gestionnaire: + colectica: XXX + search: + root: + sub-group: + ids: "6baffbbc-86d5-4982-b6a4-7b035f7043ad, 2bfe02de-0044-4a94-b516-b166d9499971" + resource-package: + ids: "d574ed7e-2a7c-491f-b25c-f84141d4e96e" + elasticsearch: + url: "https://urlElastic" + index: + name: xxxx + host: urlElastic + port: 80 + apiId: id + apikey: cle + api: + remote: + metadata: + MetadataService: + impl: MetadataServiceImpl + url: "http://ddiApi" + agency: xxxx + key: xxxx + user: xxxxx + host: localhost:8080 + name: "/ddi-access-services/api" + scheme: http + metadata: + keycloak: + secret: xxxxx + resource: xxxxx + server: "https://keycloak.xxx.fr/" + realm: xxxx + cors: + allowedOrigin: "*" + springdoc: + issuer-url-authorization: "${fr.insee.rmes.metadata.keycloak.server}/.../protocol/openid-connect/auth" + issuer-url-refresh: "${fr.insee.rmes.metadata.keycloak.server}/.../protocol/openid-connect/token" + issuer-url-token: "${fr.insee.rmes.metadata.keycloak.server}/.../protocol/openid-connect/token" + issuer-description: Keycloak + contact-name: xxxxx + contact-email: "un-email-fonctionnel@xxx.fr" + contact-url: "https://www.insee.fr/fr/accueil" + +spring: + servlet: + multipart: + max-file-size: 50MB + enabled: true + max-request-size: 50MB + profiles: + active: dev + security: + oauth2: + resourceserver: + jwt: + jwk-set-uri: "${fr.insee.rmes.metadata.keycloak.server}/.../protocol/openid-connect/certs" + +springdoc: + swagger-ui: + path: / + enabled: true + syntax-highlight: + activated: false + oauth: + clientId: localhost-frontend + +logging: + logback: + rollingpolicy: + max-file-size: 1MB + file: + name: XXXX + +auth: + api: + url: XXX + username: XXX + password: XXX + +item: + api: + url: "${fr.insee.rmes.api.remote.metadata.url}/api/v1/item/" \ No newline at end of file From 90025db80c143250054d7d55d67dcb7534158df1 Mon Sep 17 00:00:00 2001 From: Fabrice Bibonne Date: Fri, 28 Feb 2025 09:00:04 +0100 Subject: [PATCH 16/25] refactor (use spring security to retrieve bearer token for colectica) #115 --- pom.xml | 14 ++- .../fr/insee/rmes/config/SecurityConfig.java | 10 +- .../config/keycloak/KeycloakServices.java | 100 +++--------------- .../tocolecticaapi/models/AuthRequest.java | 27 ----- .../service/ColecticaServiceImpl.java | 10 +- src/main/resources/application.yaml | 18 ++-- 6 files changed, 40 insertions(+), 139 deletions(-) delete mode 100644 src/main/java/fr/insee/rmes/tocolecticaapi/models/AuthRequest.java diff --git a/pom.xml b/pom.xml index 5938785a..4745bffd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.1 + 3.4.3 fr.insee.rmes @@ -26,13 +26,6 @@ - - - com.auth0 - java-jwt - ${java-jwt.version} - - com.fasterxml.jackson.dataformat jackson-dataformat-xml @@ -89,6 +82,11 @@ spring-boot-starter-oauth2-resource-server + + org.springframework.boot + spring-boot-starter-oauth2-client + + net.sf.saxon diff --git a/src/main/java/fr/insee/rmes/config/SecurityConfig.java b/src/main/java/fr/insee/rmes/config/SecurityConfig.java index 9c45c446..72150a62 100644 --- a/src/main/java/fr/insee/rmes/config/SecurityConfig.java +++ b/src/main/java/fr/insee/rmes/config/SecurityConfig.java @@ -7,6 +7,7 @@ import org.springframework.context.annotation.Profile; import org.springframework.core.convert.converter.Converter; import org.springframework.http.HttpMethod; +import org.springframework.security.config.Customizer; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configurers.AbstractAuthenticationFilterConfigurer; @@ -20,9 +21,9 @@ import org.springframework.security.web.util.matcher.AntPathRequestMatcher; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Configuration @EnableWebSecurity @@ -73,7 +74,8 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { .anyRequest().authenticated()); http .formLogin(AbstractAuthenticationFilterConfigurer::permitAll) - .oauth2ResourceServer(oauth2 -> oauth2.jwt(jwt -> jwt.jwtAuthenticationConverter(jwtAuthenticationConverter()))); + .oauth2ResourceServer(oauth2 -> oauth2.jwt(jwt -> jwt.jwtAuthenticationConverter(jwtAuthenticationConverter()))) + .oauth2Client(Customizer.withDefaults()); return http.build(); } @@ -115,7 +117,7 @@ Converter> jwtGrantedAuthoritiesConverter() { List roles = (List) claims.getOrDefault(claimPath[claimPath.length - 1], List.of()); //if we need to add customs roles to every connected user we could define this variable (static or from properties) //roles.addAll(defaultRolesForUsers); - return Collections.unmodifiableCollection(roles.stream().map(s -> new GrantedAuthority() { + return roles.stream().map(s -> new GrantedAuthority() { @Override public String getAuthority() { return ROLE_PREFIX + s; @@ -125,7 +127,7 @@ public String getAuthority() { public String toString() { return getAuthority(); } - }).toList()); + }).collect(Collectors.toList()); } catch (ClassCastException e) { return List.of(); } diff --git a/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java b/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java index 48bc558c..1ba5962c 100644 --- a/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java +++ b/src/main/java/fr/insee/rmes/config/keycloak/KeycloakServices.java @@ -1,98 +1,22 @@ package fr.insee.rmes.config.keycloak; -import com.auth0.jwt.JWT; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; +import org.springframework.security.oauth2.client.OAuth2AuthorizedClientManager; +import org.springframework.security.oauth2.client.web.client.OAuth2ClientHttpRequestInterceptor; import org.springframework.stereotype.Service; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.web.client.RestTemplate; +import org.springframework.web.client.RestClient; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.temporal.ChronoUnit; +import java.util.function.Consumer; -@Service -@Slf4j -public class KeycloakServices { - - final String secret; - - final String resource; - - final String server; - - final String realm; - - private String token; - - private Instant expiration; - - public KeycloakServices(@Value("${fr.insee.rmes.metadata.keycloak.secret}") String secret, @Value("${fr.insee.rmes.metadata.keycloak.resource}") String resource, @Value("${fr.insee.rmes.metadata.keycloak.server}") String server, @Value("${fr.insee.rmes.metadata.keycloak.realm}") String realm) { - this.secret = secret; - this.resource = resource; - this.server = server; - this.realm = realm; - this.expiration = null; - } - - /** - * Permet de récuperer un jeton keycloak - * - * @return jeton - */ - private String getKeycloakAccessToken() { - - RestTemplate keycloakClient = new RestTemplate(); - String keycloakUrl = server + "/realms/" + realm + "/protocol/openid-connect/token"; - - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); +import static org.springframework.security.oauth2.client.web.client.RequestAttributeClientRegistrationIdResolver.clientRegistrationId; - MultiValueMap body = new LinkedMultiValueMap<>(); - body.add("grant_type", "client_credentials"); - body.add("client_id", resource); - body.add("client_secret", secret); - HttpEntity entity = new HttpEntity<>(body, headers); - - Token accessToken = keycloakClient.postForObject(keycloakUrl, entity, Token.class); - - log.trace("Keycloak token provided"); - return accessToken.accessToken(); - - } - - /** - * check if oidc token has expired - * - * @return boolean - */ - private boolean isCurrentTokenValid() { - return expiration != null && Instant.now().isBefore(expiration.minus(1, ChronoUnit.SECONDS)); - } +@Service +public record KeycloakServices(OAuth2AuthorizedClientManager authorizedClientManager) { - public String getFreshToken() { - log.atTrace().log(() -> "Check if token is valid with expiration at " + expirationAsString()); - if (!this.isCurrentTokenValid()) { - log.debug("Start refreshing token"); - token = getKeycloakAccessToken(); - this.expiration = expirationFrom(token); - log.atTrace().log(() -> "New token valid until " + expirationAsString()); - } - return token; - } + public static final String CLIENT_REGISTRATION_ID = "colectica-client"; - private String expirationAsString() { - Instant expirationInstant = expiration; - return expirationInstant == null ? null : LocalDateTime.ofInstant(expirationInstant, ZoneId.systemDefault()).toString(); + public Consumer configureOidcClientAutoAuthentication() { + return builder -> builder + .requestInterceptor(new OAuth2ClientHttpRequestInterceptor(authorizedClientManager)) + .defaultRequest(requestHeadersSpec -> requestHeadersSpec.attributes(clientRegistrationId(CLIENT_REGISTRATION_ID))); } - - private Instant expirationFrom(String token) { - return JWT.decode(token).getExpiresAtAsInstant(); - } - } diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/models/AuthRequest.java b/src/main/java/fr/insee/rmes/tocolecticaapi/models/AuthRequest.java deleted file mode 100644 index 8f2b96dc..00000000 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/models/AuthRequest.java +++ /dev/null @@ -1,27 +0,0 @@ -package fr.insee.rmes.tocolecticaapi.models; - -public class AuthRequest { - private String username; - private String password; - - public AuthRequest(String username, String password) { - this.username = username; - this.password = password; - } - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } -} diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java index 54538c2f..1c3d51cb 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/service/ColecticaServiceImpl.java @@ -66,7 +66,7 @@ public record ColecticaServiceImpl(ElasticService elasticService, private static final String TRANSACTIONID = "{\"TransactionId\":"; @Autowired - public ColecticaServiceImpl(KeycloakServices kc, + public ColecticaServiceImpl(KeycloakServices keycloakServices, ElasticService elasticService, ExportUtils exportUtils, DDIDerefencer ddiDerefencer, @@ -74,10 +74,10 @@ public ColecticaServiceImpl(KeycloakServices kc, String serviceUrl, @Value("${fr.insee.rmes.api.remote.metadata.agency}") String agency){ - this(elasticService, initRestClient(kc, serviceUrl), exportUtils, ddiDerefencer, agency); + this(elasticService, initRestClient(keycloakServices, serviceUrl), exportUtils, ddiDerefencer, agency); } - private static RestClient initRestClient(KeycloakServices kc, String serviceUrl) { + private static RestClient initRestClient(KeycloakServices keycloakServices, String serviceUrl) { SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); factory.setConnectTimeout(100_000); // Temps de connexion en millisecondes factory.setReadTimeout(100_000); // Temps de lecture en millisecondes @@ -86,9 +86,7 @@ private static RestClient initRestClient(KeycloakServices kc, String serviceUrl) .requestFactory(factory) .baseUrl(URI.create(serviceUrl + "/").resolve("api/v1/")) .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_XML_VALUE) - .defaultRequest(requestHeadersSpec -> requestHeadersSpec.header(HttpHeaders.AUTHORIZATION, - "Bearer " + kc.getFreshToken()) - ) + .apply(keycloakServices.configureOidcClientAutoAuthentication()) .build(); } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 3624fc61..d23afc9c 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -47,6 +47,7 @@ fr: resource: xxxxx server: "https://keycloak.xxx.fr/" realm: xxxx + scopes: "message.read,message.write" cors: allowedOrigin: "*" springdoc: @@ -71,6 +72,17 @@ spring: resourceserver: jwt: jwk-set-uri: "${fr.insee.rmes.metadata.keycloak.server}/.../protocol/openid-connect/certs" + client: + registration: + colectica-client: #Same value as the constant fr.insee.rmes.config.keycloak.KeycloakServices.CLIENT_REGISTRATION_ID + provider: ddi-as-provider + client-id: ${fr.insee.rmes.metadata.keycloak.resource} + client-secret: ${fr.insee.rmes.metadata.keycloak.secret} + authorization-grant-type: authorization_code + scope: ${fr.insee.rmes.metadata.keycloak.scopes} + provider: + ddi-as-provider: # same value as the key spring.security.client.registration.colectica-client.provider + issuer-uri: "${fr.insee.rmes.metadata.keycloak.server}/${fr.insee.rmes.metadata.keycloak.realm}/.../protocol/openid-connect/certs" springdoc: swagger-ui: @@ -88,12 +100,6 @@ logging: file: name: XXXX -auth: - api: - url: XXX - username: XXX - password: XXX - item: api: url: "${fr.insee.rmes.api.remote.metadata.url}/api/v1/item/" \ No newline at end of file From d7d37ce2d98ead69928db0500f8e2f5edbbf5a6f Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Fri, 28 Feb 2025 12:11:27 +0100 Subject: [PATCH 17/25] fix (use spring security to retrieve bearer token for colectica) fix configuration #115 --- .../fr/insee/rmes/config/SecurityConfig.java | 4 +-- src/main/resources/application.yaml | 25 +++++++++---------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/main/java/fr/insee/rmes/config/SecurityConfig.java b/src/main/java/fr/insee/rmes/config/SecurityConfig.java index 72150a62..f0737873 100644 --- a/src/main/java/fr/insee/rmes/config/SecurityConfig.java +++ b/src/main/java/fr/insee/rmes/config/SecurityConfig.java @@ -7,7 +7,6 @@ import org.springframework.context.annotation.Profile; import org.springframework.core.convert.converter.Converter; import org.springframework.http.HttpMethod; -import org.springframework.security.config.Customizer; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configurers.AbstractAuthenticationFilterConfigurer; @@ -74,8 +73,7 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { .anyRequest().authenticated()); http .formLogin(AbstractAuthenticationFilterConfigurer::permitAll) - .oauth2ResourceServer(oauth2 -> oauth2.jwt(jwt -> jwt.jwtAuthenticationConverter(jwtAuthenticationConverter()))) - .oauth2Client(Customizer.withDefaults()); + .oauth2ResourceServer(oauth2 -> oauth2.jwt(jwt -> jwt.jwtAuthenticationConverter(jwtAuthenticationConverter()))); return http.build(); } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index d23afc9c..707ce755 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -46,7 +46,6 @@ fr: secret: xxxxx resource: xxxxx server: "https://keycloak.xxx.fr/" - realm: xxxx scopes: "message.read,message.write" cors: allowedOrigin: "*" @@ -72,17 +71,17 @@ spring: resourceserver: jwt: jwk-set-uri: "${fr.insee.rmes.metadata.keycloak.server}/.../protocol/openid-connect/certs" - client: - registration: - colectica-client: #Same value as the constant fr.insee.rmes.config.keycloak.KeycloakServices.CLIENT_REGISTRATION_ID - provider: ddi-as-provider - client-id: ${fr.insee.rmes.metadata.keycloak.resource} - client-secret: ${fr.insee.rmes.metadata.keycloak.secret} - authorization-grant-type: authorization_code - scope: ${fr.insee.rmes.metadata.keycloak.scopes} - provider: - ddi-as-provider: # same value as the key spring.security.client.registration.colectica-client.provider - issuer-uri: "${fr.insee.rmes.metadata.keycloak.server}/${fr.insee.rmes.metadata.keycloak.realm}/.../protocol/openid-connect/certs" + client: + registration: + colectica-client: #Same value as the constant fr.insee.rmes.config.keycloak.KeycloakServices.CLIENT_REGISTRATION_ID + provider: ddi-as-provider + client-id: ${fr.insee.rmes.metadata.keycloak.resource} + client-secret: ${fr.insee.rmes.metadata.keycloak.secret} + authorization-grant-type: client_credentials + scope: ${fr.insee.rmes.metadata.keycloak.scopes} + provider: + ddi-as-provider: # same value as the key spring.security.client.registration.colectica-client.provider + issuer-uri: "${fr.insee.rmes.metadata.keycloak.server}/..." springdoc: swagger-ui: @@ -98,7 +97,7 @@ logging: rollingpolicy: max-file-size: 1MB file: - name: XXXX + path: @project.build.directory@/ddi-as.log item: api: From 51495fdbf2a8088982d9dd3dae26c08c7b05ccaa Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Fri, 28 Feb 2025 18:05:49 +0100 Subject: [PATCH 18/25] fix (tests after RestClient refactor) exclude OAuth2ClientAutoConfiguration --- .../fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java | 3 ++- .../transfoxsl/controller/TransformationControllerTest.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java b/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java index 511891dd..d5f778c7 100644 --- a/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java +++ b/src/test/java/fr/insee/rmes/tocolecticaapi/controller/GetItemTest.java @@ -7,6 +7,7 @@ import fr.insee.rmes.transfoxsl.service.XsltTransformationService; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.security.oauth2.client.servlet.OAuth2ClientAutoConfiguration; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.context.annotation.Import; @@ -25,7 +26,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -@WebMvcTest(GetItem.class) +@WebMvcTest(value=GetItem.class, excludeAutoConfiguration = OAuth2ClientAutoConfiguration.class) @Import(value = {DdiFragmentServiceImpl.class, SecurityConfig.class, XsltTransformationService.class}) @EnableConfigurationProperties(InseeSecurityTokenProperties.class) @ActiveProfiles("dev") diff --git a/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java b/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java index a283e152..b1cb862b 100644 --- a/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java +++ b/src/test/java/fr/insee/rmes/transfoxsl/controller/TransformationControllerTest.java @@ -8,6 +8,7 @@ import fr.insee.rmes.utils.export.XDocReport; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.security.oauth2.client.servlet.OAuth2ClientAutoConfiguration; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.context.annotation.Import; @@ -28,7 +29,7 @@ import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -@WebMvcTest(TransformationController.class) +@WebMvcTest(value = TransformationController.class, excludeAutoConfiguration = OAuth2ClientAutoConfiguration.class) @Import(SecurityConfig.class) @EnableConfigurationProperties(InseeSecurityTokenProperties.class) @ActiveProfiles("dev") From 33fc23f20ce092548517271d8e38134155e961f7 Mon Sep 17 00:00:00 2001 From: G TanSndil <72496065+GtanSndil@users.noreply.github.com> Date: Mon, 3 Mar 2025 11:22:32 +0100 Subject: [PATCH 19/25] Update pom -> 2.1.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4745bffd..a58108a6 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ fr.insee.rmes ddi-access-services - 2.1.1 + 2.1.2 DDI Access Services Rest API to access to Colectica repository services From 63530d03a9b844e7d76a4f458b80b24be576a1e5 Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Mon, 3 Mar 2025 16:10:13 +0100 Subject: [PATCH 20/25] tests (Authenticated calls to colectica with Spring Security work well) --- pom.xml | 13 + .../fragments/DdiFragmentServiceImpl.java | 1 - src/main/resources/application.yaml | 2 +- .../AuthenticatedCallsToColecticaTest.java | 209 ++++++++++++++++ .../AuthenticatedCalls/wiremockReturn.xml | 236 ++++++++++++++++++ 5 files changed, 459 insertions(+), 2 deletions(-) create mode 100644 src/test/java/fr/insee/rmes/AuthenticatedCallsToColecticaTest.java create mode 100644 src/test/resources/AuthenticatedCalls/wiremockReturn.xml diff --git a/pom.xml b/pom.xml index 4745bffd..4d2ed89d 100644 --- a/pom.xml +++ b/pom.xml @@ -23,6 +23,7 @@ 4.4.0 20250107 0.8.12 + 3.12.0 @@ -66,6 +67,18 @@ spring-security-test test + + org.wiremock + wiremock-standalone + ${wiremock.version} + test + + + org.wiremock + wiremock + ${wiremock.version} + test + org.json json diff --git a/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java b/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java index 81558e16..deaf62bf 100644 --- a/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java +++ b/src/main/java/fr/insee/rmes/tocolecticaapi/fragments/DdiFragmentServiceImpl.java @@ -10,7 +10,6 @@ import java.io.ByteArrayInputStream; import java.io.IOException; -import java.nio.charset.StandardCharsets; @Slf4j @Service diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 707ce755..866fc09a 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -65,7 +65,7 @@ spring: enabled: true max-request-size: 50MB profiles: - active: dev + active: prod security: oauth2: resourceserver: diff --git a/src/test/java/fr/insee/rmes/AuthenticatedCallsToColecticaTest.java b/src/test/java/fr/insee/rmes/AuthenticatedCallsToColecticaTest.java new file mode 100644 index 00000000..72cf0663 --- /dev/null +++ b/src/test/java/fr/insee/rmes/AuthenticatedCallsToColecticaTest.java @@ -0,0 +1,209 @@ +package fr.insee.rmes; + +import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import fr.insee.rmes.config.InseeSecurityTokenProperties; +import fr.insee.rmes.config.SecurityConfig; +import fr.insee.rmes.config.keycloak.KeycloakServices; +import fr.insee.rmes.tocolecticaapi.controller.GetItem; +import fr.insee.rmes.tocolecticaapi.fragments.DdiFragmentServiceImpl; +import fr.insee.rmes.tocolecticaapi.service.ColecticaServiceImpl; +import fr.insee.rmes.tocolecticaapi.service.ElasticService; +import fr.insee.rmes.transfoxsl.service.XsltTransformationService; +import fr.insee.rmes.transfoxsl.service.internal.DDIDerefencer; +import fr.insee.rmes.utils.ExportUtils; +import org.apache.hc.core5.http.ContentType; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.security.oauth2.client.servlet.OAuth2ClientAutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.http.MediaType; +import org.springframework.security.oauth2.client.InMemoryOAuth2AuthorizedClientService; +import org.springframework.security.oauth2.client.OAuth2AuthorizedClientService; +import org.springframework.security.oauth2.client.registration.ClientRegistrationRepository; +import org.springframework.security.oauth2.client.registration.InMemoryClientRegistrationRepository; +import org.springframework.security.oauth2.client.web.AuthenticatedPrincipalOAuth2AuthorizedClientRepository; +import org.springframework.security.oauth2.client.web.OAuth2AuthorizedClientRepository; +import org.springframework.security.oauth2.core.OAuth2AccessToken; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.test.web.servlet.MockMvc; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.Map; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static com.github.tomakehurst.wiremock.matching.RequestPatternBuilder.newRequestPattern; +import static fr.insee.rmes.config.keycloak.KeycloakServices.CLIENT_REGISTRATION_ID; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.oauth2Client; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@WebMvcTest(value = GetItem.class, excludeAutoConfiguration = OAuth2ClientAutoConfiguration.class, properties = { + "fr.insee.rmes.api.remote.metadata.url=" + AuthenticatedCallsToColecticaTest.URL_WIREMOCK, + "fr.insee.rmes.api.remote.metadata.agency=Insee", + "spring.profiles.active=dev", + "fr.insee.rmes.security.token.oidc-claim-role=roles", + "fr.insee.rmes.security.token.oidc-claim-username=name", + "fr.insee.rmes.security.whitelist-matchers=", + "fr.insee.rmes.role.administrateur=", + "fr.insee.rmes.role.gestionnaire.colectica=" +}) +@Import({DdiFragmentServiceImpl.class, XsltTransformationService.class, ColecticaServiceImpl.class, KeycloakServices.class, SecurityConfig.class}) +@WireMockTest(httpPort = AuthenticatedCallsToColecticaTest.WIREMOCK_PORT) +class AuthenticatedCallsToColecticaTest { + + static final int WIREMOCK_PORT = 8282; + static final String URL_WIREMOCK = "http://localhost:" + WIREMOCK_PORT; + static final String EXPECTED_JSON = """ + [ + { + "id": "urn:ddi:fr.insee:77dfee13-1a96-43c1-b1ef-7b78b45be55d:2", + "nom": "Dessin de fichier thl-CASD", + "label": [ + { + "contenu": "", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], + "variables": [ + { + "id": "urn:ddi:fr.insee:aa090e13-252a-4da7-b776-8f8e505be6d9:1", + "nom": "ACTOCCUPE", + "label": [ + { + "contenu": "Actif occupé", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], + "ordre": "1", + "representation": "codes", + "controles": { + "codes": [ + { + "code": "0", + "label": [ + { + "langue": "fr", + "contenu": "Inactif ou actif non occupé" + }, + { + "langue": "en", + "contenu": "" + } + ] + }, + { + "code": "1", + "label": [ + { + "langue": "fr", + "contenu": "Actif occupé" + }, + { + "langue": "en", + "contenu": "" + } + ] + } + ] + } + }, + { + "id": "urn:ddi:fr.insee:7f2ae666-2208-46ce-bb1d-fd3fda8c6667:1", + "nom": "AG", + "label": [ + { + "contenu": "Age au 31 décembre de l'année de l'enquête", + "langue": "fr" + }, + { + "contenu": "", + "langue": "en" + } + ], + "ordre": "2", + "representation": "numerique" + } + ] + } + ] + """; + + @Autowired + MockMvc mvc; + + @MockitoBean + ElasticService elasticService; + @MockitoBean + ExportUtils exportUtils; + @MockitoBean + DDIDerefencer ddiDerefencer; + + @Test + void authenticatedCallsToColectica() throws Exception { + String tokenValue = "blablabla"; + String uuid = "16a35b68-4479-4282-95ed-ff7d151746e4"; + String endpoint = "/Item/ddiFragment/" + uuid + "/dataRelationship"; + String colecticaRepositoryEndpoint = "/api/v1/ddiset/Insee/" + uuid; + prepareWiremock(colecticaRepositoryEndpoint); + + mvc.perform(get(endpoint) + .with(oauth2Client(CLIENT_REGISTRATION_ID) + .accessToken(new OAuth2AccessToken(OAuth2AccessToken.TokenType.BEARER, tokenValue, null, null, Collections.emptySet())))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().json(EXPECTED_JSON)); + WireMock.verify(newRequestPattern().withHeader("Authorization", equalTo("Bearer " + tokenValue))); + } + + private void prepareWiremock(String ressource) throws IOException { + ResponseDefinitionBuilder responseDefinitionBuilder = ok() + .withHeader("Content-Type", ContentType.APPLICATION_XML.getMimeType()) + .withBody(wiremockReturn()); + stubFor(WireMock.get(ressource).willReturn(responseDefinitionBuilder)); + } + + private String wiremockReturn() throws IOException { + try (InputStream resourceAsStream = AuthenticatedCallsConfigurationTest.class.getResourceAsStream("/AuthenticatedCalls/wiremockReturn.xml")) { + return new String(resourceAsStream.readAllBytes(), StandardCharsets.UTF_8); + } + } + + @TestConfiguration + @EnableConfigurationProperties(InseeSecurityTokenProperties.class) + static class AuthenticatedCallsConfigurationTest { + + @Bean + InMemoryClientRegistrationRepository clientRegistrationRepository() { + return new InMemoryClientRegistrationRepository(Map.of()); + } + + @Bean + OAuth2AuthorizedClientService authorizedClientService(ClientRegistrationRepository clientRegistrationRepository) { + return new InMemoryOAuth2AuthorizedClientService(clientRegistrationRepository); + } + + @Bean + OAuth2AuthorizedClientRepository authorizedClientRepository(OAuth2AuthorizedClientService authorizedClientService) { + return new AuthenticatedPrincipalOAuth2AuthorizedClientRepository(authorizedClientService); + } + } + +} diff --git a/src/test/resources/AuthenticatedCalls/wiremockReturn.xml b/src/test/resources/AuthenticatedCalls/wiremockReturn.xml new file mode 100644 index 00000000..f1224551 --- /dev/null +++ b/src/test/resources/AuthenticatedCalls/wiremockReturn.xml @@ -0,0 +1,236 @@ + + + + fr.insee + 16a35b68-4479-4282-95ed-ff7d151746e4 + 2 + PhysicalInstance + + + + urn:ddi:fr.insee:16a35b68-4479-4282-95ed-ff7d151746e4:2 + fr.insee + 16a35b68-4479-4282-95ed-ff7d151746e4 + 2 + AD\ACY5R8 + + + publication teo2 avec physical Instance + + + + + Fichier thl-CASD + + + + fr.insee + 77dfee13-1a96-43c1-b1ef-7b78b45be55d + 2 + DataRelationship + + + + + + urn:ddi:fr.insee:77dfee13-1a96-43c1-b1ef-7b78b45be55d:2 + fr.insee + 77dfee13-1a96-43c1-b1ef-7b78b45be55d + 2 + AD\ACY5R8 + + + publication teo2 avec physical Instance + + + + Dessin de fichier thl-CASD + + + urn:ddi:fr.insee:8398b3dd-d7c1-4779-9ecb-0b085d8433fe:2 + fr.insee + 8398b3dd-d7c1-4779-9ecb-0b085d8433fe + 2 + + Dessin de fichier thl-CASD + + + + fr.insee + aa090e13-252a-4da7-b776-8f8e505be6d9 + 1 + Variable + + + fr.insee + 7f2ae666-2208-46ce-bb1d-fd3fda8c6667 + 1 + Variable + + + + + + + + + urn:ddi:fr.insee:aa090e13-252a-4da7-b776-8f8e505be6d9:1 + fr.insee + aa090e13-252a-4da7-b776-8f8e505be6d9 + 1 + + ACTOCCUPE + + + Actif occupé + + + Nature : Contrôle + + + fr.insee + dd218170-6a82-4db7-ba9c-23ad53700f61 + 1 + RepresentedVariable + + + + + fr.insee + 57c49a53-905f-4759-8feb-06651f1e0eaf + 1 + CodeList + + + + + + + + urn:ddi:fr.insee:57c49a53-905f-4759-8feb-06651f1e0eaf:1 + fr.insee + 57c49a53-905f-4759-8feb-06651f1e0eaf + 1 + + urn:ddi:fr.insee:7a576a7a-a230-44d7-92be-395a80ae0fb5:1 + fr.insee + 7a576a7a-a230-44d7-92be-395a80ae0fb5 + 1 + + fr.insee + 017ff7c4-99a2-4885-a231-67f10484e0ba + 1 + Category + + 0 + + + urn:ddi:fr.insee:ebf128ec-54d8-4db0-8765-83f961faa5d2:1 + fr.insee + ebf128ec-54d8-4db0-8765-83f961faa5d2 + 1 + + fr.insee + 900b2c41-17ed-4a78-9e2f-0615d58e1060 + 1 + Category + + 1 + + + + + + urn:ddi:fr.insee:900b2c41-17ed-4a78-9e2f-0615d58e1060:1 + fr.insee + 900b2c41-17ed-4a78-9e2f-0615d58e1060 + 1 + + Actif occupé + + + + + + urn:ddi:fr.insee:017ff7c4-99a2-4885-a231-67f10484e0ba:1 + fr.insee + 017ff7c4-99a2-4885-a231-67f10484e0ba + 1 + + Inactif ou actif non occupé + + + + + + + urn:ddi:fr.insee:dd218170-6a82-4db7-ba9c-23ad53700f61:1 + fr.insee + dd218170-6a82-4db7-ba9c-23ad53700f61 + 1 + + ACTOCCUPE + + + Actif occupé + + + Nature : Contrôle + + + + fr.insee + 57c49a53-905f-4759-8feb-06651f1e0eaf + 1 + CodeList + + + + + + + + urn:ddi:fr.insee:7f2ae666-2208-46ce-bb1d-fd3fda8c6667:1 + fr.insee + 7f2ae666-2208-46ce-bb1d-fd3fda8c6667 + 1 + + AG + + + Age au 31 décembre de l'année de l'enquête + + + Nature : Contrôle + + + fr.insee + bf4fea39-aef1-45ed-83ec-494415d43ad0 + 1 + RepresentedVariable + + + + + + + + + + urn:ddi:fr.insee:bf4fea39-aef1-45ed-83ec-494415d43ad0:1 + fr.insee + bf4fea39-aef1-45ed-83ec-494415d43ad0 + 1 + + AG + + + Age au 31 décembre de l'année de l'enquête + + + Nature : Contrôle + + + + + From 188c8798800f494eb76ead842a4024a18839d5b1 Mon Sep 17 00:00:00 2001 From: Fabrice B Date: Mon, 3 Mar 2025 17:06:47 +0100 Subject: [PATCH 21/25] fix (mistake in configuration) wrong default profile declaration --- src/main/resources/application.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index b5940d32..866fc09a 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -65,6 +65,7 @@ spring: enabled: true max-request-size: 50MB profiles: + active: prod security: oauth2: resourceserver: From 87e76b960c4549c98e009613f9a1909ee0cd3610 Mon Sep 17 00:00:00 2001 From: Guylene Tandeau Date: Tue, 4 Mar 2025 13:37:50 +0100 Subject: [PATCH 22/25] sonar --- pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pom.xml b/pom.xml index 6c62347b..89217a77 100644 --- a/pom.xml +++ b/pom.xml @@ -24,6 +24,9 @@ 20250107 0.8.12 3.12.0 + inseefr + https://sonarcloud.io + InseeFr_DDI-Access-Services From f03b286f291a9b025d6651be26d2dc43a308e6cf Mon Sep 17 00:00:00 2001 From: Guylene Tandeau Date: Tue, 4 Mar 2025 14:52:56 +0100 Subject: [PATCH 23/25] sonar: suppress a warning --- src/main/java/fr/insee/rmes/config/SecurityConfig.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/fr/insee/rmes/config/SecurityConfig.java b/src/main/java/fr/insee/rmes/config/SecurityConfig.java index f0737873..20bf2ccc 100644 --- a/src/main/java/fr/insee/rmes/config/SecurityConfig.java +++ b/src/main/java/fr/insee/rmes/config/SecurityConfig.java @@ -49,6 +49,7 @@ public SecurityConfig(InseeSecurityTokenProperties inseeSecurityTokenProperties, } @Bean + @SuppressWarnings("java:S4502") @Profile("prod") public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { http From 8b198be25482fec4785f38c3c918d8c8e2c255c9 Mon Sep 17 00:00:00 2001 From: Fabrice Bibonne Date: Wed, 30 Apr 2025 10:37:48 +0200 Subject: [PATCH 24/25] fix (mistake while merging pom) --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 03e3b6ba..4c85c980 100644 --- a/pom.xml +++ b/pom.xml @@ -6,12 +6,12 @@ org.springframework.boot spring-boot-starter-parent - 4.6.1-beta + 3.4.5 fr.insee.rmes ddi-access-services - 2.1.4-beta5 + 4.6.1-beta DDI Access Services Rest API to access to Colectica repository services From f28ee701cb2ce2bcfa804d2cd12def6e66fcd7aa Mon Sep 17 00:00:00 2001 From: Fabrice Bibonne Date: Mon, 5 May 2025 09:40:04 +0200 Subject: [PATCH 25/25] test (add test for SecurityConfig to reach expected test coverage) red test --- .../insee/rmes/config/SecurityConfigTest.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 src/test/java/fr/insee/rmes/config/SecurityConfigTest.java diff --git a/src/test/java/fr/insee/rmes/config/SecurityConfigTest.java b/src/test/java/fr/insee/rmes/config/SecurityConfigTest.java new file mode 100644 index 00000000..b0bf43a1 --- /dev/null +++ b/src/test/java/fr/insee/rmes/config/SecurityConfigTest.java @@ -0,0 +1,48 @@ +package fr.insee.rmes.config; + +import com.nimbusds.jwt.JWT; +import com.nimbusds.jwt.JWTClaimsSet; +import com.nimbusds.jwt.JWTParser; +import org.junit.jupiter.api.Test; +import org.springframework.core.convert.converter.Converter; +import org.springframework.security.core.GrantedAuthority; +import org.springframework.security.oauth2.jwt.Jwt; +import org.springframework.security.oauth2.jwt.MappedJwtClaimSetConverter; + +import java.text.ParseException; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class SecurityConfigTest { + + @Test + void jwtGrantedAuthoritiesConverterTest() throws ParseException { + //Chemin pour récupérer la liste des rôles dans le jwt (token) + String oidcClaimRole = ""; + //Chemin pour récupérer le username dans le jwt (token) + String oidcClaimUsername = ""; + String token = ""; + GrantedAuthority[] expectedGrantedAuthorities={}; + Jwt jwt = createJwt(token, JWTParser.parse(token)); + SecurityConfig securityConfig = new SecurityConfig(new InseeSecurityTokenProperties(oidcClaimRole, oidcClaimUsername), null, null, null); + Converter> converter = securityConfig.jwtGrantedAuthoritiesConverter(); + + assertThat(converter.convert(jwt)).containsExactly(expectedGrantedAuthorities); + + } + + private Jwt createJwt(String token, JWT parsedJwt) throws ParseException { + JWTClaimsSet jwtClaimsSet = parsedJwt.getJWTClaimsSet(); + Map headers = new LinkedHashMap<>(parsedJwt.getHeader().toJSONObject()); + Map claims = MappedJwtClaimSetConverter.withDefaults(Collections.emptyMap()).convert(jwtClaimsSet.getClaims()); + return Jwt.withTokenValue(token) + .headers((h) -> h.putAll(headers)) + .claims((c) -> c.putAll(claims)) + .build(); + } + +} \ No newline at end of file