diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index c9fe9de1c..d510a114c 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -3,7 +3,8 @@ name: Build snapshot docker image on: push: branches-ignore: - - main + - * # main + # turned off for now (needs secrets renewal) jobs: build-snapshot: diff --git a/src/main/java/fr/insee/genesis/controller/exception/ExceptionController.java b/src/main/java/fr/insee/genesis/controller/exception/ExceptionController.java new file mode 100644 index 000000000..0be8fdb55 --- /dev/null +++ b/src/main/java/fr/insee/genesis/controller/exception/ExceptionController.java @@ -0,0 +1,75 @@ +package fr.insee.genesis.controller.exception; + +import fr.insee.genesis.exceptions.*; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.http.ProblemDetail; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; + +/** + * This controller uses Spring's ControllerAdvice annotation to intercept exceptions. + * It implements the RFC 9457 by returning + * Spring's ProblemDetail object. + */ +@ControllerAdvice +@Slf4j +public class ExceptionController { + + // Note: No handler for uncaught Exception.class for now since it breaks soms tests. + + @ExceptionHandler + public ProblemDetail handleGenericGenesisException(GenesisException genesisException) { + log.error("GenesisException: {}", genesisException.getMessage(), genesisException); + return ProblemDetail.forStatusAndDetail( + resolveHttpCode(genesisException.getStatus()), + genesisException.getMessage()); + } + + /** Returns the corresponding http status, or 500 if the given code does not match a http status. */ + private static HttpStatus resolveHttpCode(int statusCode) { + HttpStatus httpStatus = HttpStatus.resolve(statusCode); + return httpStatus != null ? httpStatus : HttpStatus.INTERNAL_SERVER_ERROR; + } + + @ExceptionHandler(InvalidDateIntervalException.class) + public ProblemDetail handleInvalidDateIntervalException(InvalidDateIntervalException e) { + log.error("InvalidDateIntervalException: {}", e.getMessage()); + return ProblemDetail.forStatusAndDetail( + HttpStatus.BAD_REQUEST, + e.getMessage()); + } + + @ExceptionHandler(ModesConflictException.class) + public ProblemDetail handleModesConflictException(ModesConflictException e) { + log.error("ModesConflictException: {}", e.getMessage()); + return ProblemDetail.forStatusAndDetail( + HttpStatus.CONFLICT, + e.getMessage()); + } + + @ExceptionHandler(UndefinedModesException.class) + public ProblemDetail handleUndefinedModesException(UndefinedModesException e) { + log.error("UndefinedModesException: {}", e.getMessage()); + return ProblemDetail.forStatusAndDetail( + HttpStatus.NOT_FOUND, + e.getMessage()); + } + + @ExceptionHandler(UndefinedMetadataException.class) + public ProblemDetail handleUndefinedMetadataException(UndefinedMetadataException e) { + log.error("UndefinedMetadataException: {}", e.getMessage()); + return ProblemDetail.forStatusAndDetail( + HttpStatus.NOT_FOUND, + e.getMessage()); + } + + @ExceptionHandler(InvalidMetadataException.class) + public ProblemDetail handleInvalidMetadataException(InvalidMetadataException e) { + log.error("InvalidMetadataException: {}", e.getMessage()); + return ProblemDetail.forStatusAndDetail( + HttpStatus.BAD_REQUEST, + e.getMessage()); + } + +} diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java index 157d38e16..697a377fb 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java @@ -15,6 +15,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; +import java.time.Instant; import java.time.LocalDateTime; import java.util.List; @@ -55,17 +56,17 @@ public ResponseEntity> getAllInterrogationIdsByCollectionI @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) @Parameter( description = "sinceDate", - schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00") + schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00Z") ) - LocalDateTime start, + Instant start, @RequestParam("end") @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) @Parameter( description = "untilDate", - schema = @Schema(type = "string", format = "date-time", example = "2026-01-31T23:59:59") + schema = @Schema(type = "string", format = "date-time", example = "2026-01-31T23:59:59Z") ) - LocalDateTime end) { + Instant end) { List responses = surveyUnitService.findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(collectionInstrumentId, start,end); return ResponseEntity.ok(responses); } diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index e692852e3..f79efe4cb 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -15,6 +15,7 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -43,23 +44,16 @@ @Slf4j @Controller +@RequiredArgsConstructor public class RawResponseController { private static final String SUCCESS_MESSAGE = "Interrogation %s saved"; private static final String INTERROGATION_ID = "interrogationId"; - public static final String NB_DOCS_WITH_FORMATTED = "%d document(s) processed, including %d FORMATTED after data verification for collectionInstrumentId %s"; - public static final String NB_DOCS = "%d document(s) processed for collectionInstrumentId %s"; + private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort; private final RawResponseApiPort rawResponseApiPort; private final RawResponseInputRepository rawRepository; - - public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort, RawResponseApiPort rawResponseApiPort, RawResponseInputRepository rawRepository) { - this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort; - this.rawResponseApiPort = rawResponseApiPort; - this.rawRepository = rawRepository; - } - @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database") @PutMapping(path = "/responses/raw/lunatic-json/save") @PreAuthorize("hasRole('COLLECT_PLATFORM')") @@ -118,11 +112,8 @@ public ResponseEntity processRawResponses( log.info("Try to process raw responses for collectionInstrumentId {} and {} interrogationIds", collectionInstrumentId, interrogationIdList.size()); List errors = new ArrayList<>(); try { - DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId, interrogationIdList, errors); - return result.formattedDataCount() == 0 ? - ResponseEntity.ok(NB_DOCS.formatted(result.dataCount(), collectionInstrumentId)) - : ResponseEntity.ok(NB_DOCS_WITH_FORMATTED - .formatted(result.dataCount(), result.formattedDataCount(), collectionInstrumentId)); + DataProcessResult result = rawResponseApiPort.processRawResponsesByInterrogationIds(collectionInstrumentId, interrogationIdList, errors); + return ResponseEntity.ok(result.message(collectionInstrumentId)); } catch (GenesisException e) { return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } @@ -140,11 +131,8 @@ public ResponseEntity processRawResponsesByCollectionInstrumentId( ) { log.info("Try to process raw responses for collectionInstrumentId {}", collectionInstrumentId); try { - DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId); - return result.formattedDataCount() == 0 ? - ResponseEntity.ok(NB_DOCS.formatted(result.dataCount(), collectionInstrumentId)) - : ResponseEntity.ok(NB_DOCS_WITH_FORMATTED - .formatted(result.dataCount(), result.formattedDataCount(), collectionInstrumentId)); + DataProcessResult result = rawResponseApiPort.processRawResponsesByInterrogationIds(collectionInstrumentId); + return ResponseEntity.ok(result.message(collectionInstrumentId)); } catch (GenesisException e) { return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } @@ -183,7 +171,7 @@ public ResponseEntity getJsonRawData( @RequestParam("campaignName") String campaignName, @RequestParam(value = "mode") Mode modeSpecified ) { - List data = lunaticJsonRawDataApiPort.getRawData(campaignName, modeSpecified, List.of(interrogationId)); + List data = lunaticJsonRawDataApiPort.getRawDataByQuestionnaireId(campaignName, modeSpecified, List.of(interrogationId)); return ResponseEntity.ok(data.getFirst()); } @@ -201,7 +189,7 @@ public ResponseEntity processJsonRawData( List errors = new ArrayList<>(); try { - DataProcessResult result = lunaticJsonRawDataApiPort.processRawData(campaignName, interrogationIdList, errors); + DataProcessResult result = lunaticJsonRawDataApiPort.processRawDataByInterrogationIds(campaignName, interrogationIdList, errors); return result.formattedDataCount() == 0 ? ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount())) : ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification" diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseReprocessController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseReprocessController.java new file mode 100644 index 000000000..2f9d87250 --- /dev/null +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseReprocessController.java @@ -0,0 +1,102 @@ +package fr.insee.genesis.controller.rest.responses; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; +import fr.insee.genesis.domain.ports.api.ReprocessRawResponseApiPort; +import fr.insee.genesis.exceptions.GenesisException; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.format.annotation.DateTimeFormat; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestParam; + +import java.time.Instant; + +@Controller +@RequiredArgsConstructor +@Slf4j +public class RawResponseReprocessController { + + private final ReprocessRawResponseApiPort reprocessRawResponseApiPort; + + @Operation(summary = "Reprocess raw response of a collection instrument.") + @PostMapping(path = "/raw-responses/{collectionInstrumentId}/reprocess") + @PreAuthorize("hasRole('ADMIN')") + public ResponseEntity reProcessRawResponsesByCollectionInstrumentId( + @Parameter( + description = "Id of the collection instrument (old questionnaireId)", + example = "ENQTEST2025X00") + @PathVariable("collectionInstrumentId") + String collectionInstrumentId, + + @Parameter( + description = "Extract since", + schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00Z") + ) + @RequestParam(value = "sinceDate", required = false) + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + Instant sinceDate, + + @Parameter( + description = "Extract until", + schema = @Schema(type = "string", format = "date-time", example = "2026-02-02T00:00:00Z") + ) + @RequestParam(value = "endDate", required = false) + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + Instant endDate + ) throws GenesisException { + + DataProcessResult result = reprocessRawResponseApiPort.reprocessRawResponses( + RawDataModelType.FILIERE, + collectionInstrumentId, + sinceDate, + endDate); + + return ResponseEntity.ok(result.message(collectionInstrumentId)); + } + + @Operation(summary = "Reprocess Lunatic raw data for a questionnaire model. " + + "**Note**: Lunatic raw data is the legacy format of raw responses.") + @PostMapping(path = "/responses/raw/lunatic-json/{questionnaireId}/reprocess") + @PreAuthorize("hasRole('ADMIN')") + public ResponseEntity reProcessJsonRawDataByQuestionnaireId( + @Parameter( + description = "Questionnaire model id (old name for collection instrument id).", + example = "ENQTEST2025X00") + @PathVariable("questionnaireId") + String collectionInstrumentId, // 'questionnaireId' is the legacy name for 'collectionInstrumentId' + + @Parameter( + description = "Extract since", + schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00Z") + ) + @RequestParam(value = "sinceDate", required = false) + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + Instant sinceDate, + + @Parameter( + description = "Extract until", + schema = @Schema(type = "string", format = "date-time", example = "2026-02-02T00:00:00Z") + ) + @RequestParam(value = "endDate", required = false) + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + Instant endDate + ) throws GenesisException { + + DataProcessResult result = reprocessRawResponseApiPort.reprocessRawResponses( + RawDataModelType.LEGACY, + collectionInstrumentId, + sinceDate, + endDate); + + return ResponseEntity.ok(result.message(collectionInstrumentId)); + } + +} diff --git a/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java b/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java index 83051a4e1..00398e9f5 100644 --- a/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java +++ b/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java @@ -1,16 +1,18 @@ package fr.insee.genesis.controller.utils; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.exceptions.ModesConflictException; +import fr.insee.genesis.exceptions.UndefinedModesException; +import fr.insee.genesis.infrastructure.utils.FileUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.exceptions.GenesisException; -import fr.insee.genesis.infrastructure.utils.FileUtils; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +// Note: this class should be moved in the domain service layer. @Component @Slf4j @@ -23,25 +25,23 @@ public ControllerUtils(FileUtils fileUtils) { this.fileUtils = fileUtils; } - /** * If a mode is specified, we treat only this mode. - * If no mode is specified, we treat all modes in the campaign. + * If no mode is specified, we treat all modes in the questionnaireId. * If no mode is specified and no specs are found, we return an error - * @param campaign campaign id to get modes + * @param questionnaireId questionnaireId id to get modes * @param modeSpecified a Mode to use, null if we want all modes available * @return a list with the mode in modeSpecified or all modes if null - * @throws GenesisException if error in specs structure */ - public List getModesList(String campaign, Mode modeSpecified) throws GenesisException { + public List getModesList(String questionnaireId, Mode modeSpecified) { if (modeSpecified != null){ return Collections.singletonList(modeSpecified); } List modes = new ArrayList<>(); - String specFolder = fileUtils.getSpecFolder(campaign); + String specFolder = fileUtils.getSpecFolder(questionnaireId); List modeSpecFolders = fileUtils.listFolders(specFolder); if (modeSpecFolders.isEmpty()) { - throw new GenesisException(404, "No specification folder found " + specFolder); + throw new UndefinedModesException("No specification folder found " + specFolder); } for(String modeSpecFolder : modeSpecFolders){ if(Mode.getEnumFromModeName(modeSpecFolder) == null) { @@ -51,9 +51,18 @@ public List getModesList(String campaign, Mode modeSpecified) throws Genes modes.add(Mode.getEnumFromModeName(modeSpecFolder)); } if (modes.contains(Mode.F2F) && modes.contains(Mode.TEL)) { - throw new GenesisException(409, "Cannot treat simultaneously TEL and FAF modes"); + throw new ModesConflictException("Cannot treat simultaneously TEL and FAF modes"); } return modes; } + /** + * Returns the applicable modes for the collection instrument with the given identifier. + * @param collectionInstrumentId Collection instrument identifier. + * @return A list of modes. + */ + public List getModesList(String collectionInstrumentId) { + return getModesList(collectionInstrumentId, null); + } + } diff --git a/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java b/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java index d0a147469..20775dca2 100644 --- a/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java @@ -17,18 +17,22 @@ @NoArgsConstructor @AllArgsConstructor public class DataProcessingContextModel { + + /** (Added to the class only to remove a warning) */ @Id - private ObjectId id; //Used to remove warning + private ObjectId id; @Deprecated(forRemoval = true) private String partitionId; - private String collectionInstrumentId; //QuestionnaireId + /** New name of legacy 'questionnaireId' property. */ + private String collectionInstrumentId; private LocalDateTime lastExecution; List kraftwerkExecutionScheduleList; + /** Determines if some review service must be called during the process. */ boolean withReview; public ScheduleDto toScheduleDto(){ @@ -39,4 +43,5 @@ public ScheduleDto toScheduleDto(){ .kraftwerkExecutionScheduleList(kraftwerkExecutionScheduleList) .build(); } + } diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java index b8191dd03..e7eccec1c 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java @@ -4,5 +4,31 @@ import java.util.List; -public record DataProcessResult(int dataCount, int formattedDataCount, List errors) { +public record DataProcessResult( + int dataCount, + int formattedDataCount, + List errors) { + + public String message(String collectionInstrumentId) { + return String.format("%s%s%s.", + interrogationCountMessage(dataCount), + formattedCountMessage(formattedDataCount), + collectionIdMessage(collectionInstrumentId)); + } + + private static String interrogationCountMessage(int processedInterrogationsCount) { + boolean plural = processedInterrogationsCount > 1; + return "%d interrogation%s processed".formatted(processedInterrogationsCount, plural ? "s" : ""); + } + + private static String collectionIdMessage(String collectionInstrumentId) { + return " for collectionInstrumentId '%s'".formatted(collectionInstrumentId); + } + + private static String formattedCountMessage(int formattedCount) { + if (formattedCount == 0) + return ""; + return " (including %d FORMATTED after data verification)".formatted(formattedCount); + } + } diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawDataModelType.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawDataModelType.java index adb8a88cd..9ecc12db9 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawDataModelType.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawDataModelType.java @@ -1,5 +1,20 @@ package fr.insee.genesis.domain.model.surveyunit.rawdata; +/** Format of raw data to be imported into the data storage. */ public enum RawDataModelType { - DEFAULT, FILIERE + + /** Legacy format of raw data ('Lunatic'). */ + LEGACY, + + /** 'Filière' raw response model. */ + FILIERE; + + @Override + public String toString() { + return switch (this) { + case LEGACY -> "LEGACY (Lunatic)"; + case FILIERE -> "FILIERE raw responses"; + }; + } + } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java index 2031d80d7..8d3629ab3 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java @@ -11,8 +11,8 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import java.time.LocalDateTime; import java.time.Instant; +import java.time.LocalDateTime; import java.util.List; import java.util.Map; import java.util.Set; @@ -20,9 +20,8 @@ public interface LunaticJsonRawDataApiPort { void save(LunaticJsonRawDataModel rawData); - List getRawData(String campaignName, Mode mode, List interrogationIdList); List getRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList); - List convertRawData(List rawData, VariablesMap variablesMap); + List getUnprocessedDataIds(); Set getUnprocessedDataQuestionnaireIds(); void updateProcessDates(List surveyUnitModels); @@ -33,8 +32,7 @@ public interface LunaticJsonRawDataApiPort { List getRawDataByInterrogationId(String interrogationId); - @Deprecated(since = "1.13.0") - DataProcessResult processRawData(String campaignName, List interrogationIdList, List errors) throws GenesisException; + DataProcessResult processRawDataByInterrogationIds(String campaignName, List interrogationIdList, List errors) throws GenesisException; DataProcessResult processRawData(String collectionInstrumentId) throws GenesisException; diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java index 464295a8f..2e070f44d 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java @@ -1,7 +1,5 @@ package fr.insee.genesis.domain.ports.api; -import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; @@ -16,11 +14,9 @@ public interface RawResponseApiPort { - List getRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList); - List getRawResponsesByInterrogationID(String interrogationId); - DataProcessResult processRawResponses(String collectionInstrumentId, List interrogationIdList, List errors) throws GenesisException; - DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException; - List convertRawResponse(List rawResponses, VariablesMap variablesMap); + DataProcessResult processRawResponsesByInterrogationIds(String collectionInstrumentId, List interrogationIdList, List errors) throws GenesisException; + DataProcessResult processRawResponsesByInterrogationIds(String collectionInstrumentId) throws GenesisException; + List getUnprocessedCollectionInstrumentIds(); void updateProcessDates(List surveyUnitModels); Page findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable); diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/ReprocessRawResponseApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/ReprocessRawResponseApiPort.java new file mode 100644 index 000000000..c379ed637 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/api/ReprocessRawResponseApiPort.java @@ -0,0 +1,26 @@ +package fr.insee.genesis.domain.ports.api; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; +import fr.insee.genesis.exceptions.GenesisException; + +import java.time.Instant; + +public interface ReprocessRawResponseApiPort { + + /** + * Reprocesses raw data of the collection that correspond to the given identifier. + * An optional date interval can be given to reprocess a subset of the collection. + * @param rawDataModelType {@link RawDataModelType} + * @param collectionInstrumentId Collection instrument identifier. + * @param sinceDate Start of the date interval. + * @param endDate End of the date interval. + * @return Data processing result record. + * @see DataProcessResult + */ + DataProcessResult reprocessRawResponses( + RawDataModelType rawDataModelType, + String collectionInstrumentId, Instant sinceDate, Instant endDate) + throws GenesisException; + +} diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java index 7362cc512..77769696d 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java @@ -1,16 +1,13 @@ package fr.insee.genesis.domain.ports.api; import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire; -import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign; -import fr.insee.genesis.controller.dto.SurveyUnitDto; -import fr.insee.genesis.controller.dto.SurveyUnitInputDto; -import fr.insee.genesis.controller.dto.SurveyUnitSimplifiedDto; +import fr.insee.genesis.controller.dto.*; import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.exceptions.GenesisException; +import java.time.Instant; import java.time.LocalDateTime; import java.util.List; import java.util.Set; @@ -52,7 +49,7 @@ List findSimplifiedByCollectionInstrumentIdAndInterroga List findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since); - List findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String collectionInstrumentId, LocalDateTime start, LocalDateTime end); + List findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String collectionInstrumentId, Instant start, Instant end); //========= OPTIMISATIONS PERFS (START) ========== long countResponsesByCollectionInstrumentId(String questionnaireId); @@ -73,6 +70,11 @@ List findDistinctPageableInterrogationIdsByQuestionnaireId(Stri Long deleteByCollectionInstrumentId(String collectionInstrumentId); + Long deleteByQuestionnaireIdAndInterrogationIds( + String questionnaireId, + Set interrogationIds + ); + long countResponses(); Set findQuestionnaireIdsByCampaignId(String campaignId); diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java index e8bbadb3a..cc2f355e9 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java @@ -14,12 +14,12 @@ public interface LunaticJsonRawDataPersistencePort { void save(LunaticJsonRawDataModel rawData); - List findRawData(String campaignName, Mode mode, List interrogationIdList); List findRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList); Page findRawDataByQuestionnaireId(String questionnaireId, Pageable pageable); List findRawDataByInterrogationID(String interrogationId); List getAllUnprocessedData(); void updateProcessDates(String campaignId, Set interrogationIds); + Set findDistinctQuestionnaireIds(); Set findDistinctQuestionnaireIdsByNullProcessDate(); Set findModesByQuestionnaire(String questionnaireId); @@ -31,4 +31,5 @@ public interface LunaticJsonRawDataPersistencePort { boolean existsByInterrogationId(String interrogationId); long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireId); + } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java index 649d942ae..c8db17a73 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java @@ -21,8 +21,7 @@ public interface RawResponsePersistencePort { Page findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable); long countByCollectionInstrumentId(String collectionInstrumentId); Set findDistinctCollectionInstrumentIds(); - long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId); Page findByCollectionInstrumentId(String collectionInstrumentId, Pageable pageable); - boolean existsByInterrogationId(String interrogationId); + } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponseReprocessPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponseReprocessPersistencePort.java new file mode 100644 index 000000000..8560faa10 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponseReprocessPersistencePort.java @@ -0,0 +1,16 @@ +package fr.insee.genesis.domain.ports.spi; + +import java.time.Instant; +import java.util.Set; + +public interface RawResponseReprocessPersistencePort { + + Set findProcessedInterrogationIdsByCollectionInstrumentId( + String collectionInstrumentId); + + Set findProcessedInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween( + String collectionInstrumentId, Instant sinceDate, Instant endDate); + + void resetProcessDates(String collectionInstrumentId, Set interrogationIds); + +} diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponseReprocessPersistenceRouter.java b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponseReprocessPersistenceRouter.java new file mode 100644 index 000000000..ed77a8981 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponseReprocessPersistenceRouter.java @@ -0,0 +1,9 @@ +package fr.insee.genesis.domain.ports.spi; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; + +public interface RawResponseReprocessPersistenceRouter { + + RawResponseReprocessPersistencePort resolve(RawDataModelType rawDataModelType); + +} diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java index 0629b3029..9e3b952ca 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java @@ -2,6 +2,7 @@ import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import java.time.Instant; import java.time.LocalDateTime; import java.util.List; import java.util.Set; @@ -34,8 +35,8 @@ public interface SurveyUnitPersistencePort { List findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween( String collectionInstrumentId, - LocalDateTime start, - LocalDateTime end + Instant start, + Instant end ); //======== OPTIMISATIONS PERFS (START) ======== @@ -52,6 +53,16 @@ List findInterrogationIdsByCollectionInstrumentIdAndRecordDateB Long deleteByCollectionInstrumentId(String collectionInstrumentId); + Long deleteByCollectionInstrumentIdAndInterrogationIds( + String collectionInstrumentId, + Set interrogationIds + ); + + Long deleteByQuestionnaireIdAndInterrogationIds( + String questionnaireId, + Set interrogationIds + ); + long count(); Set findQuestionnaireIdsByCampaignId(String campaignId); diff --git a/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java b/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java index 2e254a87c..898466d49 100644 --- a/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java @@ -45,7 +45,7 @@ public MetadataModel find(String collectionInstrumentId, Mode mode) throws Genes } @Override - public MetadataModel loadAndSaveIfNotExists(String campaignName, String collectionInstrumentId, Mode mode, FileUtils fileUtils, + public MetadataModel loadAndSaveIfNotExists(String questionnaireId, String collectionInstrumentId, Mode mode, FileUtils fileUtils, List errors) throws GenesisException { List questionnaireMetadataModels = questionnaireMetadataPersistencePort.find(collectionInstrumentId.toUpperCase(), mode); diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 15abc3e2d..c2d46af52 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -2,7 +2,6 @@ import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.Constants; import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.utils.ControllerUtils; @@ -11,20 +10,15 @@ import fr.insee.genesis.domain.model.surveyunit.GroupedInterrogation; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; -import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; -import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort; import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort; -import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; -import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; -import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; -import fr.insee.genesis.domain.utils.GroupUtils; -import fr.insee.genesis.domain.utils.JsonUtils; +import fr.insee.genesis.domain.utils.LunaticJsonRawDataConverter; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.utils.FileUtils; @@ -33,21 +27,16 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; -import java.io.IOException; -import java.math.BigDecimal; import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static fr.insee.genesis.domain.service.rawdata.RawResponseService.processCollectedVariable; + @Service @Slf4j @@ -56,9 +45,7 @@ public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort { private final ControllerUtils controllerUtils; private final QuestionnaireMetadataService metadataService; private final SurveyUnitService surveyUnitService; - private final SurveyUnitQualityService surveyUnitQualityService; - private final SurveyUnitQualityToolPort surveyUnitQualityToolPort; - private final DataProcessingContextService dataProcessingContextService; + private final SurveyUnitQualityToolService surveyUnitQualityToolService; private final FileUtils fileUtils; private final Config config; @@ -68,26 +55,23 @@ public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort { private final DataProcessingContextPersistancePort dataProcessingContextPersistancePort; @Autowired - public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRawDataNewPersistencePort, - ControllerUtils controllerUtils, - QuestionnaireMetadataService metadataService, - SurveyUnitService surveyUnitService, - SurveyUnitQualityService surveyUnitQualityService, - FileUtils fileUtils, - DataProcessingContextService dataProcessingContextService, - SurveyUnitQualityToolPort surveyUnitQualityToolPort, - Config config, - DataProcessingContextPersistancePort dataProcessingContextPersistancePort + public LunaticJsonRawDataService( + LunaticJsonRawDataPersistencePort lunaticJsonRawDataNewPersistencePort, + ControllerUtils controllerUtils, + QuestionnaireMetadataService metadataService, + SurveyUnitService surveyUnitService, + FileUtils fileUtils, + SurveyUnitQualityToolService surveyUnitQualityToolService, + Config config, + DataProcessingContextPersistancePort dataProcessingContextPersistancePort ) { this.controllerUtils = controllerUtils; this.metadataService = metadataService; this.surveyUnitService = surveyUnitService; - this.surveyUnitQualityService = surveyUnitQualityService; + this.surveyUnitQualityToolService = surveyUnitQualityToolService; this.fileUtils = fileUtils; this.lunaticJsonRawDataPersistencePort = lunaticJsonRawDataNewPersistencePort; this.dataProcessingContextPersistancePort = dataProcessingContextPersistancePort; - this.surveyUnitQualityToolPort = surveyUnitQualityToolPort; - this.dataProcessingContextService = dataProcessingContextService; this.config = config; } @@ -97,273 +81,91 @@ public void save(LunaticJsonRawDataModel rawData) { } @Override - public List getRawData(String campaignName, Mode mode, List interrogationIdList) { - return lunaticJsonRawDataPersistencePort.findRawData(campaignName, mode, interrogationIdList); + public List getRawDataByQuestionnaireId( + String questionnaireId, Mode mode, List interrogationIdList + ) { + return lunaticJsonRawDataPersistencePort.findRawDataByQuestionnaireId( + questionnaireId, mode, interrogationIdList); } @Override - public List getRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList) { - return lunaticJsonRawDataPersistencePort.findRawDataByQuestionnaireId(questionnaireId, mode, interrogationIdList); - } - - @Override public List getRawDataByInterrogationId(String interrogationId) { return lunaticJsonRawDataPersistencePort.findRawDataByInterrogationID(interrogationId); } @Override - @Deprecated(since = "1.13.0") - public DataProcessResult processRawData(String campaignName, List interrogationIdList, List errors) throws GenesisException { - int dataCount=0; - int formattedDataCount=0; - DataProcessingContextModel dataProcessingContext = - dataProcessingContextService.getContextByCollectionInstrumentId(campaignName); - List modesList = controllerUtils.getModesList(campaignName, null); - for (Mode mode : modesList) { - //Load and save metadata into database, throw exception if none - VariablesMap variablesMap = getVariablesMap(campaignName, mode, errors); - int totalBatchs = Math.ceilDiv(interrogationIdList.size() , config.getRawDataProcessingBatchSize()); - int batchNumber = 1; - List interrogationIdListForMode = new ArrayList<>(interrogationIdList); - while(!interrogationIdListForMode.isEmpty()){ - log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); - int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); - List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); - - List rawData = getRawData(campaignName, mode, interrogationIdToProcess); - - List surveyUnitModels = convertRawData( - rawData, - variablesMap - ); - - //Save converted data - surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); - surveyUnitService.saveSurveyUnits(surveyUnitModels); - - //Update process dates - updateProcessDates(surveyUnitModels); - - //Increment data count - dataCount += surveyUnitModels.size(); - formattedDataCount += surveyUnitModels.stream() - .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)) - .toList() - .size(); - - //Send processed ids grouped by questionnaire (if review activated) - if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { - sendProcessedIdsToQualityTool(surveyUnitModels); - } - - //Remove processed ids from list - interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); + public DataProcessResult processRawData(String questionnaireId) throws GenesisException { + List interrogationIds = lunaticJsonRawDataPersistencePort + .findUnprocessedInterrogationIdsByCollectionInstrumentId(questionnaireId) + .stream() + .toList(); - batchNumber++; - } - } - return new DataProcessResult(dataCount, formattedDataCount, errors); + return processRawDataByInterrogationIds(questionnaireId, interrogationIds, new ArrayList<>()); } @Override - public DataProcessResult processRawData(String questionnaireId) throws GenesisException { - int dataCount=0; - int formattedDataCount=0; - DataProcessingContextModel dataProcessingContext = - dataProcessingContextService.getContextByCollectionInstrumentId(questionnaireId); - List errors = new ArrayList<>(); + public DataProcessResult processRawDataByInterrogationIds( + String questionnaireId, + List interrogationIdList, + List errors + ) throws GenesisException { - List modesList = controllerUtils.getModesList(questionnaireId, null); - for (Mode mode : modesList) { - //Load and save metadata into database, throw exception if none - VariablesMap variablesMap = getVariablesMap(questionnaireId, mode, errors); - Set interrogationIds = - lunaticJsonRawDataPersistencePort.findUnprocessedInterrogationIdsByCollectionInstrumentId(questionnaireId); + List modes = controllerUtils.getModesList(questionnaireId, null); + boolean resolveWithReviewValue = surveyUnitQualityToolService.resolveWithReviewValue(questionnaireId); - int totalBatchs = Math.ceilDiv(interrogationIds.size() , config.getRawDataProcessingBatchSize()); - int batchNumber = 1; - List interrogationIdListForMode = new ArrayList<>(interrogationIds); - while(!interrogationIdListForMode.isEmpty()){ - log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); - - int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); - List surveyUnitModels = getConvertedSurveyUnits( - questionnaireId, - mode, - interrogationIdListForMode, - maxIndex, - variablesMap - ); - - //Save converted data - surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); - surveyUnitService.saveSurveyUnits(surveyUnitModels); - - //Update process dates - updateProcessDates(surveyUnitModels); - - //Increment data count - dataCount += surveyUnitModels.size(); - formattedDataCount += surveyUnitModels.stream() - .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)) - .toList() - .size(); - - //Send processed ids grouped by questionnaire (if review activated) - if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { - sendProcessedIdsToQualityTool(surveyUnitModels); - } + int batchSize = config.getRawDataProcessingBatchSize(); + int totalBatches = Math.ceilDiv(interrogationIdList.size(), batchSize); + int dataCount = 0; + int formattedDataCount = 0; - //Remove processed ids from list - interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); - batchNumber++; - } - } - return new DataProcessResult(dataCount, formattedDataCount, errors); - } + for (Mode mode : modes) { + VariablesMap variablesMap = getVariablesMap(questionnaireId, mode, errors); + List interrogationIdListForMode = new ArrayList<>(interrogationIdList); + int batchNumber = 1; - private VariablesMap getVariablesMap(String questionnaireId, Mode mode, List errors) throws GenesisException { - VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(questionnaireId, questionnaireId, mode, fileUtils, - errors).getVariables(); - if (variablesMap == null) { - throw new GenesisException(400, - "Error during metadata parsing for mode %s :%n%s" - .formatted(mode, errors.getLast().getMessage()) - ); - } - return variablesMap; - } + while (!interrogationIdListForMode.isEmpty()) { + log.info("Processing raw data batch {}/{}", batchNumber, totalBatches); - private List getConvertedSurveyUnits(String questionnaireId, Mode mode, List interrogationIdListForMode, int maxIndex, VariablesMap variablesMap) { - List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); - List rawData = getRawDataByQuestionnaireId(questionnaireId, mode, interrogationIdToProcess); - return convertRawData( - rawData, - variablesMap - ); - } + int maxIndex = Math.min(interrogationIdListForMode.size(), batchSize); + List batch = interrogationIdListForMode.subList(0, maxIndex); - private void sendProcessedIdsToQualityTool(List surveyUnitModels) { - try { - ResponseEntity response = - surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels)); - - if (response.getStatusCode().is2xxSuccessful()) { - log.info("Successfully sent {} ids to quality tool", getProcessedIdsMap(surveyUnitModels).size()); - }else{ - log.warn("Survey unit quality tool responded non-2xx code {} and body {}", - response.getStatusCode(), response.getBody()); - } - }catch (IOException e){ - log.error("Error during Perret call request building : {}", e.toString()); - } - } + List rawData = lunaticJsonRawDataPersistencePort + .findRawDataByQuestionnaireId(questionnaireId, mode, batch); - private Map> getProcessedIdsMap(List surveyUnitModels) { - Map> processedInterrogationIdsPerQuestionnaire = new HashMap<>(); - surveyUnitModels.forEach(model -> - processedInterrogationIdsPerQuestionnaire - .computeIfAbsent(model.getCollectionInstrumentId(), k -> new HashSet<>()) - .add(model.getInterrogationId()) - ); - return processedInterrogationIdsPerQuestionnaire; - } + List surveyUnits = + LunaticJsonRawDataConverter.convertRawData(rawData, variablesMap); - @Override - public List convertRawData(List rawDataList, VariablesMap variablesMap) { - //Convert to genesis model - List surveyUnitModels = new ArrayList<>(); - List emptySurveyUnitModels = new ArrayList<>(); - //For each possible data state (we receive COLLECTED or EDITED) - for(DataState dataState : List.of(DataState.COLLECTED,DataState.EDITED)){ - for (LunaticJsonRawDataModel rawData : rawDataList) { - RawDataModelType rawDataModelType = getRawDataModelType(rawData); - - //Get optional fields - Boolean isCapturedIndirectly = getIsCapturedIndirectly(rawData); - LocalDateTime validationDate = getValidationDate(rawData); - - SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .campaignId(rawData.campaignId()) - .collectionInstrumentId(rawData.questionnaireId()) - .mode(rawData.mode()) - .interrogationId(rawData.interrogationId()) - .usualSurveyUnitId(rawData.idUE()) - .validationDate(validationDate) - .isCapturedIndirectly(isCapturedIndirectly) - .state(dataState) - .fileDate(rawData.recordDate()) - .recordDate(LocalDateTime.now()) - .collectedVariables(new ArrayList<>()) - .externalVariables(new ArrayList<>()) - .build(); - - //Data collected variables conversion - convertRawDataCollectedVariables(rawData, surveyUnitModel, dataState, rawDataModelType, variablesMap); - - //External variables conversion into COLLECTED document - if(dataState == DataState.COLLECTED){ - convertRawDataExternalVariables(rawData, surveyUnitModel, rawDataModelType, variablesMap); - } + surveyUnitService.saveSurveyUnits(surveyUnits); + updateProcessDates(surveyUnits); - boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty() - && surveyUnitModel.getExternalVariables().isEmpty(); + dataCount += surveyUnits.size(); + formattedDataCount += (int) surveyUnits.stream() + .filter(su -> su.getState() == DataState.FORMATTED) + .count(); - if(hasNoVariable){ - if(surveyUnitModel.getState() == DataState.COLLECTED){ - log.warn("No collected or external variable for interrogation {}, raw data is ignored.", rawData.interrogationId()); - } - emptySurveyUnitModels.add(surveyUnitModel); - continue;// don't add suModel but update processDate + if (resolveWithReviewValue) { + surveyUnitQualityToolService.sendProcessedIdsToQualityTool(surveyUnits); } - surveyUnitModels.add(surveyUnitModel); - } - } - if(!emptySurveyUnitModels.isEmpty()){ - updateProcessDates(emptySurveyUnitModels); - } - return surveyUnitModels; - } - - private static RawDataModelType getRawDataModelType(LunaticJsonRawDataModel rawData) { - return rawData.data().containsKey("data") ? - RawDataModelType.FILIERE : - RawDataModelType.DEFAULT; - } - private static LocalDateTime getValidationDate(LunaticJsonRawDataModel rawData) { - try{ - return rawData.data().get("validationDate") == null ? null : - LocalDateTime.parse(rawData.data().get("validationDate").toString()); - }catch(Exception e){ - log.warn("Exception when parsing validation date : {}}",e.toString()); - return null; - } - } - - private static Boolean getIsCapturedIndirectly(LunaticJsonRawDataModel rawData) { - try{ - return rawData.data().get("isCapturedIndirectly") == null ? null : - Boolean.parseBoolean(rawData.data().get("isCapturedIndirectly").toString()); - }catch(Exception e){ - log.warn("Exception when parsing isCapturedIndirectly : {}}",e.toString()); - return Boolean.FALSE; + interrogationIdListForMode = + interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); + batchNumber++; + } } + return new DataProcessResult(dataCount, formattedDataCount, errors); } @Override public List getUnprocessedDataIds() { List dtos = new ArrayList<>(); - for (GroupedInterrogation groupedInterrogation : lunaticJsonRawDataPersistencePort.findUnprocessedIds()) { - for (String interrogationId : groupedInterrogation.interrogationIds()){ + for (String interrogationId : groupedInterrogation.interrogationIds()) { dtos.add(LunaticJsonRawDataUnprocessedDto.builder() .campaignId(groupedInterrogation.partitionOrCampaignId()) .questionnaireId(groupedInterrogation.questionnaireId()) .interrogationId(interrogationId) - .build() - ); + .build()); } } return dtos; @@ -371,173 +173,27 @@ public List getUnprocessedDataIds() { @Override public Set getUnprocessedDataQuestionnaireIds() { - Set unprocessedQuestionnaireIds = lunaticJsonRawDataPersistencePort.findDistinctQuestionnaireIdsByNullProcessDate(); - Set unprocessedQuestionnaireIdsWithSpecs = new HashSet<>(); - for (String unprocessedQuestionnaireId : unprocessedQuestionnaireIds){ - Set modes = lunaticJsonRawDataPersistencePort.findModesByQuestionnaire(unprocessedQuestionnaireId); - if (modes.isEmpty()){ - continue; - } - - boolean areAllSpecsOK = true; - for(Mode mode : modes){ - if(!isSpecsPresentForQuestionnaireAndMode(unprocessedQuestionnaireId, mode)){ - areAllSpecsOK = false; - } - } - if(areAllSpecsOK){ - unprocessedQuestionnaireIdsWithSpecs.add(unprocessedQuestionnaireId); - } - } - - return unprocessedQuestionnaireIdsWithSpecs; - } - - private boolean isSpecsPresentForQuestionnaireAndMode(String unprocessedQuestionnaireId, Mode mode) { - List genesisErrors = new ArrayList<>(); - MetadataModel metadataModel; - try { - metadataModel = metadataService.loadAndSaveIfNotExists( - unprocessedQuestionnaireId, - unprocessedQuestionnaireId, - mode, - fileUtils, - genesisErrors - ); - } catch (GenesisException ge) { - log.warn("Genesis exception thrown for questionnaire %s and mode %s, excluding from get questionnaire ids...".formatted(unprocessedQuestionnaireId, mode)); - return false; - } - return metadataModel != null && genesisErrors.isEmpty(); - } - - @SuppressWarnings("unchecked") - private static void convertRawDataExternalVariables( - LunaticJsonRawDataModel srcRawData, - SurveyUnitModel dstSurveyUnitModel, - RawDataModelType rawDataModelType, - VariablesMap variablesMap - ) { - Map dataMap = srcRawData.data(); - if (rawDataModelType.equals(RawDataModelType.FILIERE)) { - dataMap = (Map) dataMap.get("data"); - } - - dataMap = (Map)dataMap.get("EXTERNAL"); - Map externalMap = JsonUtils.asMap(dataMap); - if (externalMap != null && !externalMap.isEmpty()){ - convertToExternalVar(dstSurveyUnitModel, variablesMap, externalMap); - } - } - - private static void convertToExternalVar(SurveyUnitModel dstSurveyUnitModel, VariablesMap variablesMap, Map externalMap) { - for(Map.Entry externalVariableEntry : externalMap.entrySet()){ - Object valueObject = externalVariableEntry.getValue(); - if (valueObject instanceof List){ - //Array of values - convertListVar(valueObject, externalVariableEntry, variablesMap, dstSurveyUnitModel.getExternalVariables()); - continue; - } - //Value - if (valueObject != null) { - convertOneVar(externalVariableEntry, valueObject.toString(), variablesMap, 1, dstSurveyUnitModel.getExternalVariables()); - } - } - } - - private static void convertOneVar(Map.Entry externalVariableEntry, String valueObject, VariablesMap variablesMap, int iteration, List dstSurveyUnitModel) { - VariableModel externalVariableModel = VariableModel.builder() - .varId(externalVariableEntry.getKey()) - .value(valueObject) - .scope(getIdLoop(variablesMap, externalVariableEntry.getKey())) - .iteration(iteration) - .parentId(GroupUtils.getParentGroupName(externalVariableEntry.getKey(), variablesMap)) - .build(); - dstSurveyUnitModel.add(externalVariableModel); - } - - @SuppressWarnings("unchecked") - private void convertRawDataCollectedVariables( - LunaticJsonRawDataModel srcRawData, - SurveyUnitModel dstSurveyUnitModel, - DataState dataState, - RawDataModelType rawDataModelType, - VariablesMap variablesMap - ) { - Map dataMap = srcRawData.data(); - if (rawDataModelType.equals(RawDataModelType.FILIERE)) { - dataMap = (Map) dataMap.get("data"); - } - - dataMap = (Map)dataMap.get("COLLECTED"); - - - Map collectedMap = JsonUtils.asMap(dataMap); - if (collectedMap == null || collectedMap.isEmpty()){ - if(dataState.equals(DataState.COLLECTED)) { - log.warn("No collected data for interrogation {}", srcRawData.interrogationId()); - } - return; - } - convertToCollectedVar(dstSurveyUnitModel, dataState, variablesMap, collectedMap); - - } - - private static void convertToCollectedVar( - SurveyUnitModel dstSurveyUnitModel, - DataState dataState, - VariablesMap variablesMap, - Map collectedMap - ) { - final String stateKey = dataState.toString(); - final var dest = dstSurveyUnitModel.getCollectedVariables(); - - for (Map.Entry collectedVariable : collectedMap.entrySet()) { - processCollectedVariable(collectedVariable, stateKey, variablesMap, dstSurveyUnitModel, dest); - } - } - - - private static void convertListVar(Object valuesForState, Map.Entry collectedVariable, VariablesMap variablesMap, List dstSurveyUnitModel) { - List values = JsonUtils.asStringList(valuesForState); - if (!values.isEmpty()) { - int iteration = 1; - for (String value : values) { - if (value != null && !value.isEmpty()) { - convertOneVar(collectedVariable, value, variablesMap, iteration, dstSurveyUnitModel); - } - iteration++; - } - } - } - - private static String getIdLoop(VariablesMap variablesMap, String variableName) { - if (variablesMap.getVariable(variableName) == null) { - log.warn("Variable {} not present in metadatas, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); - return Constants.ROOT_GROUP_NAME; - } - return variablesMap.getVariable(variableName).getGroupName(); + return lunaticJsonRawDataPersistencePort + .findDistinctQuestionnaireIdsByNullProcessDate() + .stream() + .filter(this::hasValidModesWithSpecs) + .collect(Collectors.toSet()); } @Override public void updateProcessDates(List surveyUnitModels) { - Set campaignIds = new HashSet<>(); - for (SurveyUnitModel surveyUnitModel : surveyUnitModels) { - campaignIds.add(surveyUnitModel.getCampaignId()); - } - - for (String campaignId : campaignIds) { - Set interrogationIds = new HashSet<>(); - for (SurveyUnitModel surveyUnitModel : - surveyUnitModels.stream().filter( - surveyUnitModel -> surveyUnitModel.getCampaignId().equals(campaignId) - ).toList()) { - interrogationIds.add(surveyUnitModel.getInterrogationId()); - } - lunaticJsonRawDataPersistencePort.updateProcessDates(campaignId, interrogationIds); - } + surveyUnitModels.stream() + .map(SurveyUnitModel::getCampaignId) + .distinct() + .forEach(campaignId -> { + Set ids = surveyUnitModels.stream() + .filter(su -> su.getCampaignId().equals(campaignId)) + .map(SurveyUnitModel::getInterrogationId) + .collect(Collectors.toSet()); + lunaticJsonRawDataPersistencePort.updateProcessDates(campaignId, ids); + }); } - + @Override public Set findDistinctQuestionnaireIds() { return lunaticJsonRawDataPersistencePort.findDistinctQuestionnaireIds(); @@ -555,19 +211,27 @@ public long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireI @Override public Map> findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime since) { - List idsByQuestionnaire = lunaticJsonRawDataPersistencePort.findProcessedIdsGroupedByQuestionnaireSince(since); - List partitionIds = idsByQuestionnaire.stream().map(GroupedInterrogation::partitionOrCampaignId).toList(); - List contexts = dataProcessingContextPersistancePort.findByPartitionIds(partitionIds); - List partitionIdsWithReview = contexts.stream().filter(DataProcessingContextModel::isWithReview).map(DataProcessingContextModel::getPartitionId).toList(); - return idsByQuestionnaire.stream().filter(groupedInterrogation -> partitionIdsWithReview.contains(groupedInterrogation.partitionOrCampaignId())) + List idsByQuestionnaire = + lunaticJsonRawDataPersistencePort.findProcessedIdsGroupedByQuestionnaireSince(since); + List partitionIds = idsByQuestionnaire.stream() + .map(GroupedInterrogation::partitionOrCampaignId).toList(); + List partitionIdsWithReview = dataProcessingContextPersistancePort + .findByPartitionIds(partitionIds).stream() + .filter(DataProcessingContextModel::isWithReview) + .map(DataProcessingContextModel::getPartitionId) + .toList(); + return idsByQuestionnaire.stream() + .filter(g -> partitionIdsWithReview.contains(g.partitionOrCampaignId())) .collect(Collectors.toMap( - GroupedInterrogation::questionnaireId, - GroupedInterrogation::interrogationIds - )); + GroupedInterrogation::questionnaireId, + GroupedInterrogation::interrogationIds + )); } @Override - public Page findRawDataByQuestionnaireId(String questionnaireId, Pageable pageable) { + public Page findRawDataByQuestionnaireId( + String questionnaireId, Pageable pageable + ) { return lunaticJsonRawDataPersistencePort.findRawDataByQuestionnaireId(questionnaireId, pageable); } @@ -577,20 +241,42 @@ public boolean existsByInterrogationId(String interrogationId) { } @Override - public Page findRawDataByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable){ - return lunaticJsonRawDataPersistencePort.findByCampaignIdAndDate(campaignId,startDt, endDt,pageable); - + public Page findRawDataByCampaignIdAndDate( + String campaignId, Instant startDt, Instant endDt, Pageable pageable + ) { + return lunaticJsonRawDataPersistencePort.findByCampaignIdAndDate(campaignId, startDt, endDt, pageable); } - //Utils - protected static String getValueString(Object value) { - if (value instanceof Double || value instanceof Float) { - BigDecimal bd = new BigDecimal(value.toString()); - return bd.stripTrailingZeros().toPlainString(); + private VariablesMap getVariablesMap( + String questionnaireId, Mode mode, List errors + ) throws GenesisException { + VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists( + questionnaireId, questionnaireId, mode, fileUtils, errors + ).getVariables(); + if (variablesMap == null) { + throw new GenesisException(400, + "Error during metadata parsing for mode %s :%n%s" + .formatted(mode, errors.getLast().getMessage())); } - if (value instanceof Number) { - return value.toString(); + return variablesMap; + } + + private boolean hasValidModesWithSpecs(String questionnaireId) { + Set modes = lunaticJsonRawDataPersistencePort.findModesByQuestionnaire(questionnaireId); + if (modes.isEmpty()) return false; + return modes.stream().allMatch(mode -> isSpecsPresentForMode(questionnaireId, mode)); + } + + private boolean isSpecsPresentForMode(String questionnaireId, Mode mode) { + List errors = new ArrayList<>(); + try { + MetadataModel metadata = metadataService.loadAndSaveIfNotExists( + questionnaireId, questionnaireId, mode, fileUtils, errors + ); + return metadata != null && errors.isEmpty(); + } catch (GenesisException e) { + log.warn("Genesis exception for questionnaire {} and mode {}, excluding.", questionnaireId, mode); + return false; } - return String.valueOf(value); } -} +} \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java index 9635a4f16..09850450e 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java @@ -2,349 +2,149 @@ import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.Constants; import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.utils.ControllerUtils; -import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; -import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.domain.ports.spi.RawResponsePersistencePort; -import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; -import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; -import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; -import fr.insee.genesis.domain.utils.GroupUtils; -import fr.insee.genesis.domain.utils.JsonUtils; +import fr.insee.genesis.domain.utils.RawResponseConverter; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; +import fr.insee.genesis.exceptions.InvalidMetadataException; +import fr.insee.genesis.exceptions.UndefinedMetadataException; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.modelefiliere.ModeDto; -import fr.insee.modelefiliere.RawResponseDto; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; -import java.io.IOException; import java.time.Instant; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import static fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService.getValueString; - @Service @Slf4j -public class RawResponseService implements RawResponseApiPort { +public class RawResponseService implements RawResponseApiPort { private final ControllerUtils controllerUtils; private final QuestionnaireMetadataService metadataService; private final SurveyUnitService surveyUnitService; - private final SurveyUnitQualityService surveyUnitQualityService; - private final SurveyUnitQualityToolPort surveyUnitQualityToolPort; - private final DataProcessingContextService dataProcessingContextService; + private final SurveyUnitQualityToolService surveyUnitQualityToolService; private final FileUtils fileUtils; private final Config config; @Qualifier("rawResponseMongoAdapter") private final RawResponsePersistencePort rawResponsePersistencePort; - public RawResponseService(ControllerUtils controllerUtils, QuestionnaireMetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, SurveyUnitQualityToolPort surveyUnitQualityToolPort, DataProcessingContextService dataProcessingContextService, FileUtils fileUtils, Config config, RawResponsePersistencePort rawResponsePersistencePort) { + public RawResponseService( + ControllerUtils controllerUtils, + QuestionnaireMetadataService metadataService, + SurveyUnitService surveyUnitService, + SurveyUnitQualityToolService surveyUnitQualityToolService, + FileUtils fileUtils, + Config config, + RawResponsePersistencePort rawResponsePersistencePort + ) { this.controllerUtils = controllerUtils; this.metadataService = metadataService; this.surveyUnitService = surveyUnitService; - this.surveyUnitQualityService = surveyUnitQualityService; - this.surveyUnitQualityToolPort = surveyUnitQualityToolPort; - this.dataProcessingContextService = dataProcessingContextService; + this.surveyUnitQualityToolService = surveyUnitQualityToolService; this.fileUtils = fileUtils; this.config = config; this.rawResponsePersistencePort = rawResponsePersistencePort; } @Override - public List getRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList) { - return rawResponsePersistencePort.findRawResponses(collectionInstrumentId,mode,interrogationIdList); + public DataProcessResult processRawResponsesByInterrogationIds(String collectionInstrumentId) { + List interrogationIds = rawResponsePersistencePort + .findUnprocessedInterrogationIdsByCollectionInstrumentId(collectionInstrumentId) + .stream().toList(); + return processRawResponsesByInterrogationIds(collectionInstrumentId, interrogationIds, new ArrayList<>()); } @Override - public List getRawResponsesByInterrogationID(String interrogationId) { - return rawResponsePersistencePort.findRawResponsesByInterrogationID(interrogationId); - } + public DataProcessResult processRawResponsesByInterrogationIds( + String collectionInstrumentId, + List interrogationIdList, + List errors + ) { + List modes = controllerUtils.getModesList(collectionInstrumentId); + boolean resolvedWithReviewValue = surveyUnitQualityToolService.resolveWithReviewValue(collectionInstrumentId); - @Override - public DataProcessResult processRawResponses(String collectionInstrumentId, List interrogationIdList, List errors) throws GenesisException { - int dataCount=0; - int formattedDataCount=0; - DataProcessingContextModel dataProcessingContext = - dataProcessingContextService.getContextByCollectionInstrumentId(collectionInstrumentId); - List modesList = controllerUtils.getModesList(collectionInstrumentId, null); - for (Mode mode : modesList) { - //Load and save metadata into database, throw exception if none - VariablesMap variablesMap = getVariablesMap(collectionInstrumentId,mode,errors); - int totalBatchs = Math.ceilDiv(interrogationIdList.size() , config.getRawDataProcessingBatchSize()); - int batchNumber = 1; + int batchSize = config.getRawDataProcessingBatchSize(); + int totalBatches = Math.ceilDiv(interrogationIdList.size(), batchSize); + int dataCount = 0; + int formattedDataCount = 0; + + for (Mode mode : modes) { + VariablesMap variablesMap = loadAndSaveMetadata(collectionInstrumentId, mode, errors); List interrogationIdListForMode = new ArrayList<>(interrogationIdList); - while(!interrogationIdListForMode.isEmpty()){ - log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); - int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); - List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); - - List rawResponseModels = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess); - - List surveyUnitModels = convertRawResponse( - rawResponseModels, - variablesMap - ); - - //Save converted data - surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); - surveyUnitService.saveSurveyUnits(surveyUnitModels); - - //Update process dates - updateProcessDates(surveyUnitModels); - - //Increment data count - dataCount += surveyUnitModels.size(); - formattedDataCount += surveyUnitModels.stream() - .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)) - .toList() - .size(); - - //Send processed ids grouped by questionnaire (if review activated) - if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { - sendProcessedIdsToQualityTool(surveyUnitModels); - } else { - log.warn("Data processing context not found for collection instrument {}. Ids processed not send to quality tool.",collectionInstrumentId); - } + int batchNumber = 1; - //Remove processed ids from list - interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); + while (!interrogationIdListForMode.isEmpty()) { + log.info("Processing raw data batch {}/{}", batchNumber, totalBatches); - batchNumber++; - } - } - return new DataProcessResult(dataCount, formattedDataCount, errors); - } + int maxIndex = Math.min(interrogationIdListForMode.size(), batchSize); + List batch = interrogationIdListForMode.subList(0, maxIndex); - @Override - public DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException { - int dataCount=0; - int formattedDataCount=0; - DataProcessingContextModel dataProcessingContext = - dataProcessingContextService.getContextByCollectionInstrumentId(collectionInstrumentId); - List errors = new ArrayList<>(); + List rawModels = rawResponsePersistencePort + .findRawResponses(collectionInstrumentId, mode, batch); + rawModels.removeIf(r -> r.processDate() != null); - List modesList = controllerUtils.getModesList(collectionInstrumentId, null); - for (Mode mode : modesList) { - //Load and save metadata into database, throw exception if none - VariablesMap variablesMap = getVariablesMap(collectionInstrumentId,mode,errors); - Set interrogationIds = - rawResponsePersistencePort.findUnprocessedInterrogationIdsByCollectionInstrumentId(collectionInstrumentId); + List surveyUnits = RawResponseConverter.convertRawData(rawModels, variablesMap); - int totalBatchs = Math.ceilDiv(interrogationIds.size() , config.getRawDataProcessingBatchSize()); - int batchNumber = 1; - List interrogationIdListForMode = new ArrayList<>(interrogationIds); - while(!interrogationIdListForMode.isEmpty()){ - log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); - int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); - - List surveyUnitModels = getConvertedSurveyUnits( - collectionInstrumentId, - mode, - interrogationIdListForMode, - maxIndex, - variablesMap); - - //Save converted data - surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); - surveyUnitService.saveSurveyUnits(surveyUnitModels); - - //Update process dates - updateProcessDates(surveyUnitModels); - - //Increment data count - dataCount += surveyUnitModels.size(); - formattedDataCount += surveyUnitModels.stream() - .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)) - .toList() - .size(); - - //Send processed ids grouped by questionnaire (if review activated) - if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { - sendProcessedIdsToQualityTool(surveyUnitModels); + surveyUnitService.saveSurveyUnits(surveyUnits); + updateProcessDates(surveyUnits); + + dataCount += surveyUnits.size(); + formattedDataCount += (int) surveyUnits.stream() + .filter(su -> su.getState().equals(DataState.FORMATTED)) + .count(); + + if (resolvedWithReviewValue) { + surveyUnitQualityToolService.sendProcessedIdsToQualityTool(surveyUnits); } - //Remove processed ids from list interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); batchNumber++; } } - return new DataProcessResult(dataCount, formattedDataCount, errors); - } - - private List getConvertedSurveyUnits(String collectionInstrumentId, Mode mode, List interrogationIdListForMode, int maxIndex, VariablesMap variablesMap) { - List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); - List rawResponseModels = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess); - return convertRawResponse( - rawResponseModels, - variablesMap - ); - } - - private VariablesMap getVariablesMap(String collectionInstrumentId, Mode mode, List errors) throws GenesisException { - VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(collectionInstrumentId, collectionInstrumentId, mode, fileUtils, - errors).getVariables(); - if (variablesMap == null) { - throw new GenesisException(400, - "Error during metadata parsing for mode %s :%n%s" - .formatted(mode, errors.getLast().getMessage()) - ); - } - return variablesMap; - } - - @Override - public List convertRawResponse(List rawResponseModels, VariablesMap variablesMap) { - //Convert to genesis model - List surveyUnitModels = new ArrayList<>(); - List emptySurveyUnitModels = new ArrayList<>(); - //For each possible data state (we receive COLLECTED or EDITED) - for(DataState dataState : List.of(DataState.COLLECTED,DataState.EDITED)){ - for (RawResponseModel rawResponseModel : rawResponseModels) { - //Get optional fields - Boolean isCapturedIndirectly = getIsCapturedIndirectly(rawResponseModel); - String questionnaireStateString = getStringFieldInPayload(rawResponseModel, "questionnaireState"); - RawResponseDto.QuestionnaireStateEnum questionnaireStateEnum = null; - try{ - questionnaireStateEnum = RawResponseDto.QuestionnaireStateEnum.valueOf(questionnaireStateString); - } catch (IllegalArgumentException iae){ - log.warn("'{}' is not a valid questionnaire state according to filiere model", questionnaireStateString); - } - LocalDateTime validationDate = getValidationDate(rawResponseModel); - String usualSurveyUnitId = getStringFieldInPayload(rawResponseModel,"usualSurveyUnitId"); - String majorModelVersion = getStringFieldInPayload(rawResponseModel, "majorModelVersion"); - - SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .collectionInstrumentId(rawResponseModel.collectionInstrumentId()) - .majorModelVersion(majorModelVersion) - .mode(rawResponseModel.mode()) - .interrogationId(rawResponseModel.interrogationId()) - .usualSurveyUnitId(usualSurveyUnitId) - .questionnaireState(questionnaireStateEnum) - .validationDate(validationDate) - .isCapturedIndirectly(isCapturedIndirectly) - .state(dataState) - .fileDate(rawResponseModel.recordDate()) - .recordDate(LocalDateTime.now()) - .collectedVariables(new ArrayList<>()) - .externalVariables(new ArrayList<>()) - .build(); - - //Data collected variables conversion - convertRawDataCollectedVariables(rawResponseModel, surveyUnitModel, dataState, variablesMap); - - //External variables conversion into COLLECTED document - if(dataState == DataState.COLLECTED){ - convertRawDataExternalVariables(rawResponseModel, surveyUnitModel, variablesMap); - } - - boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty() - && surveyUnitModel.getExternalVariables().isEmpty(); - if(hasNoVariable){ - if(surveyUnitModel.getState() == DataState.COLLECTED){ - log.warn("No collected or external variable for interrogation {}, raw data is ignored.", rawResponseModel.interrogationId()); - } - emptySurveyUnitModels.add(surveyUnitModel); - continue;// don't add suModel - } - surveyUnitModels.add(surveyUnitModel); - } - } - if(!emptySurveyUnitModels.isEmpty()){ - updateProcessDates(emptySurveyUnitModels); - } - return surveyUnitModels; + return new DataProcessResult(dataCount, formattedDataCount, errors); } @Override public List getUnprocessedCollectionInstrumentIds() { - List unprocessedCollectionInstrumentIds = rawResponsePersistencePort.getUnprocessedCollectionIds(); - List unprocessedCollectionInstrumentIdsWithSpecs = new ArrayList<>(); - for (String unprocessedCollectionInstrumentId : unprocessedCollectionInstrumentIds){ - Set modes = new HashSet<>(rawResponsePersistencePort.findModesByCollectionInstrument(unprocessedCollectionInstrumentId)); - if (modes.isEmpty()){ - continue; - } - - boolean areAllSpecsOK = true; - if(modes.contains(null) && modes.size() == 1){ - areAllSpecsOK = false; - } - for(ModeDto modeDto : modes){ - if(modeDto == null){ - continue; - } - Mode mode = Mode.getEnumFromJsonName(modeDto.toString()); - if(!isSpecsPresentForCollectionInstrumentAndMode(unprocessedCollectionInstrumentId, mode)){ - areAllSpecsOK = false; - } - } - if(areAllSpecsOK){ - unprocessedCollectionInstrumentIdsWithSpecs.add(unprocessedCollectionInstrumentId); - } - } - - return unprocessedCollectionInstrumentIdsWithSpecs; - } - - private boolean isSpecsPresentForCollectionInstrumentAndMode(String unprocessedCollectionInstrumentId, Mode mode) { - List genesisErrors = new ArrayList<>(); - MetadataModel metadataModel; - try { - metadataModel = metadataService.loadAndSaveIfNotExists( - unprocessedCollectionInstrumentId, - unprocessedCollectionInstrumentId, - mode, - fileUtils, - genesisErrors - ); - } catch (GenesisException ge) { - log.warn("Genesis exception thrown for collection instrument %s and mode %s, excluding from get collection instrument ids..." - .formatted(unprocessedCollectionInstrumentId, mode)); - return false; - } - return metadataModel != null && genesisErrors.isEmpty(); + return rawResponsePersistencePort.getUnprocessedCollectionIds().stream() + .filter(this::hasValidModesWithSpecs) + .collect(Collectors.toList()); } @Override public void updateProcessDates(List surveyUnitModels) { - Set collectionInstrumentIds = new HashSet<>(); - for (SurveyUnitModel surveyUnitModel : surveyUnitModels) { - collectionInstrumentIds.add(surveyUnitModel.getCollectionInstrumentId()); - } - - for (String collectionInstrumentId : collectionInstrumentIds) { - Set interrogationIds = surveyUnitModels.stream() - .filter(su -> su.getCollectionInstrumentId().equals(collectionInstrumentId)) - .map(SurveyUnitModel::getInterrogationId) - .collect(Collectors.toSet()); - rawResponsePersistencePort.updateProcessDates(collectionInstrumentId, interrogationIds); - } + surveyUnitModels.stream() + .map(SurveyUnitModel::getCollectionInstrumentId) + .distinct() + .forEach(id -> { + Set ids = surveyUnitModels.stream() + .filter(su -> su.getCollectionInstrumentId().equals(id)) + .map(SurveyUnitModel::getInterrogationId) + .collect(Collectors.toSet()); + rawResponsePersistencePort.updateProcessDates(id, ids); + }); } @Override @@ -352,193 +152,11 @@ public boolean existsByInterrogationId(String interrogationId) { return rawResponsePersistencePort.existsByInterrogationId(interrogationId); } - private Map> getProcessedIdsMap(List surveyUnitModels) { - Map> processedInterrogationIdsPerQuestionnaire = new HashMap<>(); - surveyUnitModels.forEach(model -> - processedInterrogationIdsPerQuestionnaire - .computeIfAbsent(model.getCollectionInstrumentId(), k -> new HashSet<>()) - .add(model.getInterrogationId()) - ); - return processedInterrogationIdsPerQuestionnaire; - } - - private void sendProcessedIdsToQualityTool(List surveyUnitModels) { - try { - Map> processedIdsMap = getProcessedIdsMap(surveyUnitModels); - ResponseEntity response = - surveyUnitQualityToolPort.sendProcessedIds(processedIdsMap); - - if (response.getStatusCode().is2xxSuccessful()) { - log.info("Successfully sent {} ids to quality tool", processedIdsMap.size()); - }else{ - log.warn("Survey unit quality tool responded non-2xx code {} and body {}", - response.getStatusCode(), response.getBody()); - } - }catch (IOException e){ - log.error("Error during Perret call request building : {}", e.toString()); - } - } - - private static Boolean getIsCapturedIndirectly(RawResponseModel rawResponseModel) { - try{ - return rawResponseModel.payload().get("isCapturedIndirectly") == null ? null : - Boolean.parseBoolean(rawResponseModel.payload().get("isCapturedIndirectly").toString()); - }catch(Exception e){ - log.warn("Exception when parsing isCapturedIndirectly : {}",e.toString()); - return Boolean.FALSE; - } - } - - private static LocalDateTime getValidationDate(RawResponseModel rawResponseModel) { - try{ - return rawResponseModel.payload().get("validationDate") == null ? null : - LocalDateTime.parse(rawResponseModel.payload().get("validationDate").toString(), DateTimeFormatter.ISO_OFFSET_DATE_TIME); - }catch(Exception e){ - log.warn("Exception when parsing validation date : {}",e.toString()); - return null; - } - } - - private static String getStringFieldInPayload(RawResponseModel rawResponseModel, String field) { - try{ - return rawResponseModel.payload().get(field).toString(); - }catch(Exception e){ - log.warn("Exception when parsing {} : {}",field, e.toString()); - return null; - } - } - - @SuppressWarnings("unchecked") - private void convertRawDataCollectedVariables( - RawResponseModel rawResponseModel, - SurveyUnitModel dstSurveyUnitModel, - DataState dataState, - VariablesMap variablesMap - ) { - Map dataMap = rawResponseModel.payload(); - dataMap = (Map) dataMap.get("data"); - - dataMap = (Map)dataMap.get("COLLECTED"); - - - Map collectedMap = JsonUtils.asMap(dataMap); - if (collectedMap == null || collectedMap.isEmpty()){ - if(dataState.equals(DataState.COLLECTED)) { - log.warn("No collected data for interrogation {}", rawResponseModel.interrogationId()); - } - return; - } - convertToCollectedVar(dstSurveyUnitModel, dataState, variablesMap, collectedMap); - } - - private static void convertToCollectedVar( - SurveyUnitModel dstSurveyUnitModel, - DataState dataState, - VariablesMap variablesMap, - Map collectedMap - ) { - final String stateKey = dataState.toString(); - final var collectedVariables = dstSurveyUnitModel.getCollectedVariables(); - - for (Map.Entry collectedVariable : collectedMap.entrySet()) { - processCollectedVariable(collectedVariable, stateKey, variablesMap, dstSurveyUnitModel, collectedVariables); - } - } - - - static void processCollectedVariable( - Map.Entry entry, - String stateKey, - VariablesMap variablesMap, - SurveyUnitModel dstSurveyUnitModel, - List variableModelList - ) { - if (Constants.PAIRWISES.equals(entry.getKey())) { - handlePairwiseCollectedVariable(entry, DataState.valueOf(stateKey), variablesMap, dstSurveyUnitModel); - return; - } - - Map states = JsonUtils.asMap(entry.getValue()); - if (states == null) return; - - Object value = states.get(stateKey); - if (value == null) return; - - if (value instanceof List list) { - convertListVar(list, entry, variablesMap, variableModelList); - } else { - convertOneVar(entry, getValueString(value), variablesMap, 1, variableModelList); - } - } - - - private static void convertListVar(Object valuesForState, Map.Entry collectedVariable, VariablesMap variablesMap, List dstSurveyUnitModel) { - List values = JsonUtils.asStringList(valuesForState); - if (!values.isEmpty()) { - int iteration = 1; - for (String value : values) { - if (value != null && !value.isEmpty()) { - convertOneVar(collectedVariable, value, variablesMap, iteration, dstSurveyUnitModel); - } - iteration++; - } - } - } - - private static void convertOneVar(Map.Entry externalVariableEntry, String valueObject, VariablesMap variablesMap, int iteration, List dstSurveyUnitModel) { - VariableModel externalVariableModel = VariableModel.builder() - .varId(externalVariableEntry.getKey()) - .value(valueObject) - .scope(getIdLoop(variablesMap, externalVariableEntry.getKey())) - .iteration(iteration) - .parentId(GroupUtils.getParentGroupName(externalVariableEntry.getKey(), variablesMap)) - .build(); - dstSurveyUnitModel.add(externalVariableModel); - } - - private static String getIdLoop(VariablesMap variablesMap, String variableName) { - if (variablesMap.getVariable(variableName) == null) { - log.warn("Variable {} not present in metadata, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); - return Constants.ROOT_GROUP_NAME; - } - return variablesMap.getVariable(variableName).getGroupName(); - } - - @SuppressWarnings("unchecked") - private static void convertRawDataExternalVariables( - RawResponseModel rawResponseModel, - SurveyUnitModel dstSurveyUnitModel, - VariablesMap variablesMap - ) { - Map dataMap = rawResponseModel.payload(); - dataMap = (Map) dataMap.get("data"); - - - dataMap = (Map)dataMap.get("EXTERNAL"); - Map externalMap = JsonUtils.asMap(dataMap); - if (externalMap != null && !externalMap.isEmpty()){ - convertToExternalVar(dstSurveyUnitModel, variablesMap, externalMap); - } - } - - private static void convertToExternalVar(SurveyUnitModel dstSurveyUnitModel, VariablesMap variablesMap, Map externalMap) { - for(Map.Entry externalVariableEntry : externalMap.entrySet()){ - Object valueObject = externalVariableEntry.getValue(); - if (valueObject instanceof List){ - //Array of values - convertListVar(valueObject, externalVariableEntry, variablesMap, dstSurveyUnitModel.getExternalVariables()); - continue; - } - //Value - if (valueObject != null) { - convertOneVar(externalVariableEntry, valueObject.toString(), variablesMap, 1, dstSurveyUnitModel.getExternalVariables()); - } - } - } - @Override - public Page findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) { - return rawResponsePersistencePort.findByCampaignIdAndDate(campaignId,startDate, endDate,pageable); + public Page findRawResponseDataByCampaignIdAndDate( + String campaignId, Instant startDate, Instant endDate, Pageable pageable + ) { + return rawResponsePersistencePort.findByCampaignIdAndDate(campaignId, startDate, endDate, pageable); } @Override @@ -557,77 +175,59 @@ public Set getDistinctCollectionInstrumentIds() { } @Override - public Page findRawResponseDataByCollectionInstrumentId(String collectionInstrumentId, Pageable pageable) { + public Page findRawResponseDataByCollectionInstrumentId( + String collectionInstrumentId, Pageable pageable + ) { return rawResponsePersistencePort.findByCollectionInstrumentId(collectionInstrumentId, pageable); } - - @SuppressWarnings("unchecked") - static void handlePairwiseCollectedVariable( - Map.Entry collectedVariable, - DataState dataState, - VariablesMap variablesMap, - SurveyUnitModel dstSurveyUnitModel + /** Load and save metadata into database, throw exception if none. */ + private VariablesMap loadAndSaveMetadata( + String collectionInstrumentId, Mode mode, List errors ) { - Object value = getValueForState(collectedVariable, dataState.toString()); - - if (isInvalidPairwiseVariable(value, variablesMap)) { - return; - } - - List individuals = (List) value; - - String groupName = variablesMap - .getVariable(Constants.PAIRWISE_PREFIX + 1) - .getGroupName(); - - for (int individualIndex = 0; individualIndex < individuals.size(); individualIndex++) { - List individualLinks = (List) individuals.get(individualIndex); - - for (int linkIndex = 1; linkIndex < Constants.MAX_LINKS_ALLOWED; linkIndex++) { - dstSurveyUnitModel.getCollectedVariables().add( - buildPairwiseVariable(individualLinks, linkIndex, individualIndex+ 1, groupName) - ); - } + VariablesMap variablesMap; + try { + variablesMap = metadataService.loadAndSaveIfNotExists( + collectionInstrumentId, collectionInstrumentId, mode, fileUtils, errors + ).getVariables(); + } catch (GenesisException e) { + throw new UndefinedMetadataException( + "Cannot load metadata for collection instrument %s and mode %s." + .formatted(collectionInstrumentId, mode), e); } - } - - private static VariableModel buildPairwiseVariable( - List individualLinks, - int linkIndex, - int iteration, - String groupName - ) { - String value = Constants.NO_PAIRWISE_VALUE; - - if (linkIndex <= individualLinks.size()) { - String v = individualLinks.get(linkIndex - 1); - value = (v == null || v.isBlank()) - ? Constants.SAME_AXIS_VALUE - : v; + if (variablesMap == null) { + throw new InvalidMetadataException( + "Error during metadata parsing for mode %s :%n%s" + .formatted(mode, errors.getLast().getMessage())); } - - return VariableModel.builder() - .varId(Constants.PAIRWISE_PREFIX + linkIndex) - .value(value) - .scope(groupName) - .iteration(iteration) - .parentId(Constants.ROOT_GROUP_NAME) - .build(); + return variablesMap; } + private boolean hasValidModesWithSpecs(String collectionInstrumentId) { + Set modes = new HashSet<>( + rawResponsePersistencePort.findModesByCollectionInstrument(collectionInstrumentId) + ); + if (modes.isEmpty()) return false; + if (modes.contains(null) && modes.size() == 1) return false; - private static Object getValueForState( - Map.Entry collectedVariable, - String stateKey - ) { - Map states = JsonUtils.asMap(collectedVariable.getValue()); - return states != null ? states.get(stateKey) : null; + return modes.stream() + .filter(Objects::nonNull) + .map(m -> Mode.getEnumFromJsonName(m.toString())) + .allMatch(mode -> isSpecsPresentForMode(collectionInstrumentId, mode)); } - private static boolean isInvalidPairwiseVariable(Object value, VariablesMap variablesMap) { - return !(value instanceof List) || !variablesMap.hasVariable(Constants.PAIRWISE_PREFIX + 1); + private boolean isSpecsPresentForMode(String collectionInstrumentId, Mode mode) { + List errors = new ArrayList<>(); + try { + MetadataModel metadata = metadataService.loadAndSaveIfNotExists( + collectionInstrumentId, collectionInstrumentId, mode, fileUtils, errors + ); + return metadata != null && errors.isEmpty(); + } catch (GenesisException e) { + log.warn("Genesis exception for collection instrument {} and mode {}, excluding.", + collectionInstrumentId, mode); + return false; + } } - -} +} \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/ReprocessRawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/ReprocessRawResponseService.java new file mode 100644 index 000000000..f0e2c10eb --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/ReprocessRawResponseService.java @@ -0,0 +1,118 @@ +package fr.insee.genesis.domain.service.rawdata; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; +import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.domain.ports.api.RawResponseApiPort; +import fr.insee.genesis.domain.ports.api.ReprocessRawResponseApiPort; +import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistencePort; +import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistenceRouter; +import fr.insee.genesis.domain.ports.spi.SurveyUnitPersistencePort; +import fr.insee.genesis.exceptions.GenesisException; +import fr.insee.genesis.exceptions.InvalidDateIntervalException; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Service; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Set; + +@Service +@RequiredArgsConstructor +@Slf4j +public class ReprocessRawResponseService implements ReprocessRawResponseApiPort { + + private final SurveyUnitPersistencePort surveyUnitPersistence; + + private final RawResponseApiPort rawResponseService; + private final LunaticJsonRawDataApiPort lunaticJsonRawDataService; + // Polymorphism for the raw data services would cause too much refactor. + + private final RawResponseReprocessPersistenceRouter rawResponseReprocessPersistenceRouter; + private RawResponseReprocessPersistencePort rawResponseReprocessPersistencePort; + + private enum InputFilterType { + COLLECTION_ID, + COLLECTION_ID_AND_DATE + } + + @Override + public DataProcessResult reprocessRawResponses( + @NonNull RawDataModelType rawDataModelType, + @NonNull String collectionInstrumentId, + Instant sinceDate, + Instant endDate) throws GenesisException { + + log.info("Start reprocess {} data for collectionInstrumentId={}, sinceDate={}, endDate={}", + rawDataModelType, collectionInstrumentId, sinceDate, endDate); + + InputFilterType inputFilterType = validateInputs(sinceDate, endDate); + + rawResponseReprocessPersistencePort = rawResponseReprocessPersistenceRouter.resolve(rawDataModelType); + + Set interrogationIds = switch (inputFilterType) { + case COLLECTION_ID -> + rawResponseReprocessPersistencePort + .findProcessedInterrogationIdsByCollectionInstrumentId( + collectionInstrumentId); + case COLLECTION_ID_AND_DATE -> + rawResponseReprocessPersistencePort + .findProcessedInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween( + collectionInstrumentId, + sinceDate, + effectiveEndDate(endDate)); + }; + + return reprocessInterrogations(rawDataModelType, collectionInstrumentId, interrogationIds); + } + + private static InputFilterType validateInputs(Instant sinceDate, Instant endDate) { + if (bothDatesAreNull(sinceDate, endDate)) { + return InputFilterType.COLLECTION_ID; + } + if (sinceDate == null) { + throw new InvalidDateIntervalException("'endDate' cannot be provided without 'sinceDate'."); + } + if (endIsBeforeSince(sinceDate, endDate)) { + throw new InvalidDateIntervalException("'endDate' value cannot be before 'sinceDate'."); + } + return InputFilterType.COLLECTION_ID_AND_DATE; + } + + private static boolean endIsBeforeSince(Instant sinceDate, Instant endDate) { + return endDate != null && endDate.isBefore(sinceDate); + } + + private static boolean bothDatesAreNull(Instant sinceDate, Instant endDate) { + return sinceDate == null && endDate == null; + } + + private static Instant effectiveEndDate(Instant endDate) { + if (endDate != null) + return endDate; + var now = Instant.now(); + log.info("Effective end date: {}", now); + return now; + } + + private DataProcessResult reprocessInterrogations( + RawDataModelType rawDataModelType, String collectionInstrumentId, Set interrogationIds) + throws GenesisException { + if (interrogationIds.isEmpty()) { + return new DataProcessResult(0, 0, new ArrayList<>()); + } + + surveyUnitPersistence.deleteByCollectionInstrumentIdAndInterrogationIds(collectionInstrumentId, interrogationIds); + rawResponseReprocessPersistencePort.resetProcessDates(collectionInstrumentId, interrogationIds); + + return switch (rawDataModelType) { + case FILIERE -> rawResponseService.processRawResponsesByInterrogationIds( + collectionInstrumentId, new ArrayList<>(interrogationIds), new ArrayList<>()); + case LEGACY -> lunaticJsonRawDataService.processRawDataByInterrogationIds( + collectionInstrumentId, new ArrayList<>(interrogationIds), new ArrayList<>()); + }; + } + +} diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitQualityToolService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitQualityToolService.java new file mode 100644 index 000000000..4e6452fd4 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitQualityToolService.java @@ -0,0 +1,73 @@ +package fr.insee.genesis.domain.service.surveyunit; + +import fr.insee.genesis.domain.model.context.DataProcessingContextModel; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Service +@Slf4j +public class SurveyUnitQualityToolService { + + private final SurveyUnitQualityToolPort surveyUnitQualityToolPort; + private final DataProcessingContextService dataProcessingContextService; + + public SurveyUnitQualityToolService( + SurveyUnitQualityToolPort surveyUnitQualityToolPort, + DataProcessingContextService dataProcessingContextService + ) { + this.surveyUnitQualityToolPort = surveyUnitQualityToolPort; + this.dataProcessingContextService = dataProcessingContextService; + } + + /** + * Returns the value of the 'withReview' property in the context object. + * dataProcessingContext {@link DataProcessingContextModel} + * @param collectionInstrumentId Passed for logging purposes. + * @return The 'withReview' value, false if context is null. + */ + public boolean resolveWithReviewValue(String collectionInstrumentId) { + DataProcessingContextModel dataProcessingContext = + dataProcessingContextService.getContextByCollectionInstrumentId(collectionInstrumentId); + if (dataProcessingContext == null) { + log.warn("Data processing context not found for collection instrument {}. " + + "Ids processed not sent to quality tool.", collectionInstrumentId); + return false; + } + return dataProcessingContext.isWithReview(); + } + + public void sendProcessedIdsToQualityTool(List surveyUnitModels) { + try { + Map> idsPerQuestionnaire = getProcessedIdsMap(surveyUnitModels); + ResponseEntity response = surveyUnitQualityToolPort.sendProcessedIds(idsPerQuestionnaire); + + if (response.getStatusCode().is2xxSuccessful()) { + log.info("Successfully sent {} ids to quality tool", idsPerQuestionnaire.size()); + } else { + log.warn("Survey unit quality tool responded non-2xx code {} and body {}", + response.getStatusCode(), response.getBody()); + } + } catch (IOException e) { + log.error("Error during Perret call request building : {}", e.toString()); + } + } + + private static Map> getProcessedIdsMap(List models) { + Map> processedInterrogationIdsPerQuestionnaire = new HashMap<>(); + models.forEach(m -> processedInterrogationIdsPerQuestionnaire + .computeIfAbsent(m.getCollectionInstrumentId(), k -> new HashSet<>()) + .add(m.getInterrogationId())); + return processedInterrogationIdsPerQuestionnaire; + } +} diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index 5be97c319..00f56fd66 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -2,20 +2,8 @@ import fr.insee.bpm.metadata.model.VariableType; import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire; -import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign; -import fr.insee.genesis.controller.dto.SurveyUnitDto; -import fr.insee.genesis.controller.dto.SurveyUnitInputDto; -import fr.insee.genesis.controller.dto.SurveyUnitSimplifiedDto; -import fr.insee.genesis.controller.dto.VariableDto; -import fr.insee.genesis.controller.dto.VariableInputDto; -import fr.insee.genesis.controller.dto.VariableStateDto; -import fr.insee.genesis.domain.model.surveyunit.DataState; -import fr.insee.genesis.domain.model.surveyunit.InterrogationId; -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; -import fr.insee.genesis.domain.model.surveyunit.VarIdScopeTuple; -import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.controller.dto.*; +import fr.insee.genesis.domain.model.surveyunit.*; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import fr.insee.genesis.domain.ports.spi.SurveyUnitPersistencePort; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; @@ -29,14 +17,9 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; +import java.time.Instant; import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; @Service @@ -397,9 +380,18 @@ public List findDistinctInterrogationIdsByQuestionnaireIdAndDat } @Override - public List findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String collectionInstrumentId, LocalDateTime start, LocalDateTime end) { + public List findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween( + String collectionInstrumentId, + Instant start, + Instant end + ) { + return surveyUnitPersistencePort - .findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(collectionInstrumentId,start,end) + .findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween( + collectionInstrumentId, + start, + end + ) .stream() .map(su -> new InterrogationId(su.getInterrogationId())) .distinct() @@ -496,6 +488,14 @@ public Long deleteByCollectionInstrumentId(String collectionInstrumentId) { return surveyUnitPersistencePort.deleteByCollectionInstrumentId(collectionInstrumentId); } + @Override + public Long deleteByQuestionnaireIdAndInterrogationIds( + String questionnaireId, + Set interrogationIds + ) { + return surveyUnitPersistencePort.deleteByQuestionnaireIdAndInterrogationIds(questionnaireId, interrogationIds); + } + @Override public long countResponses() { return surveyUnitPersistencePort.count(); diff --git a/src/main/java/fr/insee/genesis/domain/utils/LunaticJsonRawDataConverter.java b/src/main/java/fr/insee/genesis/domain/utils/LunaticJsonRawDataConverter.java new file mode 100644 index 000000000..8278a588a --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/utils/LunaticJsonRawDataConverter.java @@ -0,0 +1,213 @@ +package fr.insee.genesis.domain.utils; + +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.model.surveyunit.DataState; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; +import lombok.extern.slf4j.Slf4j; + +import java.math.BigDecimal; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static fr.insee.genesis.domain.utils.LunaticJsonRawDataParser.parseIsCapturedIndirectly; +import static fr.insee.genesis.domain.utils.LunaticJsonRawDataParser.parseValidationDate; + +@Slf4j +public final class LunaticJsonRawDataConverter { + + private LunaticJsonRawDataConverter() {} + + /** + * Converts a list of LunaticJsonRawDataModel into SurveyUnitModels. + * Empty models (no variables) have their process dates updated immediately. + */ + public static List convertRawData( + List rawDataList, + VariablesMap variablesMap + ) { + List result = new ArrayList<>(); + List emptyModels = new ArrayList<>(); + + for (DataState dataState : List.of(DataState.COLLECTED, DataState.EDITED)) { + for (LunaticJsonRawDataModel rawData : rawDataList) { + SurveyUnitModel model = buildModel(rawData, dataState, variablesMap); + + boolean hasNoVariable = model.getCollectedVariables().isEmpty() + && model.getExternalVariables().isEmpty(); + + if (hasNoVariable) { + if (dataState == DataState.COLLECTED) { + log.warn("No collected or external variable for interrogation {}, raw data is ignored.", + rawData.interrogationId()); + } + emptyModels.add(model); + } else { + result.add(model); + } + } + } + + DataVerifier.verifySurveyUnits(result, variablesMap); + + return result; + } + + private static SurveyUnitModel buildModel( + LunaticJsonRawDataModel rawData, + DataState dataState, + VariablesMap variablesMap + ) { + RawDataModelType modelType = resolveModelType(rawData); + + SurveyUnitModel model = SurveyUnitModel.builder() + .campaignId(rawData.campaignId()) + .collectionInstrumentId(rawData.questionnaireId()) + .mode(rawData.mode()) + .interrogationId(rawData.interrogationId()) + .usualSurveyUnitId(rawData.idUE()) + .validationDate(parseValidationDate(rawData)) + .isCapturedIndirectly(parseIsCapturedIndirectly(rawData)) + .state(dataState) + .fileDate(rawData.recordDate()) + .recordDate(LocalDateTime.now()) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + convertCollectedVariables(rawData, model, dataState, modelType, variablesMap); + + if (dataState == DataState.COLLECTED) { + convertExternalVariables(rawData, model, modelType, variablesMap); + } + + return model; + } + + private static RawDataModelType resolveModelType(LunaticJsonRawDataModel rawData) { + return rawData.data().containsKey("data") + ? RawDataModelType.FILIERE + : RawDataModelType.LEGACY; + } + + @SuppressWarnings("unchecked") + private static void convertCollectedVariables( + LunaticJsonRawDataModel rawData, + SurveyUnitModel dst, + DataState dataState, + RawDataModelType modelType, + VariablesMap variablesMap + ) { + Map dataMap = rawData.data(); + if (modelType == RawDataModelType.FILIERE) { + dataMap = (Map) dataMap.get("data"); + } + + Map collectedMap = JsonUtils.asMap( + (Map) dataMap.get("COLLECTED") + ); + + if (collectedMap == null || collectedMap.isEmpty()) { + if (dataState == DataState.COLLECTED) { + log.warn("No collected data for interrogation {}", rawData.interrogationId()); + } + return; + } + + String stateKey = dataState.toString(); + for (Map.Entry entry : collectedMap.entrySet()) { + RawResponseConverter.processCollectedVariable( + entry, stateKey, variablesMap, dst, dst.getCollectedVariables() + ); + } + } + + + @SuppressWarnings("unchecked") + private static void convertExternalVariables( + LunaticJsonRawDataModel rawData, + SurveyUnitModel dst, + RawDataModelType modelType, + VariablesMap variablesMap + ) { + Map dataMap = rawData.data(); + if (modelType == RawDataModelType.FILIERE) { + dataMap = (Map) dataMap.get("data"); + } + + Map externalMap = JsonUtils.asMap( + (Map) dataMap.get("EXTERNAL") + ); + + if (externalMap == null || externalMap.isEmpty()) return; + + for (Map.Entry entry : externalMap.entrySet()) { + Object value = entry.getValue(); + if (value instanceof List) { + convertListVar(value, entry, variablesMap, dst.getExternalVariables()); + } else if (value != null) { + convertOneVar(entry, getValueString(value), variablesMap, 1, + dst.getExternalVariables()); + } + } + } + + public static String getValueString(Object value) { + if (value instanceof Double || value instanceof Float) { + BigDecimal bd = new BigDecimal(value.toString()); + return bd.stripTrailingZeros().toPlainString(); + } + if (value instanceof Number) { + return value.toString(); + } + return String.valueOf(value); + } + + + private static void convertListVar( + Object values, + Map.Entry entry, + VariablesMap variablesMap, + List target + ) { + List list = JsonUtils.asStringList(values); + int iteration = 1; + for (String value : list) { + if (value != null && !value.isEmpty()) { + convertOneVar(entry, value, variablesMap, iteration, target); + } + iteration++; + } + } + + private static void convertOneVar( + Map.Entry entry, + String value, + VariablesMap variablesMap, + int iteration, + List target + ) { + target.add(VariableModel.builder() + .varId(entry.getKey()) + .value(value) + .scope(resolveScope(variablesMap, entry.getKey())) + .iteration(iteration) + .parentId(GroupUtils.getParentGroupName(entry.getKey(), variablesMap)) + .build()); + } + + private static String resolveScope(VariablesMap variablesMap, String variableName) { + if (variablesMap.getVariable(variableName) == null) { + log.warn("Variable {} not present in metadata, assigning to {}", + variableName, Constants.ROOT_GROUP_NAME); + return Constants.ROOT_GROUP_NAME; + } + return variablesMap.getVariable(variableName).getGroupName(); + } + +} \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/utils/LunaticJsonRawDataParser.java b/src/main/java/fr/insee/genesis/domain/utils/LunaticJsonRawDataParser.java new file mode 100644 index 000000000..1836b6714 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/utils/LunaticJsonRawDataParser.java @@ -0,0 +1,35 @@ +package fr.insee.genesis.domain.utils; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; +import lombok.extern.slf4j.Slf4j; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +@Slf4j +public class LunaticJsonRawDataParser { + + private LunaticJsonRawDataParser(){} + + + static LocalDateTime parseValidationDate(LunaticJsonRawDataModel rawData) { + try { + Object val = rawData.data().get("validationDate"); + return val == null ? null : + LocalDateTime.parse(val.toString(), DateTimeFormatter.ISO_OFFSET_DATE_TIME); + } catch (Exception e) { + log.warn("Exception when parsing validation date : {}", e.toString()); + return null; + } + } + + static Boolean parseIsCapturedIndirectly(LunaticJsonRawDataModel rawData) { + try { + Object val = rawData.data().get("isCapturedIndirectly"); + return val == null ? null : Boolean.parseBoolean(val.toString()); + } catch (Exception e) { + log.warn("Exception when parsing isCapturedIndirectly : {}", e.toString()); + return Boolean.FALSE; + } + } +} diff --git a/src/main/java/fr/insee/genesis/domain/utils/RawResponseConverter.java b/src/main/java/fr/insee/genesis/domain/utils/RawResponseConverter.java new file mode 100644 index 000000000..73ec86041 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/utils/RawResponseConverter.java @@ -0,0 +1,259 @@ +package fr.insee.genesis.domain.utils; + +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.model.surveyunit.DataState; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; +import fr.insee.modelefiliere.RawResponseDto; +import lombok.extern.slf4j.Slf4j; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Slf4j +public final class RawResponseConverter { + + private RawResponseConverter() {} + + /** + * Converts a list of RawResponseModel into SurveyUnitModels. + * Empty models (no variables) have their process dates updated immediately. + */ + public static List convertRawData( + List rawResponseModels, + VariablesMap variablesMap + ) { + List result = new ArrayList<>(); + List emptyModels = new ArrayList<>(); + + for (DataState dataState : List.of(DataState.COLLECTED, DataState.EDITED)) { + for (RawResponseModel raw : rawResponseModels) { + SurveyUnitModel model = buildModel(raw, dataState, variablesMap); + + boolean hasNoVariable = model.getCollectedVariables().isEmpty() + && model.getExternalVariables().isEmpty(); + + if (hasNoVariable) { + if (dataState == DataState.COLLECTED) { + log.warn("No collected or external variable for interrogation {}, raw data is ignored.", + raw.interrogationId()); + } + emptyModels.add(model); + } else { + result.add(model); + } + } + } + + // Verify and add FORCED models if needed + DataVerifier.verifySurveyUnits(result, variablesMap); + + return result; + } + + private static SurveyUnitModel buildModel( + RawResponseModel raw, + DataState dataState, + VariablesMap variablesMap + ) { + RawResponseDto.QuestionnaireStateEnum questionnaireState = + parseQuestionnaireState(RawResponsePayloadParser.getStringField(raw, "questionnaireState")); + + SurveyUnitModel model = SurveyUnitModel.builder() + .collectionInstrumentId(raw.collectionInstrumentId()) + .majorModelVersion(RawResponsePayloadParser.getStringField(raw, "majorModelVersion")) + .mode(raw.mode()) + .interrogationId(raw.interrogationId()) + .usualSurveyUnitId(RawResponsePayloadParser.getStringField(raw, "usualSurveyUnitId")) + .questionnaireState(questionnaireState) + .validationDate(RawResponsePayloadParser.getValidationDate(raw)) + .isCapturedIndirectly(RawResponsePayloadParser.getIsCapturedIndirectly(raw)) + .state(dataState) + .fileDate(raw.recordDate()) + .recordDate(LocalDateTime.now()) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + convertCollectedVariables(raw, model, dataState, variablesMap); + + if (dataState == DataState.COLLECTED) { + convertExternalVariables(raw, model, variablesMap); + } + + return model; + } + + private static RawResponseDto.QuestionnaireStateEnum parseQuestionnaireState(String value) { + if (value == null) return null; + try { + return RawResponseDto.QuestionnaireStateEnum.valueOf(value); + } catch (IllegalArgumentException e) { + log.warn("'{}' is not a valid questionnaire state according to filiere model", value); + return null; + } + } + + @SuppressWarnings("unchecked") + private static void convertCollectedVariables( + RawResponseModel raw, + SurveyUnitModel dst, + DataState dataState, + VariablesMap variablesMap + ) { + Map dataMap = (Map) raw.payload().get("data"); + Map collectedMap = JsonUtils.asMap( + (Map) dataMap.get("COLLECTED") + ); + + if (collectedMap == null || collectedMap.isEmpty()) { + if (dataState == DataState.COLLECTED) { + log.warn("No collected data for interrogation {}", raw.interrogationId()); + } + return; + } + + String stateKey = dataState.toString(); + for (Map.Entry entry : collectedMap.entrySet()) { + processCollectedVariable(entry, stateKey, variablesMap, dst, dst.getCollectedVariables()); + } + } + + public static void processCollectedVariable( + Map.Entry entry, + String stateKey, + VariablesMap variablesMap, + SurveyUnitModel dst, + List target + ) { + if (Constants.PAIRWISES.equals(entry.getKey())) { + handlePairwise(entry, DataState.valueOf(stateKey), variablesMap, dst); + return; + } + + Map states = JsonUtils.asMap(entry.getValue()); + if (states == null) return; + + Object value = states.get(stateKey); + if (value == null) return; + + if (value instanceof List list) { + convertListVar(list, entry, variablesMap, target); + } else { + convertOneVar(entry, getValueString(value), variablesMap, 1, target); + } + } + + @SuppressWarnings("unchecked") + private static void convertExternalVariables( + RawResponseModel raw, + SurveyUnitModel dst, + VariablesMap variablesMap + ) { + Map dataMap = (Map) raw.payload().get("data"); + Map externalMap = JsonUtils.asMap( + (Map) dataMap.get("EXTERNAL") + ); + + if (externalMap == null || externalMap.isEmpty()) return; + + for (Map.Entry entry : externalMap.entrySet()) { + Object value = entry.getValue(); + if (value instanceof List) { + convertListVar(value, entry, variablesMap, dst.getExternalVariables()); + } else if (value != null) { + convertOneVar(entry, value.toString(), variablesMap, 1, dst.getExternalVariables()); + } + } + } + + @SuppressWarnings("unchecked") + static void handlePairwise( + Map.Entry entry, + DataState dataState, + VariablesMap variablesMap, + SurveyUnitModel dst + ) { + Map states = JsonUtils.asMap(entry.getValue()); + Object value = states != null ? states.get(dataState.toString()) : null; + + if (!(value instanceof List) || !variablesMap.hasVariable(Constants.PAIRWISE_PREFIX + 1)) { + return; + } + + List individuals = (List) value; + String groupName = variablesMap.getVariable(Constants.PAIRWISE_PREFIX + 1).getGroupName(); + + for (int i = 0; i < individuals.size(); i++) { + List links = (List) individuals.get(i); + for (int linkIdx = 1; linkIdx < Constants.MAX_LINKS_ALLOWED; linkIdx++) { + dst.getCollectedVariables().add(buildPairwiseVariable(links, linkIdx, i + 1, groupName)); + } + } + } + + private static VariableModel buildPairwiseVariable( + List links, int linkIndex, int iteration, String groupName + ) { + String value = Constants.NO_PAIRWISE_VALUE; + if (linkIndex <= links.size()) { + String v = links.get(linkIndex - 1); + value = (v == null || v.isBlank()) ? Constants.SAME_AXIS_VALUE : v; + } + return VariableModel.builder() + .varId(Constants.PAIRWISE_PREFIX + linkIndex) + .value(value) + .scope(groupName) + .iteration(iteration) + .parentId(Constants.ROOT_GROUP_NAME) + .build(); + } + + private static void convertListVar( + Object values, + Map.Entry entry, + VariablesMap variablesMap, + List target + ) { + List list = JsonUtils.asStringList(values); + int iteration = 1; + for (String value : list) { + if (value != null && !value.isEmpty()) { + convertOneVar(entry, value, variablesMap, iteration, target); + } + iteration++; + } + } + + private static void convertOneVar( + Map.Entry entry, + String value, + VariablesMap variablesMap, + int iteration, + List target + ) { + target.add(VariableModel.builder() + .varId(entry.getKey()) + .value(value) + .scope(resolveScope(variablesMap, entry.getKey())) + .iteration(iteration) + .parentId(GroupUtils.getParentGroupName(entry.getKey(), variablesMap)) + .build()); + } + + private static String resolveScope(VariablesMap variablesMap, String variableName) { + if (variablesMap.getVariable(variableName) == null) { + log.warn("Variable {} not present in metadata, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); + return Constants.ROOT_GROUP_NAME; + } + return variablesMap.getVariable(variableName).getGroupName(); + } + + static String getValueString(Object value) { + return value == null ? null : value.toString(); + } +} diff --git a/src/main/java/fr/insee/genesis/domain/utils/RawResponsePayloadParser.java b/src/main/java/fr/insee/genesis/domain/utils/RawResponsePayloadParser.java new file mode 100644 index 000000000..290d7cdd2 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/utils/RawResponsePayloadParser.java @@ -0,0 +1,43 @@ +package fr.insee.genesis.domain.utils; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; +import lombok.extern.slf4j.Slf4j; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +@Slf4j +public final class RawResponsePayloadParser { + + private RawResponsePayloadParser() {} + + public static Boolean getIsCapturedIndirectly(RawResponseModel rawResponseModel) { + try { + Object val = rawResponseModel.payload().get("isCapturedIndirectly"); + return val == null ? null : Boolean.parseBoolean(val.toString()); + } catch (Exception e) { + log.warn("Exception when parsing isCapturedIndirectly : {}", e.toString()); + return Boolean.FALSE; + } + } + + public static LocalDateTime getValidationDate(RawResponseModel rawResponseModel) { + try { + Object val = rawResponseModel.payload().get("validationDate"); + return val == null ? null : + LocalDateTime.parse(val.toString(), DateTimeFormatter.ISO_OFFSET_DATE_TIME); + } catch (Exception e) { + log.warn("Exception when parsing validation date : {}", e.toString()); + return null; + } + } + + public static String getStringField(RawResponseModel rawResponseModel, String field) { + try { + return rawResponseModel.payload().get(field).toString(); + } catch (Exception e) { + log.warn("Exception when parsing {} : {}", field, e.toString()); + return null; + } + } +} diff --git a/src/main/java/fr/insee/genesis/exceptions/InvalidDateIntervalException.java b/src/main/java/fr/insee/genesis/exceptions/InvalidDateIntervalException.java new file mode 100644 index 000000000..c4b7d1018 --- /dev/null +++ b/src/main/java/fr/insee/genesis/exceptions/InvalidDateIntervalException.java @@ -0,0 +1,9 @@ +package fr.insee.genesis.exceptions; + +public class InvalidDateIntervalException extends RuntimeException { + + public InvalidDateIntervalException(String message) { + super(message); + } + +} diff --git a/src/main/java/fr/insee/genesis/exceptions/InvalidMetadataException.java b/src/main/java/fr/insee/genesis/exceptions/InvalidMetadataException.java new file mode 100644 index 000000000..85477ddde --- /dev/null +++ b/src/main/java/fr/insee/genesis/exceptions/InvalidMetadataException.java @@ -0,0 +1,7 @@ +package fr.insee.genesis.exceptions; + +public class InvalidMetadataException extends RuntimeException { + public InvalidMetadataException(String message) { + super(message); + } +} diff --git a/src/main/java/fr/insee/genesis/exceptions/ModesConflictException.java b/src/main/java/fr/insee/genesis/exceptions/ModesConflictException.java new file mode 100644 index 000000000..000466326 --- /dev/null +++ b/src/main/java/fr/insee/genesis/exceptions/ModesConflictException.java @@ -0,0 +1,7 @@ +package fr.insee.genesis.exceptions; + +public class ModesConflictException extends RuntimeException { + public ModesConflictException(String message) { + super(message); + } +} diff --git a/src/main/java/fr/insee/genesis/exceptions/UndefinedMetadataException.java b/src/main/java/fr/insee/genesis/exceptions/UndefinedMetadataException.java new file mode 100644 index 000000000..c031c356c --- /dev/null +++ b/src/main/java/fr/insee/genesis/exceptions/UndefinedMetadataException.java @@ -0,0 +1,12 @@ +package fr.insee.genesis.exceptions; + +public class UndefinedMetadataException extends RuntimeException { + + public UndefinedMetadataException(String message) { + super(message); + } + + public UndefinedMetadataException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/fr/insee/genesis/exceptions/UndefinedModesException.java b/src/main/java/fr/insee/genesis/exceptions/UndefinedModesException.java new file mode 100644 index 000000000..245c4e6cc --- /dev/null +++ b/src/main/java/fr/insee/genesis/exceptions/UndefinedModesException.java @@ -0,0 +1,7 @@ +package fr.insee.genesis.exceptions; + +public class UndefinedModesException extends RuntimeException { + public UndefinedModesException(String message) { + super(message); + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java index cd12f1094..9a5eb9ddb 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java @@ -30,6 +30,7 @@ @Service @Qualifier("lunaticJsonMongoAdapter") public class LunaticJsonRawDataMongoAdapter implements LunaticJsonRawDataPersistencePort { + private final LunaticJsonMongoDBRepository repository; private final MongoTemplate mongoTemplate; @@ -59,12 +60,6 @@ public Set findModesByQuestionnaire(String questionnaireId) { return new HashSet<>(repository.findModesByQuestionnaireId(questionnaireId)); } - @Override - public List findRawData(String campaignName, Mode mode, List interrogationIdList) { - List rawDataDocs = repository.findByCampaignModeAndInterrogations(campaignName, mode, interrogationIdList); - return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs); - } - @Override public List findRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList) { List rawDataDocs = repository.findByQuestionnaireModeAndInterrogations(questionnaireId, mode, interrogationIdList); @@ -147,4 +142,5 @@ public long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireI Long count = repository.countDistinctInterrogationIdsByQuestionnaireId(questionnaireId); return count != null ? count : 0; } + } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonReprocessMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonReprocessMongoAdapter.java new file mode 100644 index 000000000..028b3fada --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonReprocessMongoAdapter.java @@ -0,0 +1,60 @@ +package fr.insee.genesis.infrastructure.adapter; + +import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistencePort; +import fr.insee.genesis.infrastructure.repository.LunaticJsonMongoDBRepository; +import lombok.AllArgsConstructor; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.core.query.Update; +import org.springframework.stereotype.Service; + +import java.time.Instant; +import java.util.HashSet; +import java.util.Set; + +@Service +@AllArgsConstructor +@Qualifier("lunaticJsonReprocessMongoAdapter") +public class LunaticJsonReprocessMongoAdapter implements RawResponseReprocessPersistencePort { + + private final LunaticJsonMongoDBRepository repository; + private final MongoTemplate mongoTemplate; + + /** + * @param questionnaireId Legacy name for 'collectionInstrumentId'. + */ + @Override + public Set findProcessedInterrogationIdsByCollectionInstrumentId(String questionnaireId) { + return new HashSet<>(repository.findProcessedInterrogationIdsByQuestionnaireId(questionnaireId)); + } + + /** + * @param questionnaireId Legacy name for 'collectionInstrumentId'. + */ + @Override + public Set findProcessedInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween( + String questionnaireId, Instant sinceDate, Instant endDate) { + return new HashSet<>( + repository.findProcessedInterrogationIdsByQuestionnaireIdAndRecordDateBetween( + questionnaireId, sinceDate, endDate)); + } + + /** + * @param questionnaireId Legacy name for 'collectionInstrumentId'. + */ + @Override + public void resetProcessDates(String questionnaireId, Set interrogationIds) { + mongoTemplate.updateMulti( + Query.query( + Criteria.where("questionnaireId").is(questionnaireId) + .and("interrogationId").in(interrogationIds) + ), + new Update().unset("processDate"), + Constants.MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME + ); + } + +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java index bdb4c2fe7..b8aea6374 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java @@ -53,10 +53,11 @@ public List findRawResponsesByInterrogationID(String interroga @Override public void updateProcessDates(String collectionInstrumentId, Set interrogationIds) { mongoTemplate.updateMulti( - Query.query(Criteria.where("collectionInstrumentId").is(collectionInstrumentId).and("interrogationId").in(interrogationIds)) - , new Update().set("processDate", LocalDateTime.now()) - , Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME - ); + Query.query(Criteria.where("collectionInstrumentId") + .is(collectionInstrumentId) + .and("interrogationId").in(interrogationIds)), + new Update().set("processDate", LocalDateTime.now()), + Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME); } @Override @@ -92,12 +93,6 @@ public Set findDistinctCollectionInstrumentIds() { return new HashSet<>(repository.findDistinctCollectionInstrumentId()); } - @Override - public long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { - Long count = repository.countDistinctInterrogationIdsByCollectionInstrumentId(collectionInstrumentId); - return count != null ? count : 0; - } - @Override public Page findByCollectionInstrumentId(String collectionInstrumentId, Pageable pageable) { Page rawDataDocs = repository.findByCollectionInstrumentId(collectionInstrumentId, pageable); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseReprocessMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseReprocessMongoAdapter.java new file mode 100644 index 000000000..731ade641 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseReprocessMongoAdapter.java @@ -0,0 +1,51 @@ +package fr.insee.genesis.infrastructure.adapter; + +import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistencePort; +import fr.insee.genesis.infrastructure.repository.RawResponseRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.core.query.Update; +import org.springframework.stereotype.Service; + +import java.time.Instant; +import java.util.HashSet; +import java.util.Set; + +@Service +@RequiredArgsConstructor +@Qualifier("rawResponseMongoAdapter") +public class RawResponseReprocessMongoAdapter implements RawResponseReprocessPersistencePort { + + private final RawResponseRepository repository; + private final MongoTemplate mongoTemplate; + + @Override + public Set findProcessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { + return new HashSet<>(repository.findProcessedInterrogationIdsByCollectionInstrumentId(collectionInstrumentId)); + } + + @Override + public Set findProcessedInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween( + String collectionInstrumentId, Instant sinceDate, Instant endDate) { + return new HashSet<>( + repository.findProcessedInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween( + collectionInstrumentId, sinceDate, endDate)); + } + + @Override + public void resetProcessDates(String collectionInstrumentId, Set interrogationIds) { + mongoTemplate.updateMulti( + Query.query( + Criteria.where("collectionInstrumentId").is(collectionInstrumentId) + .and("interrogationId").in(interrogationIds) + ), + new Update().unset("processDate"), + Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME + ); + } + +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseReprocessMongoRouter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseReprocessMongoRouter.java new file mode 100644 index 000000000..b059bd356 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseReprocessMongoRouter.java @@ -0,0 +1,24 @@ +package fr.insee.genesis.infrastructure.adapter; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; +import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistencePort; +import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistenceRouter; +import lombok.AllArgsConstructor; +import org.springframework.stereotype.Component; + +@Component +@AllArgsConstructor +public class RawResponseReprocessMongoRouter implements RawResponseReprocessPersistenceRouter { + + private final RawResponseReprocessMongoAdapter rawResponseReprocessMongoAdapter; + private final LunaticJsonReprocessMongoAdapter lunaticJsonReprocessMongoAdapter; + + @Override + public RawResponseReprocessPersistencePort resolve(RawDataModelType rawDataModelType) { + return switch (rawDataModelType) { + case FILIERE -> rawResponseReprocessMongoAdapter; + case LEGACY -> lunaticJsonReprocessMongoAdapter; + }; + } + +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java index bfdd3e840..cf661c1c9 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java @@ -18,6 +18,7 @@ import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.stereotype.Service; +import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Collections; @@ -107,6 +108,28 @@ public Long deleteByCollectionInstrumentId(String collectionInstrumentId) { return countDeleted; } + @Override + public Long deleteByCollectionInstrumentIdAndInterrogationIds( + String collectionInstrumentId, + Set interrogationIds + ) { + return mongoRepository.deleteByCollectionInstrumentIdAndInterrogationIdIn( + collectionInstrumentId, + interrogationIds + ); + } + + @Override + public Long deleteByQuestionnaireIdAndInterrogationIds( + String questionnaireId, + Set interrogationIds + ) { + return mongoRepository.deleteByQuestionnaireIdAndInterrogationIdIn( + questionnaireId, + interrogationIds + ); + } + @Override public long count() { return mongoRepository.count(); @@ -176,7 +199,7 @@ public List findInterrogationIdsByQuestionnaireIdAndDateAfter(S } @Override - public List findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String collectionInstrumentId, LocalDateTime start, LocalDateTime end) { + public List findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String collectionInstrumentId, Instant start, Instant end) { List results = new ArrayList<>(); results.addAll(mongoRepository.findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(collectionInstrumentId,start,end)); results.addAll(mongoRepository.findInterrogationIdsQuestionnaireIdAndRecordDateBetween(collectionInstrumentId,start,end)); diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java index 5d06a0b5b..a0e1121dd 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java @@ -111,4 +111,23 @@ public interface LunaticJsonMongoDBRepository extends MongoRepository findByQuestionnaireId(String questionnaireId, Pageable pageable); boolean existsByInterrogationId(String interrogationId); + + + @Aggregation(pipeline = { + "{ $match: { questionnaireId: ?0, processDate: { $ne: null }, recordDate: { $gte: ?1, $lte: ?2 } } }", + "{ $group: { _id: '$interrogationId' } }", + "{ $project: { _id: 0, interrogationId: '$_id' } }" + }) + List findProcessedInterrogationIdsByQuestionnaireIdAndRecordDateBetween( + String questionnaireId, + Instant sinceDate, + Instant endDate + ); + + @Aggregation(pipeline = { + "{ $match: { questionnaireId: ?0, processDate: { $ne: null } } }", + "{ $group: { _id: '$interrogationId' } }", + "{ $project: { _id: 0, interrogationId: '$_id' } }" + }) + List findProcessedInterrogationIdsByQuestionnaireId(String questionnaireId); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java index de0583aa7..d37b039b3 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java @@ -24,6 +24,24 @@ public interface RawResponseRepository extends MongoRepository findDistinctCollectionInstrumentIdByProcessDateIsNull(); + @Aggregation(pipeline = { + "{ $match: { collectionInstrumentId: ?0, processDate: { $ne: null }, recordDate: { $gte: ?1, $lte: ?2 } } }", + "{ $group: { _id: '$interrogationId' } }", + "{ $project: { _id: 0, interrogationId: '$_id' } }" + }) + List findProcessedInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween( + String collectionInstrumentId, + Instant sinceDate, + Instant endDate + ); + + @Aggregation(pipeline = { + "{ $match: { collectionInstrumentId: ?0, processDate: { $ne: null } } }", + "{ $group: { _id: '$interrogationId' } }", + "{ $project: { _id: 0, interrogationId: '$_id' } }" + }) + List findProcessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId); + @Aggregation(pipeline = { "{ $match: { collectionInstrumentId: ?0,processDate: null } }", "{ $project: { _id: 0, interrogationId: '$interrogationId' } }" @@ -54,11 +72,4 @@ public interface RawResponseRepository extends MongoRepository findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween( String collectionInstrumentId, - LocalDateTime start, - LocalDateTime end + Instant start, + Instant end ); @Query( @@ -59,8 +60,8 @@ List findInterrogationIdsByCollectionInstrumentIdAndRecordDa ) List findInterrogationIdsQuestionnaireIdAndRecordDateBetween( String questionnaireId, - LocalDateTime start, - LocalDateTime end + Instant start, + Instant end ); /** @@ -103,6 +104,16 @@ List findInterrogationIdsQuestionnaireIdAndRecordDateBetween Long deleteByQuestionnaireId(String questionnaireId); Long deleteByCollectionInstrumentId(String collectionInstrumentId); + Long deleteByCollectionInstrumentIdAndInterrogationIdIn( + String collectionInstrumentId, + Set interrogationIds + ); + + Long deleteByQuestionnaireIdAndInterrogationIdIn( + String questionnaireId, + Set interrogationIds + ); + @Meta(cursorBatchSize = 20) Stream findByQuestionnaireId(String questionnaireId); diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index 8c1790ea5..acd07d400 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -1,7 +1,6 @@ package cucumber.functional_tests; import cucumber.functional_tests.config.CucumberSpringConfiguration; -import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.TestConstants; import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.rest.responses.RawResponseController; @@ -16,7 +15,7 @@ import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; -import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.JsonUtils; import fr.insee.genesis.exceptions.GenesisError; @@ -41,12 +40,7 @@ import org.springframework.boot.test.web.server.LocalServerPort; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; +import org.springframework.http.*; import org.springframework.test.context.ContextConfiguration; import java.io.IOException; @@ -81,22 +75,25 @@ public class RawDataDefinitions { FileUtils fileUtils = new FileUtils(config); SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), fileUtils); ControllerUtils controllerUtils = new ControllerUtils(fileUtils); - SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = new DataProcessingContextPersistancePortStub(); SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); + DataProcessingContextService dataProcessingContextService = new DataProcessingContextService( + new DataProcessingContextPersistancePortStub(), + surveyUnitPersistencePortStub); + + SurveyUnitQualityToolService surveyUnitQualityToolService = new SurveyUnitQualityToolService( + surveyUnitQualityToolPerretAdapterStub, + dataProcessingContextService); + LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService( lunaticJsonRawDataPersistanceStub, controllerUtils, new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), surveyUnitService, - surveyUnitQualityService, fileUtils, - new DataProcessingContextService( - dataProcessingContextPersistancePortStub, - surveyUnitPersistencePortStub), - surveyUnitQualityToolPerretAdapterStub, + surveyUnitQualityToolService, config, dataProcessingContextPersistancePortStub); @@ -109,31 +106,17 @@ public void saveAsRawJson(RawResponseDto dto) { }; RawResponseApiPort rawResponseApiPortStub = new RawResponseApiPort() { - @Override - public List getRawResponses(String questionnaireModelId, Mode mode, List interrogationIdList) { - return List.of(); - } @Override - public List getRawResponsesByInterrogationID(String interrogationId) { - return List.of(); - } - - @Override - public DataProcessResult processRawResponses(String questionnaireId, List interrogationIdList, List errors) throws GenesisException { + public DataProcessResult processRawResponsesByInterrogationIds(String questionnaireId, List interrogationIdList, List errors) throws GenesisException { return null; } @Override - public DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException { + public DataProcessResult processRawResponsesByInterrogationIds(String collectionInstrumentId) throws GenesisException { return null; } - @Override - public List convertRawResponse(List rawResponsModels, VariablesMap variablesMap) { - return List.of(); - } - @Override public List getUnprocessedCollectionInstrumentIds() { return List.of(); diff --git a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java index d8647c9d7..bd6d96dc1 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java @@ -4,6 +4,7 @@ import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; import fr.insee.genesis.infrastructure.repository.ContextualExternalVariableMongoDBRepository; import fr.insee.genesis.infrastructure.repository.ContextualPreviousVariableMongoDBRepository; import fr.insee.genesis.infrastructure.repository.DataProcessingContextMongoDBRepository; @@ -102,7 +103,8 @@ class ControllerAccessTest { private QuestionnaireMetadataMongoDBRepository questionnaireMetadataMongoDBRepository; @MockitoBean private RawResponseRepository rawResponseRepository; - + @MockitoBean + private SurveyUnitQualityToolService surveyUnitQualityToolService; /** * Provides a stream of URIs that are allowed for reader. */ diff --git a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java index 1e9fae6c7..e81839cc7 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java @@ -15,8 +15,8 @@ import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; import fr.insee.genesis.domain.service.rawdata.RawResponseService; -import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.service.volumetry.VolumetryLogService; import fr.insee.genesis.infrastructure.utils.FileUtils; @@ -63,7 +63,6 @@ class UtilsControllerTest { static ControllerUtils controllerUtils = new ControllerUtils(fileUtils); static QuestionnaireMetadataService metadataService = new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()); static SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); - static SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); static Path logFileFolder = Path.of(new ConfigStub().getLogFolder()).resolve(Constants.VOLUMETRY_FOLDER_NAME); @@ -73,17 +72,20 @@ static void init() { lunaticJsonRawDataPersistencePort = new LunaticJsonRawDataPersistanceStub(); rawResponseDataPersistanceStub = new RawResponseDataPersistanceStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); + DataProcessingContextService dataProcessingContextService = new DataProcessingContextService( + new DataProcessingContextPersistancePortStub(), + surveyUnitPersistencePortStub); + + SurveyUnitQualityToolService surveyUnitQualityToolService = new SurveyUnitQualityToolService( + surveyUnitQualityToolPerretAdapterStub, + dataProcessingContextService); LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService( lunaticJsonRawDataPersistencePort, controllerUtils, metadataService, surveyUnitService, - surveyUnitQualityService, fileUtils, - new DataProcessingContextService( - new DataProcessingContextPersistancePortStub(), - surveyUnitPersistencePortStub), - surveyUnitQualityToolPerretAdapterStub, + surveyUnitQualityToolService, new ConfigStub(), contextStub); @@ -91,15 +93,10 @@ static void init() { controllerUtils, metadataService, surveyUnitService, - surveyUnitQualityService, - surveyUnitQualityToolPerretAdapterStub, - new DataProcessingContextService( - new DataProcessingContextPersistancePortStub(), - surveyUnitPersistencePortStub), + surveyUnitQualityToolService, fileUtils, new ConfigStub(), - rawResponseDataPersistanceStub - ); + rawResponseDataPersistanceStub); utilsControllerStatic = new UtilsController( diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index 533740dc5..d65de60f7 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -1,7 +1,6 @@ package fr.insee.genesis.controller.rest.responses; -import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.utils.ControllerUtils; import fr.insee.genesis.domain.model.context.DataProcessingContextModel; @@ -16,7 +15,7 @@ import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; -import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.JsonUtils; import fr.insee.genesis.exceptions.GenesisError; @@ -26,13 +25,7 @@ import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; import fr.insee.genesis.infrastructure.utils.FileUtils; -import fr.insee.genesis.stubs.ConfigStub; -import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; -import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; -import fr.insee.genesis.stubs.RawResponseDataPersistanceStub; -import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; -import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; +import fr.insee.genesis.stubs.*; import fr.insee.modelefiliere.RawResponseDto; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; @@ -64,15 +57,20 @@ class RawResponseControllerTest { private static final QuestionnaireMetadataPersistencePortStub questionnaireMetadataPersistencePortStub = new QuestionnaireMetadataPersistencePortStub(); + private final DataProcessingContextService dataProcessingContextService = new DataProcessingContextService( + dataProcessingContextPersistancePortStub, + surveyUnitPersistencePortStub); + + private final SurveyUnitQualityToolService surveyUnitQualityToolService = new SurveyUnitQualityToolService( + surveyUnitQualityToolPerretAdapterStub, + dataProcessingContextService); private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub, new ControllerUtils(fileUtils), new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), new SurveyUnitService(surveyUnitPersistencePortStub, new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), fileUtils), - new SurveyUnitQualityService(), fileUtils, - new DataProcessingContextService(dataProcessingContextPersistancePortStub, surveyUnitPersistencePortStub), - surveyUnitQualityToolPerretAdapterStub, + surveyUnitQualityToolService, new ConfigStub(), new DataProcessingContextPersistancePortStub() ); @@ -86,31 +84,17 @@ public void saveAsRawJson(RawResponseDto dto) { }; RawResponseApiPort rawResponseApiPortStub = new RawResponseApiPort() { - @Override - public List getRawResponses(String questionnaireModelId, Mode mode, List interrogationIdList) { - return List.of(); - } - - @Override - public List getRawResponsesByInterrogationID(String interrogationId) { - return List.of(); - } @Override - public DataProcessResult processRawResponses(String questionnaireId, List interrogationIdList, List errors) throws GenesisException { + public DataProcessResult processRawResponsesByInterrogationIds(String questionnaireId, List interrogationIdList, List errors) throws GenesisException { return null; } @Override - public DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException { + public DataProcessResult processRawResponsesByInterrogationIds(String collectionInstrumentId) throws GenesisException { return null; } - @Override - public List convertRawResponse(List rawResponsModels, VariablesMap variablesMap) { - return List.of(); - } - @Override public List getUnprocessedCollectionInstrumentIds() { return List.of(); diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java index 09c6049ed..01701e81e 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -12,7 +12,7 @@ import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; -import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.JsonUtils; import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; @@ -34,7 +34,8 @@ import java.util.List; import java.util.Map; -import static fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService.getValueString; +import static fr.insee.genesis.domain.utils.LunaticJsonRawDataConverter.convertRawData; +import static fr.insee.genesis.domain.utils.LunaticJsonRawDataConverter.getValueString; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; class LunaticJsonRawDataServiceTest { @@ -49,16 +50,23 @@ class LunaticJsonRawDataServiceTest { DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = new DataProcessingContextPersistancePortStub(); SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); ConfigStub configStub = new ConfigStub(); + DataProcessingContextService dataProcessingContextService = new DataProcessingContextService( + dataProcessingContextPersistancePortStub, + surveyUnitPersistencePortStub); + static QuestionnaireMetadataPersistencePortStub questionnaireMetadataPersistencePortStub = new QuestionnaireMetadataPersistencePortStub(); + SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), fileUtils); + + SurveyUnitQualityToolService surveyUnitQualityToolService = new SurveyUnitQualityToolService( + surveyUnitQualityToolPerretAdapterStub, + dataProcessingContextService); LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService( lunaticJsonRawDataPersistanceStub, controllerUtils, metadataService, - new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils), - new SurveyUnitQualityService(), + surveyUnitService, fileUtils, - new DataProcessingContextService(dataProcessingContextPersistancePortStub, surveyUnitPersistencePortStub), - surveyUnitQualityToolPerretAdapterStub, + surveyUnitQualityToolService, configStub, dataProcessingContextPersistancePortStub ); @@ -355,7 +363,7 @@ void convertRawData_should_not_throw_exception_if_external_not_present() throws .mode(Mode.WEB) .build(); - assertDoesNotThrow(() -> lunaticJsonRawDataService.convertRawData(List.of(rawDataModel),new VariablesMap())); + assertDoesNotThrow(() -> convertRawData(List.of(rawDataModel),new VariablesMap())); } @Test @@ -374,7 +382,7 @@ void convertRawData_if_external_not_present_test() throws Exception { .mode(Mode.WEB) .build(); - List suModels = lunaticJsonRawDataService.convertRawData(List.of(rawDataModel),new VariablesMap()); + List suModels = convertRawData(List.of(rawDataModel),new VariablesMap()); Assertions.assertThat(suModels).hasSize(1); Assertions.assertThat(suModels.getFirst().getCollectedVariables()).hasSize(1); Assertions.assertThat(suModels.getFirst().getExternalVariables()).isEmpty(); @@ -437,7 +445,7 @@ void convertRawData_should_not_throw_exception_if_collected_not_present() throws .mode(Mode.WEB) .build(); - assertDoesNotThrow(() -> lunaticJsonRawDataService.convertRawData(List.of(rawDataModel),new VariablesMap())); + assertDoesNotThrow(() -> convertRawData(List.of(rawDataModel),new VariablesMap())); } @Test @@ -456,7 +464,7 @@ void convertRawData_if_collected_not_present_test() throws Exception { .mode(Mode.WEB) .build(); - List suModels = lunaticJsonRawDataService.convertRawData(List.of(rawDataModel),new VariablesMap()); + List suModels = convertRawData(List.of(rawDataModel),new VariablesMap()); Assertions.assertThat(suModels).hasSize(1); Assertions.assertThat(suModels.getFirst().getExternalVariables()).hasSize(1); Assertions.assertThat(suModels.getFirst().getCollectedVariables()).isEmpty(); @@ -639,7 +647,7 @@ void convertRawData_collected_array_with_null_and_empty_values_should_ignore_the // WHEN List suModels = - lunaticJsonRawDataService.convertRawData(List.of(rawDataModel), new VariablesMap()); + convertRawData(List.of(rawDataModel), new VariablesMap()); // THEN Assertions.assertThat(suModels).hasSize(1); @@ -692,4 +700,5 @@ void getValueString_double_test(){ Assertions.assertThat(getValueString(doubleObject)).isEqualTo("101010101010.111"); } -} \ No newline at end of file + +} diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceUnitTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceUnitTest.java index 1c24ff3f1..75118616d 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceUnitTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceUnitTest.java @@ -6,10 +6,8 @@ import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort; import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort; import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistencePort; -import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; -import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; -import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; @@ -45,10 +43,8 @@ void init() { new ControllerUtils(new FileUtils(new ConfigStub())), metadataService, mock(SurveyUnitService.class), - mock(SurveyUnitQualityService.class), new FileUtils(new ConfigStub()), - mock(DataProcessingContextService.class), - mock(SurveyUnitQualityToolPort.class), + mock(SurveyUnitQualityToolService.class), new ConfigStub(), mock(DataProcessingContextPersistancePort.class) ); @@ -89,10 +85,8 @@ void getUnprocessedDataQuestionnaireIds_shouldnt_return_if_no_spec() { new ControllerUtils(new FileUtils(new ConfigStub())), metadataService, mock(SurveyUnitService.class), - mock(SurveyUnitQualityService.class), new FileUtils(new ConfigStub()), - mock(DataProcessingContextService.class), - mock(SurveyUnitQualityToolPort.class), + mock(SurveyUnitQualityToolService.class), new ConfigStub(), mock(DataProcessingContextPersistancePort.class) ); diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticRawDataReprocessTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticRawDataReprocessTest.java new file mode 100644 index 000000000..3c9849a2c --- /dev/null +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticRawDataReprocessTest.java @@ -0,0 +1,71 @@ +package fr.insee.genesis.domain.service.rawdata; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; +import fr.insee.genesis.exceptions.InvalidDateIntervalException; +import fr.insee.genesis.stubs.LunaticJsonRawDataServiceStub; +import fr.insee.genesis.stubs.RawResponseReprocessPersistenceRouterStub; +import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +class LunaticRawDataReprocessTest { + + private ReprocessRawResponseService reprocessRawResponseService; + + @BeforeEach + void freshStart() { + reprocessRawResponseService = new ReprocessRawResponseService( + new SurveyUnitPersistencePortStub(), + null, + new LunaticJsonRawDataServiceStub(), + new RawResponseReprocessPersistenceRouterStub()); + } + + @Test + void reprocessRawData_should_return_empty_result_when_no_processed_interrogation_ids_found() throws Exception { + // GIVEN + String questionnaireId = "TESTIDQUEST"; + + // WHEN + DataProcessResult result = reprocessRawResponseService.reprocessRawResponses( + RawDataModelType.LEGACY, questionnaireId, null, null); + + // THEN + Assertions.assertThat(result).isNotNull(); + Assertions.assertThat(result.dataCount()).isZero(); + Assertions.assertThat(result.formattedDataCount()).isZero(); + Assertions.assertThat(result.errors()).isEmpty(); + } + + @Test + void reprocessRawData_should_throw_when_endDate_is_provided_without_sinceDate() { + String questionnaireId = "TESTIDQUEST"; + Instant endDate = Instant.now(); + + Assertions.assertThatThrownBy(() -> + reprocessRawResponseService.reprocessRawResponses( + RawDataModelType.LEGACY, questionnaireId, null, endDate)) + .isInstanceOf(InvalidDateIntervalException.class) + .hasMessage("'endDate' cannot be provided without 'sinceDate'."); + } + + @Test + void reprocessRawData_should_throw_when_endDate_is_before_sinceDate() { + String questionnaireId = "TESTIDQUEST"; + Instant sinceDate = LocalDateTime.of(2024, 1, 10, 10, 0).toInstant(ZoneOffset.UTC); + Instant endDate = LocalDateTime.of(2024, 1, 9, 10, 0).toInstant(ZoneOffset.UTC); + + Assertions.assertThatThrownBy(() -> + reprocessRawResponseService.reprocessRawResponses( + RawDataModelType.LEGACY, questionnaireId, sinceDate, endDate)) + .isInstanceOf(InvalidDateIntervalException.class) + .hasMessage("'endDate' value cannot be before 'sinceDate'."); + } + +} diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java index 9349c33df..7969e7527 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java @@ -6,13 +6,12 @@ import fr.insee.genesis.controller.utils.ControllerUtils; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistencePort; import fr.insee.genesis.domain.ports.spi.RawResponsePersistencePort; -import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; -import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; -import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.utils.FileUtils; @@ -21,10 +20,8 @@ import fr.insee.modelefiliere.RawResponseDto; import lombok.SneakyThrows; import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; +import org.bson.types.ObjectId; +import org.junit.jupiter.api.*; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.mockito.ArgumentCaptor; @@ -32,18 +29,12 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import static fr.insee.genesis.TestConstants.DEFAULT_INTERROGATION_ID; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.*; class RawResponseServiceUnitTest { @@ -52,11 +43,10 @@ class RawResponseServiceUnitTest { static ControllerUtils controllerUtils; static QuestionnaireMetadataService metadataService; static SurveyUnitService surveyUnitService; - + static SurveyUnitQualityToolService surveyUnitQualityToolService; private ArgumentCaptor> surveyUnitModelsCaptor; private static final String TEST_VALIDATION_DATE = "2025-11-11T06:00:00Z"; - @BeforeEach @SuppressWarnings("unchecked") void init() { @@ -64,14 +54,13 @@ void init() { controllerUtils = mock(ControllerUtils.class); metadataService = mock(QuestionnaireMetadataService.class); surveyUnitService = mock(SurveyUnitService.class); + surveyUnitQualityToolService = mock(SurveyUnitQualityToolService.class); rawResponseService = new RawResponseService( controllerUtils, metadataService, surveyUnitService, - mock(SurveyUnitQualityService.class), - mock(SurveyUnitQualityToolPort.class), - mock(DataProcessingContextService.class), + surveyUnitQualityToolService, new FileUtils(new ConfigStub()), new ConfigStub(), rawResponsePersistencePort @@ -91,10 +80,9 @@ void getUnprocessedCollectionInstrumentIds_test() { doReturn(List.of(ModeDto.CAWI)).when(rawResponsePersistencePort).findModesByCollectionInstrument(any()); doReturn(new MetadataModel()).when(metadataService).loadAndSaveIfNotExists(any(), any(), any(), any(), any()); - //WHEN + THEN Assertions.assertThat(rawResponseService.getUnprocessedCollectionInstrumentIds()) - .containsExactlyInAnyOrder("QUEST1","QUEST2"); + .containsExactlyInAnyOrder("QUEST1", "QUEST2"); } @Test @@ -114,15 +102,12 @@ void getUnprocessedCollectionInstrumentIds_shouldnt_return_if_no_spec() { new ControllerUtils(new FileUtils(new ConfigStub())), metadataService, mock(SurveyUnitService.class), - mock(SurveyUnitQualityService.class), - mock(SurveyUnitQualityToolPort.class), - mock(DataProcessingContextService.class), + mock(SurveyUnitQualityToolService.class), new FileUtils(new ConfigStub()), new ConfigStub(), rawResponsePersistencePort ); - //WHEN + THEN Assertions.assertThat(rawResponseService.getUnprocessedCollectionInstrumentIds()) .containsExactly("TEST-TABLEAUX"); @@ -152,7 +137,14 @@ void existsByInterrogationId_shouldReturnFalse_whenNotExists() { } @Nested - @DisplayName("Non regression tests of #22875 : validation date and questionnaire state in processed responses") + @DisplayName("Non regression tests of InseeFr/Genesis-API#365: validation date and questionnaire state in processed responses") + @Disabled("to be fixed") /* hint: + since processRawResponsesByInterrogationIds is mocked, after refactor where + processRawResponsesByInterrogationIds(String collectionInstrumentId) doesn't directly call + surveyUnitService.saveSurveyUnits(...), but calls + processRawResponsesByInterrogationIds(String collectionInstrumentId, List interrogationIds, List errors), + the assertion "surveyUnitService.saveSurveyUnits(...) should be called" no longer passes. + */ class ValidationDateAndQuestionnaireStateTests{ //OK cases @ParameterizedTest @@ -354,12 +346,12 @@ private void givenInvalidValidationDate(){ //WHENS private List whenProcessByCollectionInstrumentIdAndInterrogationIdList() throws GenesisException { - rawResponseService.processRawResponses(TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID); + rawResponseService.processRawResponsesByInterrogationIds(TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID); verify(surveyUnitService).saveSurveyUnits(surveyUnitModelsCaptor.capture()); return surveyUnitModelsCaptor.getValue(); } private List whenProcessRawResponsesCollectionInstrumentId() throws GenesisException { - rawResponseService.processRawResponses( + rawResponseService.processRawResponsesByInterrogationIds( TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID, Collections.singletonList(TestConstants.DEFAULT_INTERROGATION_ID), new ArrayList<>() @@ -412,4 +404,43 @@ void getDistinctCollectionInstrumentIds_test(){ //WHEN + THEN Assertions.assertThat(rawResponseService.getDistinctCollectionInstrumentIds()).containsExactly(collectionInstrumentId); } -} \ No newline at end of file + + @Test + void processWithDuplicateInterrogationId() throws GenesisException { + // Given + String fooCollectionInstrumentId = "FOOX00"; + Mode fooMode = Mode.WEB; + + Set interrogationIds = Set.of("interrogation-id-1", "interrogation-id-2"); + List interrogationIdList = interrogationIds.stream().toList(); + Map fooVariable = Map.of("COLLECTED", "some value"); + Map fooCollectedContent = Map.of("SOME_VARIABLE", fooVariable); + Map fooData = Map.of("COLLECTED", fooCollectedContent); + Map fooPayload = Map.of( + "questionnaireState", "FOO_QUESTIONNAIRE_STATE", + "data", fooData); + + LocalDateTime recordDate1 = LocalDateTime.of(2026, 1, 1, 8, 0); + LocalDateTime processDate = LocalDateTime.of(2026, 1, 1, 9, 0); + LocalDateTime recordDate2 = LocalDateTime.of(2026, 1, 1, 10, 0); + + List mockedRawResponses = new ArrayList<>(List.of( + new RawResponseModel(new ObjectId(), "interrogation-id-1", fooCollectionInstrumentId, fooMode, fooPayload, recordDate1, processDate), + new RawResponseModel(new ObjectId(), "interrogation-id-1", fooCollectionInstrumentId, fooMode, fooPayload, recordDate2, null), + new RawResponseModel(new ObjectId(), "interrogation-id-2", fooCollectionInstrumentId, fooMode, fooPayload, recordDate2, null) + )); + + Mockito.when(rawResponsePersistencePort.findUnprocessedInterrogationIdsByCollectionInstrumentId(fooCollectionInstrumentId)).thenReturn(interrogationIds); + Mockito.when(rawResponsePersistencePort.findRawResponses(fooCollectionInstrumentId, fooMode, interrogationIdList)).thenReturn(mockedRawResponses); + Mockito.when(controllerUtils.getModesList(fooCollectionInstrumentId)).thenReturn(List.of(fooMode)); + Mockito.when(surveyUnitQualityToolService.resolveWithReviewValue(fooCollectionInstrumentId)).thenReturn(false); + Mockito.when(metadataService.loadAndSaveIfNotExists(eq(fooCollectionInstrumentId), eq(fooCollectionInstrumentId), eq(fooMode), any(), any())).thenReturn(new MetadataModel()); + + // When + DataProcessResult dataProcessResult = rawResponseService.processRawResponsesByInterrogationIds(fooCollectionInstrumentId); + + // Then + assertEquals(2, dataProcessResult.dataCount()); + } + +} diff --git a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java index 8cb4453bc..e68e4c151 100644 --- a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java @@ -18,6 +18,7 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import java.time.Instant; import java.time.LocalDateTime; import java.time.Month; import java.util.ArrayList; @@ -270,8 +271,8 @@ void findDistinctInterrogationIdsByQuestionnaireIdAndDateAfterTest_doc_in_period void findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetweenTest_no_doc_in_period() { addAdditionnalSurveyUnitModelToMongoStub(); - LocalDateTime start = LocalDateTime.of(2025, 9, 1, 0, 0, 0); - LocalDateTime end = LocalDateTime.of(2025, 9, 2, 0, 0, 0); + Instant start = Instant.parse("2025-09-01T00:00:00Z"); + Instant end = Instant.parse("2025-09-02T00:00:00Z"); Assertions.assertThat( surveyUnitServiceStatic.findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween( @@ -285,8 +286,8 @@ void findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetweenTes void findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetweenTest_doc_in_period() { addAdditionnalSurveyUnitModelToMongoStub(); - LocalDateTime start = LocalDateTime.of(2022, 1, 1, 0, 0, 0); - LocalDateTime end = LocalDateTime.of(2026, 1, 1, 0, 0, 0); + Instant start = Instant.parse("2022-01-01T00:00:00Z"); + Instant end = Instant.parse("2026-01-01T00:00:00Z"); Assertions.assertThat( surveyUnitServiceStatic.findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween( diff --git a/src/test/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapterTest.java b/src/test/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapterTest.java index 594b15557..79cc23e44 100644 --- a/src/test/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapterTest.java +++ b/src/test/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapterTest.java @@ -73,7 +73,7 @@ void findRawDataTest(){ //WHEN repository.getDocuments().add(doc); //THEN - List rawdatas = adapter.findRawData("campaign01",Mode.WEB,List.of("interrogation01")); + List rawdatas = adapter.findRawDataByQuestionnaireId("questionnaire01",Mode.WEB,List.of("interrogation01")); Assertions.assertThat(rawdatas).hasSize(1); } diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java index 2c0e4dc77..547e8918d 100644 --- a/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java @@ -13,6 +13,7 @@ import java.time.LocalDateTime; import java.time.Instant; +import java.time.chrono.ChronoLocalDateTime; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -196,6 +197,33 @@ public boolean existsByInterrogationId(String interrogationId) { return DEFAULT_INTERROGATION_ID.equals(interrogationId); } + @Override + public List findProcessedInterrogationIdsByQuestionnaireId(String questionnaireId) { + return documents.stream() + .filter(doc -> Objects.equals(doc.questionnaireId(), questionnaireId)) + .filter(doc -> doc.processDate() != null) + .map(LunaticJsonRawDataDocument::interrogationId) + .distinct() + .toList(); + } + + @Override + public List findProcessedInterrogationIdsByQuestionnaireIdAndRecordDateBetween( + String questionnaireId, + Instant sinceDate, + Instant endDate + ) { + return documents.stream() + .filter(doc -> Objects.equals(doc.questionnaireId(), questionnaireId)) + .filter(doc -> doc.processDate() != null) + .filter(doc -> doc.recordDate() != null) + .filter(doc -> !doc.recordDate().isBefore(ChronoLocalDateTime.from(sinceDate))) + .filter(doc -> !doc.recordDate().isAfter(ChronoLocalDateTime.from(endDate))) + .map(LunaticJsonRawDataDocument::interrogationId) + .distinct() + .toList(); + } + // Implémentations vides requises par MongoRepository @Override public S save(S entity) { return null; } @Override public Optional findById(String s) { return Optional.empty(); } diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java index 9f32a324b..9f2aba791 100644 --- a/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java @@ -69,23 +69,19 @@ public Set findModesByQuestionnaire(String questionnaireId) { } @Override - public List findRawData(String campaignName, Mode mode, List interrogationIdList) { + public List findRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList) { List docs = mongoStub.stream().filter(lunaticJsonRawDataDocument -> - lunaticJsonRawDataDocument.campaignId().equals(campaignName) - && lunaticJsonRawDataDocument.mode().equals(mode) - && interrogationIdList.contains(lunaticJsonRawDataDocument.interrogationId()) + getQuestionnaireId(lunaticJsonRawDataDocument).equals(questionnaireId) + && lunaticJsonRawDataDocument.mode().equals(mode) + && interrogationIdList.contains(lunaticJsonRawDataDocument.interrogationId()) ).toList(); return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(docs); } - - @Override - public List findRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList) { - List docs = mongoStub.stream().filter(lunaticJsonRawDataDocument -> - lunaticJsonRawDataDocument.questionnaireId().equals(questionnaireId) - && lunaticJsonRawDataDocument.mode().equals(mode) - && interrogationIdList.contains(lunaticJsonRawDataDocument.interrogationId()) - ).toList(); - return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(docs); } + /** 'questionnaireId' is the new name of the 'campaignId' property. */ + private static String getQuestionnaireId(LunaticJsonRawDataDocument lunaticJsonRawDataDocument) { + return lunaticJsonRawDataDocument.questionnaireId() != null ? + lunaticJsonRawDataDocument.questionnaireId() : lunaticJsonRawDataDocument.campaignId(); + } @Override public Page findRawDataByQuestionnaireId(String questionnaireId, Pageable pageable) { @@ -246,4 +242,5 @@ public boolean existsByInterrogationId(String interrogationId) { public long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireId) { return 0; } + } diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataServiceStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataServiceStub.java new file mode 100644 index 000000000..a3465d14b --- /dev/null +++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataServiceStub.java @@ -0,0 +1,95 @@ +package fr.insee.genesis.stubs; + +import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; +import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.exceptions.GenesisError; +import fr.insee.genesis.exceptions.GenesisException; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class LunaticJsonRawDataServiceStub implements LunaticJsonRawDataApiPort { + @Override + public void save(LunaticJsonRawDataModel rawData) { + // stub, this method unused in tests yet. + } + + @Override + public List getRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList) { + return List.of(); + } + + @Override + public List getUnprocessedDataIds() { + return List.of(); + } + + @Override + public Set getUnprocessedDataQuestionnaireIds() { + return Set.of(); + } + + @Override + public void updateProcessDates(List surveyUnitModels) { + // stub, this method unused in tests yet. + } + + @Override + public Set findDistinctQuestionnaireIds() { + return Set.of(); + } + + @Override + public long countRawResponsesByQuestionnaireId(String campaignId) { + return 0; + } + + @Override + public long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireId) { + return 0; + } + + @Override + public Page findRawDataByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable) { + return null; + } + + @Override + public List getRawDataByInterrogationId(String interrogationId) { + return List.of(); + } + + @Override + public DataProcessResult processRawDataByInterrogationIds(String campaignName, List interrogationIdList, List errors) throws GenesisException { + return null; + } + + @Override + public DataProcessResult processRawData(String collectionInstrumentId) throws GenesisException { + return null; + } + + @Override + public Map> findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime since) { + return Map.of(); + } + + @Override + public Page findRawDataByQuestionnaireId(String questionnaireId, Pageable pageable) { + return null; + } + + @Override + public boolean existsByInterrogationId(String interrogationId) { + return false; + } +} diff --git a/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java index 697a57d08..ac922a5e0 100644 --- a/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java +++ b/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java @@ -44,6 +44,7 @@ public void updateProcessDates(String collectionInstrumentId, Set interr return; } + @Override public List getUnprocessedCollectionIds() { return List.of(); @@ -104,9 +105,4 @@ public Set findDistinctCollectionInstrumentIds() { public boolean existsByInterrogationId(String interrogationId) { return DEFAULT_INTERROGATION_ID.equals(interrogationId); } - - @Override - public long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { - return 0; - } } diff --git a/src/test/java/fr/insee/genesis/stubs/RawResponseReprocessPersistenceRouterStub.java b/src/test/java/fr/insee/genesis/stubs/RawResponseReprocessPersistenceRouterStub.java new file mode 100644 index 000000000..e50aba991 --- /dev/null +++ b/src/test/java/fr/insee/genesis/stubs/RawResponseReprocessPersistenceRouterStub.java @@ -0,0 +1,14 @@ +package fr.insee.genesis.stubs; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; +import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistencePort; +import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistenceRouter; + +public class RawResponseReprocessPersistenceRouterStub implements RawResponseReprocessPersistenceRouter { + + @Override + public RawResponseReprocessPersistencePort resolve(RawDataModelType rawDataModelType) { + return new RawResponseReprocessPersistenceStub(); + } + +} diff --git a/src/test/java/fr/insee/genesis/stubs/RawResponseReprocessPersistenceStub.java b/src/test/java/fr/insee/genesis/stubs/RawResponseReprocessPersistenceStub.java new file mode 100644 index 000000000..4732c2451 --- /dev/null +++ b/src/test/java/fr/insee/genesis/stubs/RawResponseReprocessPersistenceStub.java @@ -0,0 +1,50 @@ +package fr.insee.genesis.stubs; + +import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistencePort; +import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public class RawResponseReprocessPersistenceStub implements RawResponseReprocessPersistencePort { + + List mongoStub = new ArrayList<>(); + + @Override + public Set findProcessedInterrogationIdsByCollectionInstrumentId(String questionnaireId) { + return Set.of(); + } + + @Override + public Set findProcessedInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String questionnaireId, Instant sinceDate, Instant endDate) { + return Set.of(); + } + + @Override + public void resetProcessDates(String questionnaireId, Set interrogationIds) { + for (int i = 0; i < mongoStub.size(); i++) { + LunaticJsonRawDataDocument document = mongoStub.get(i); + + if (document.questionnaireId().equals(questionnaireId) + && interrogationIds.contains(document.interrogationId())) { + + LunaticJsonRawDataDocument newDocument = LunaticJsonRawDataDocument.builder() + .id(document.id()) + .campaignId(document.campaignId()) + .questionnaireId(document.questionnaireId()) + .interrogationId(document.interrogationId()) + .idUE(document.idUE()) + .mode(document.mode()) + .data(document.data()) + .recordDate(document.recordDate()) + .processDate(null) + .build(); + + mongoStub.set(i, newDocument); + } + } + } + +} diff --git a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java index 65f5ddd6b..c6da9de62 100644 --- a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java @@ -4,7 +4,9 @@ import fr.insee.genesis.domain.ports.spi.SurveyUnitPersistencePort; import lombok.Getter; +import java.time.Instant; import java.time.LocalDateTime; +import java.time.ZoneId; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -135,18 +137,30 @@ public List findInterrogationIdsByQuestionnaireIdAndDateAfter(S } @Override - public List findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String collectionInstrumentId, LocalDateTime start, LocalDateTime end) { + public List findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween( + String collectionInstrumentId, + Instant start, + Instant end + ) { List surveyUnitModelList = new ArrayList<>(); - for(SurveyUnitModel surveyUnitModel : mongoStub){ - if(surveyUnitModel.getCollectionInstrumentId().equals(collectionInstrumentId) - && !surveyUnitModel.getRecordDate().isBefore(start) - && surveyUnitModel.getRecordDate().isBefore(end)) + ZoneId zone = ZoneId.of("Europe/Paris"); + + for (SurveyUnitModel surveyUnitModel : mongoStub) { + Instant recordDateInstant = surveyUnitModel.getRecordDate() + .atZone(zone) + .toInstant(); + + if (surveyUnitModel.getCollectionInstrumentId().equals(collectionInstrumentId) + && !recordDateInstant.isBefore(start) + && recordDateInstant.isBefore(end)) { surveyUnitModelList.add( new SurveyUnitModel(surveyUnitModel.getInterrogationId(), surveyUnitModel.getMode()) ); + } } - return surveyUnitModelList; } + return surveyUnitModelList; + } //======== OPTIMISATIONS PERFS (START) ======== @@ -175,6 +189,16 @@ public Long deleteByCollectionInstrumentId(String collectionInstrumentId) { return (long) mongoStub.stream().filter(su -> !su.getCollectionInstrumentId().equals(collectionInstrumentId)).toList().size(); } + @Override + public Long deleteByCollectionInstrumentIdAndInterrogationIds(String collectionInstrumentId, Set interrogationIds) { + return 0L; + } + + @Override + public Long deleteByQuestionnaireIdAndInterrogationIds(String questionnaireId, Set interrogationIds) { + return 0L; + } + @Override public long count() { return mongoStub.size();