From 88255985a959559690f7422df24f2dbfe9a784d0 Mon Sep 17 00:00:00 2001 From: rmohammed-goaco Date: Thu, 12 Mar 2026 12:25:36 +0000 Subject: [PATCH 1/3] Release/13.2 (#332) * Update GrantMandatoryQuestionFundingLocation enum to replace 'OUTSIDE_UK' with 'INTERNATIONAL' * Update GrantMandatoryQuestionFundingLocation enum to replace 'INTERNATIONAL' with 'OUTSIDE_UK' From d7a3b50786a438619c4713a56525f26353382d99 Mon Sep 17 00:00:00 2001 From: rmohammed-goaco Date: Tue, 24 Mar 2026 14:57:06 +0000 Subject: [PATCH 2/3] Release/13.3 (#334) * Add submission anonymisation feature - Introduced SubmissionAnonymisationConfigProperties for configuration settings. - Added EXPIRED status to SubmissionStatus enum. - Enhanced GrantAttachmentRepository and GrantMandatoryQuestionRepository with delete methods for submissions. - Updated SubmissionRepository with methods for anonymising submissions and deleting related data. - Implemented SubmissionAnonymisationScheduler to handle scheduled anonymisation of submissions. - Created SubmissionAnonymisationService to manage the anonymisation process, including S3 object deletion and database cleanup. - Added application properties for submission anonymisation configuration. - Created database migration to document the new EXPIRED status in the submission table. * Refactor SubmissionAnonymisationConfigProperties - Removed Lombok annotations: @Builder, @AllArgsConstructor, and @NoArgsConstructor. - Simplified the class by retaining only @Getter and @Setter annotations. - Adjusted the default value for daysBeforeExpiry to 90, ensuring clarity in configuration settings. * Enhance S3Service to support deletion of attachments using S3 URI - Updated deleteAttachment method to accept an S3 URI instead of just the object key. - Extracted bucket name and key from the S3 URI for improved flexibility. - Added logging to indicate which bucket and object are being deleted. - Ensured deletion from both the specified bucket and the attachments bucket. * Refactor SubmissionAnonymisationService to improve S3 deletion handling - Updated the S3 deletion logic to abort anonymisation if any deletion fails, ensuring the submission remains in IN_PROGRESS for retry. - Enhanced logging to provide clearer context on failures during S3 object deletion, improving traceability and error handling. --- ...bmissionAnonymisationConfigProperties.java | 16 +++++ .../adminbackend/enums/SubmissionStatus.java | 2 +- .../GrantAttachmentRepository.java | 9 +++ .../GrantMandatoryQuestionRepository.java | 4 ++ .../repositories/SubmissionRepository.java | 29 ++++++++ .../SubmissionAnonymisationScheduler.java | 51 ++++++++++++++ .../gap/adminbackend/services/S3Service.java | 17 ++++- .../SubmissionAnonymisationService.java | 68 +++++++++++++++++++ src/main/resources/application.properties | 8 +++ .../V1_104__add_expired_submission_status.sql | 2 + 10 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 src/main/java/gov/cabinetoffice/gap/adminbackend/config/SubmissionAnonymisationConfigProperties.java create mode 100644 src/main/java/gov/cabinetoffice/gap/adminbackend/schedulers/SubmissionAnonymisationScheduler.java create mode 100644 src/main/java/gov/cabinetoffice/gap/adminbackend/services/SubmissionAnonymisationService.java create mode 100644 src/main/resources/db/migration/V1_104__add_expired_submission_status.sql diff --git a/src/main/java/gov/cabinetoffice/gap/adminbackend/config/SubmissionAnonymisationConfigProperties.java b/src/main/java/gov/cabinetoffice/gap/adminbackend/config/SubmissionAnonymisationConfigProperties.java new file mode 100644 index 00000000..eefc0b36 --- /dev/null +++ b/src/main/java/gov/cabinetoffice/gap/adminbackend/config/SubmissionAnonymisationConfigProperties.java @@ -0,0 +1,16 @@ +package gov.cabinetoffice.gap.adminbackend.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Getter +@Setter +@Configuration +@ConfigurationProperties(prefix = "submission-anonymisation-scheduler") +public class SubmissionAnonymisationConfigProperties { + + private int daysBeforeExpiry = 90; + +} diff --git a/src/main/java/gov/cabinetoffice/gap/adminbackend/enums/SubmissionStatus.java b/src/main/java/gov/cabinetoffice/gap/adminbackend/enums/SubmissionStatus.java index 99428db1..d89cf5f5 100644 --- a/src/main/java/gov/cabinetoffice/gap/adminbackend/enums/SubmissionStatus.java +++ b/src/main/java/gov/cabinetoffice/gap/adminbackend/enums/SubmissionStatus.java @@ -1,5 +1,5 @@ package gov.cabinetoffice.gap.adminbackend.enums; public enum SubmissionStatus { - IN_PROGRESS, SUBMITTED, GRANT_CLOSED + IN_PROGRESS, SUBMITTED, GRANT_CLOSED, EXPIRED } diff --git a/src/main/java/gov/cabinetoffice/gap/adminbackend/repositories/GrantAttachmentRepository.java b/src/main/java/gov/cabinetoffice/gap/adminbackend/repositories/GrantAttachmentRepository.java index 25ed092f..a3efcdc8 100644 --- a/src/main/java/gov/cabinetoffice/gap/adminbackend/repositories/GrantAttachmentRepository.java +++ b/src/main/java/gov/cabinetoffice/gap/adminbackend/repositories/GrantAttachmentRepository.java @@ -3,10 +3,19 @@ import gov.cabinetoffice.gap.adminbackend.entities.GrantAttachment; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; +import java.util.List; import java.util.UUID; @Repository public interface GrantAttachmentRepository extends JpaRepository { + boolean existsBySubmissionId(UUID id); + + List findBySubmission_Id(UUID submissionId); + + @Transactional + void deleteBySubmission_Id(UUID submissionId); + } diff --git a/src/main/java/gov/cabinetoffice/gap/adminbackend/repositories/GrantMandatoryQuestionRepository.java b/src/main/java/gov/cabinetoffice/gap/adminbackend/repositories/GrantMandatoryQuestionRepository.java index f80e74a6..3fed809c 100644 --- a/src/main/java/gov/cabinetoffice/gap/adminbackend/repositories/GrantMandatoryQuestionRepository.java +++ b/src/main/java/gov/cabinetoffice/gap/adminbackend/repositories/GrantMandatoryQuestionRepository.java @@ -3,6 +3,7 @@ import gov.cabinetoffice.gap.adminbackend.entities.GrantMandatoryQuestions; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; +import org.springframework.transaction.annotation.Transactional; import java.util.List; import java.util.UUID; @@ -35,4 +36,7 @@ public interface GrantMandatoryQuestionRepository extends JpaRepository findByStatusAndLastUpdatedBefore(SubmissionStatus status, LocalDateTime cutoff); + + @Transactional + @Modifying + @Query(value = """ + UPDATE grant_submission + SET status = 'EXPIRED', + definition = NULL, + submission_name = NULL, + gap_id = NULL, + applicant_id = NULL, + created_by = NULL, + last_updated_by = NULL, + last_updated = :now + WHERE id IN :ids + """, nativeQuery = true) + void anonymiseSubmissions(@Param("ids") List ids, @Param("now") LocalDateTime now); + + @Transactional + @Modifying + @Query(value = "DELETE FROM grant_beneficiary WHERE submission_id IN :ids", nativeQuery = true) + void deleteBeneficiaryRowsBySubmissionIds(@Param("ids") List ids); + + @Transactional + @Modifying + @Query(value = "DELETE FROM diligence_check WHERE submission_id IN :ids", nativeQuery = true) + void deleteDiligenceCheckRowsBySubmissionIds(@Param("ids") List ids); + } diff --git a/src/main/java/gov/cabinetoffice/gap/adminbackend/schedulers/SubmissionAnonymisationScheduler.java b/src/main/java/gov/cabinetoffice/gap/adminbackend/schedulers/SubmissionAnonymisationScheduler.java new file mode 100644 index 00000000..2bcadcc7 --- /dev/null +++ b/src/main/java/gov/cabinetoffice/gap/adminbackend/schedulers/SubmissionAnonymisationScheduler.java @@ -0,0 +1,51 @@ +package gov.cabinetoffice.gap.adminbackend.schedulers; + +import gov.cabinetoffice.gap.adminbackend.config.SubmissionAnonymisationConfigProperties; +import gov.cabinetoffice.gap.adminbackend.entities.Submission; +import gov.cabinetoffice.gap.adminbackend.enums.SubmissionStatus; +import gov.cabinetoffice.gap.adminbackend.repositories.SubmissionRepository; +import gov.cabinetoffice.gap.adminbackend.services.SubmissionAnonymisationService; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.util.List; + +@Log4j2 +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(name = "submission-anonymisation-scheduler.enabled", havingValue = "true") +public class SubmissionAnonymisationScheduler { + + private final SubmissionRepository submissionRepository; + + private final SubmissionAnonymisationService submissionAnonymisationService; + + private final SubmissionAnonymisationConfigProperties config; + + @Scheduled(cron = "${submission-anonymisation-scheduler.cronExpression:0 0 3 * * ?}", zone = "UTC") + @SchedulerLock(name = "submissionAnonymisation_anonymiseInactiveSubmissions", + lockAtMostFor = "${submission-anonymisation-scheduler.lock.atMostFor:30m}", + lockAtLeastFor = "${submission-anonymisation-scheduler.lock.atLeastFor:5m}") + public void anonymiseInactiveSubmissions() { + final LocalDateTime cutoff = LocalDateTime.now().minusDays(config.getDaysBeforeExpiry()); + + log.info("Submission anonymisation scheduler started. Anonymising IN_PROGRESS submissions last updated before {}", + cutoff); + + final List dueForAnonymisation = submissionRepository + .findByStatusAndLastUpdatedBefore(SubmissionStatus.IN_PROGRESS, cutoff); + + log.info("Found {} submission(s) to anonymise", dueForAnonymisation.size()); + + dueForAnonymisation + .forEach(submission -> submissionAnonymisationService.anonymiseSubmission(submission.getId())); + + log.info("Submission anonymisation scheduler completed successfully."); + } + +} diff --git a/src/main/java/gov/cabinetoffice/gap/adminbackend/services/S3Service.java b/src/main/java/gov/cabinetoffice/gap/adminbackend/services/S3Service.java index ec4e6af1..66d8dc95 100644 --- a/src/main/java/gov/cabinetoffice/gap/adminbackend/services/S3Service.java +++ b/src/main/java/gov/cabinetoffice/gap/adminbackend/services/S3Service.java @@ -3,6 +3,8 @@ import com.amazonaws.HttpMethod; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.AmazonS3URI; +import com.amazonaws.services.s3.model.DeleteObjectRequest; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; @@ -18,13 +20,24 @@ public class S3Service { @Value("${cloud.aws.s3.submissions-export-bucket-name}") - private String attachmentsBucket; + private String exportBucketName; + + @Value("${aws.attachmentsBucket}") + private String attachmentsBucketName; private final AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient(); + public void deleteAttachment(String location) { + final AmazonS3URI s3Uri = new AmazonS3URI(location); + final String key = s3Uri.getKey(); + log.info("Deleting S3 object {} from bucket {}", key, s3Uri.getBucket()); + s3Client.deleteObject(new DeleteObjectRequest(s3Uri.getBucket(), key)); + s3Client.deleteObject(new DeleteObjectRequest(attachmentsBucketName, key)); + } + public String generateExportDocSignedUrl(String objectKey) { int linkTimeoutDuration = 604800; - GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(attachmentsBucket, + GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(exportBucketName, objectKey).withMethod(HttpMethod.GET) .withExpiration(Date.from(Instant.now().plusSeconds(linkTimeoutDuration))); diff --git a/src/main/java/gov/cabinetoffice/gap/adminbackend/services/SubmissionAnonymisationService.java b/src/main/java/gov/cabinetoffice/gap/adminbackend/services/SubmissionAnonymisationService.java new file mode 100644 index 00000000..17986af5 --- /dev/null +++ b/src/main/java/gov/cabinetoffice/gap/adminbackend/services/SubmissionAnonymisationService.java @@ -0,0 +1,68 @@ +package gov.cabinetoffice.gap.adminbackend.services; + +import gov.cabinetoffice.gap.adminbackend.entities.GrantAttachment; +import gov.cabinetoffice.gap.adminbackend.repositories.GrantAttachmentRepository; +import gov.cabinetoffice.gap.adminbackend.repositories.GrantMandatoryQuestionRepository; +import gov.cabinetoffice.gap.adminbackend.repositories.SubmissionRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.UUID; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class SubmissionAnonymisationService { + + private final SubmissionRepository submissionRepository; + + private final GrantAttachmentRepository grantAttachmentRepository; + + private final GrantMandatoryQuestionRepository grantMandatoryQuestionRepository; + + private final S3Service s3Service; + + @Transactional + public void anonymiseSubmission(UUID submissionId) { + log.info("Anonymising submission {}", submissionId); + + // 1. Delete S3 objects — if any fail, abort and leave the submission in + // IN_PROGRESS so the scheduler retries it on the next run + final List attachments = grantAttachmentRepository.findBySubmission_Id(submissionId); + for (GrantAttachment attachment : attachments) { + try { + s3Service.deleteAttachment(attachment.getLocation()); + log.debug("Deleted S3 object {} for submission {}", attachment.getLocation(), submissionId); + } + catch (Exception e) { + log.warn( + "Aborting anonymisation of submission {} — failed to delete S3 object {}: {}. " + + "Submission will be retried on the next scheduler run.", + submissionId, attachment.getLocation(), e.getMessage()); + return; + } + } + + // 2. Delete diligence_check rows (no cascade on this FK) + submissionRepository.deleteDiligenceCheckRowsBySubmissionIds(List.of(submissionId)); + + // 3. Delete grant_beneficiary rows + submissionRepository.deleteBeneficiaryRowsBySubmissionIds(List.of(submissionId)); + + // 4. Delete mandatory question rows + grantMandatoryQuestionRepository.deleteBySubmission_Id(submissionId); + + // 5. Delete attachment DB rows + grantAttachmentRepository.deleteBySubmission_Id(submissionId); + + // 6. Null out personal data on the submission row and mark EXPIRED + submissionRepository.anonymiseSubmissions(List.of(submissionId), LocalDateTime.now()); + + log.info("Anonymisation complete for submission {}", submissionId); + } + +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 634e719c..5a6b036f 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -30,6 +30,7 @@ cloud.aws.sqs.submissions-export-queue=cloud-platform-gap-apply-submissions-expo cloud.aws.sqs.event-service-queue=gap-apply-events-service-queue cloud.aws.sqs.event-service-queue-enabled=true cloud.aws.s3.submissions-export-bucket-name=cloud-platform-gap-apply-submissions-export-bucket +aws.attachmentsBucket=attachments gov-notify-api-key=secretGovNotifyApiKey gov-notify-lambda-export-template-id=exportEmailTemplateId @@ -45,6 +46,13 @@ completion-statistics-scheduler.cronExpression=0 6 * * * ? completion-statistics-scheduler.lock.atMostFor=30m completion-statistics-scheduler.lock.atLeastFor=5m +#submissionAnonymisationScheduler configurable properties +submission-anonymisation-scheduler.enabled=false +submission-anonymisation-scheduler.cronExpression=0 0 3 * * ? +submission-anonymisation-scheduler.daysBeforeExpiry=90 +submission-anonymisation-scheduler.lock.atMostFor=30m +submission-anonymisation-scheduler.lock.atLeastFor=5m + #grantAdvertsScheduler configurable properties grant-adverts-scheduler.cronExpression=0 01 0 * * ? grant-adverts-scheduler.lock.atMostFor=30m diff --git a/src/main/resources/db/migration/V1_104__add_expired_submission_status.sql b/src/main/resources/db/migration/V1_104__add_expired_submission_status.sql new file mode 100644 index 00000000..6098e616 --- /dev/null +++ b/src/main/resources/db/migration/V1_104__add_expired_submission_status.sql @@ -0,0 +1,2 @@ +COMMENT ON COLUMN grant_submission.status IS + 'Valid values: IN_PROGRESS, SUBMITTED, GRANT_CLOSED, EXPIRED'; From 3be2311c507d9de06dab6bcc907ba0b504a6c712 Mon Sep 17 00:00:00 2001 From: rmohammed-goaco Date: Mon, 18 May 2026 13:30:10 +0100 Subject: [PATCH 3/3] Add gapId to ExportedSubmissionsDto and implement mapping in CustomGrantExportMapper (#340) - Added a new field `gapId` to the `ExportedSubmissionsDto` class. - Updated the `CustomGrantExportMapperImpl` to map the `gapId` from `GrantExportEntity`. - Enhanced the `GrantExportMapper` interface to include the new mapping method for `gapId`. --- .../dtos/grantExport/ExportedSubmissionsDto.java | 1 + .../mappers/CustomGrantExportMapperImpl.java | 14 ++++++++++++++ .../adminbackend/mappers/GrantExportMapper.java | 5 +++++ 3 files changed, 20 insertions(+) diff --git a/src/main/java/gov/cabinetoffice/gap/adminbackend/dtos/grantExport/ExportedSubmissionsDto.java b/src/main/java/gov/cabinetoffice/gap/adminbackend/dtos/grantExport/ExportedSubmissionsDto.java index 9f082540..9a5a26e2 100644 --- a/src/main/java/gov/cabinetoffice/gap/adminbackend/dtos/grantExport/ExportedSubmissionsDto.java +++ b/src/main/java/gov/cabinetoffice/gap/adminbackend/dtos/grantExport/ExportedSubmissionsDto.java @@ -13,6 +13,7 @@ @Builder public class ExportedSubmissionsDto { + private String gapId; private String name; private String zipFileLocation; private UUID submissionId; diff --git a/src/main/java/gov/cabinetoffice/gap/adminbackend/mappers/CustomGrantExportMapperImpl.java b/src/main/java/gov/cabinetoffice/gap/adminbackend/mappers/CustomGrantExportMapperImpl.java index d3d04e2f..076f9d53 100644 --- a/src/main/java/gov/cabinetoffice/gap/adminbackend/mappers/CustomGrantExportMapperImpl.java +++ b/src/main/java/gov/cabinetoffice/gap/adminbackend/mappers/CustomGrantExportMapperImpl.java @@ -61,6 +61,7 @@ public ExportedSubmissionsDto grantExportEntityToExportedSubmissions(GrantExport exportedSubmissionsDto.name(mapExportedSubmissionName(grantExportEntity)); exportedSubmissionsDto.submittedDate(mapExportedSubmissionSubmittedDate(grantExportEntity)); exportedSubmissionsDto.submissionName(mapExportedSubmissionSubmissionName(grantExportEntity)); + exportedSubmissionsDto.gapId(mapExportedSubmissionGapId(grantExportEntity)); return exportedSubmissionsDto.build(); } @@ -111,6 +112,19 @@ public String mapExportedSubmissionSubmissionName(GrantExportEntity grantExportE return submission.get().getSubmissionName(); } + @Override + public String mapExportedSubmissionGapId(GrantExportEntity grantExportEntity) { + log.info("Getting gap ID from grant export {} and submission {}", grantExportEntity.getId(), + grantExportEntity.getId().getSubmissionId()); + final UUID submissionId = grantExportEntity.getId().getSubmissionId(); + final Optional submission = submissionRepository.findById(submissionId); + if (submission.isEmpty()) { + log.error("Submission not found for id: {}", submissionId); + return null; + } + return submission.get().getGapId(); + } + private UUID grantExportEntityIdSubmissionId(GrantExportEntity grantExportEntity) { if (grantExportEntity == null) { return null; diff --git a/src/main/java/gov/cabinetoffice/gap/adminbackend/mappers/GrantExportMapper.java b/src/main/java/gov/cabinetoffice/gap/adminbackend/mappers/GrantExportMapper.java index 94dc4194..e62eb728 100644 --- a/src/main/java/gov/cabinetoffice/gap/adminbackend/mappers/GrantExportMapper.java +++ b/src/main/java/gov/cabinetoffice/gap/adminbackend/mappers/GrantExportMapper.java @@ -22,6 +22,7 @@ public interface GrantExportMapper { @Mapping(target ="status", source = "status") @Mapping(target = "submittedDate", expression = "java(mapExportedSubmissionSubmittedDate(grantExportEntity))") @Mapping(target = "submissionName", expression = "java(mapExportedSubmissionSubmissionName(grantExportEntity))") + @Mapping(target = "gapId", expression = "java(mapExportedSubmissionGapId(grantExportEntity))") ExportedSubmissionsDto grantExportEntityToExportedSubmissions(GrantExportEntity grantExportEntity); default String mapExportedSubmissionName(GrantExportEntity grantExportEntity) { @@ -36,4 +37,8 @@ default String mapExportedSubmissionSubmissionName(GrantExportEntity grantExport return ""; } + default String mapExportedSubmissionGapId(GrantExportEntity grantExportEntity) { + return ""; + } + }