diff --git a/.github/workflows/codex-review.yml b/.github/workflows/codex-review.yml index 2b3ac38..561b866 100644 --- a/.github/workflows/codex-review.yml +++ b/.github/workflows/codex-review.yml @@ -20,7 +20,7 @@ jobs: if: ${{ github.event.pull_request.draft == false }} env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - CODEX_REVIEW_MODEL: ${{ vars.CODEX_REVIEW_MODEL || 'gpt-5.4-mini' }} + CODEX_REVIEW_MODEL: ${{ vars.CODEX_REVIEW_MODEL || 'gpt-5.6-luna' }} steps: - name: Checkout base branch prompts @@ -37,7 +37,7 @@ jobs: const marker = ''; const maxDiffChars = 90000; - const model = process.env.CODEX_REVIEW_MODEL || 'gpt-5.4-mini'; + const model = process.env.CODEX_REVIEW_MODEL || 'gpt-5.6-luna'; const apiKey = process.env.OPENAI_API_KEY; if (!apiKey) { diff --git a/src/main/java/com/son/soccerStreaming/admin/repository/AdminSyncJobRepository.java b/src/main/java/com/son/soccerStreaming/admin/repository/AdminSyncJobRepository.java index b449930..c3c040f 100644 --- a/src/main/java/com/son/soccerStreaming/admin/repository/AdminSyncJobRepository.java +++ b/src/main/java/com/son/soccerStreaming/admin/repository/AdminSyncJobRepository.java @@ -24,6 +24,9 @@ public interface AdminSyncJobRepository extends JpaRepository findAllByStatusIn(Collection statuses); + boolean existsByTaskAndDetailsAndStatusIn( + String task, String details, Collection statuses); + @EntityGraph(attributePaths = "adminUser") List findAllByStatusInOrderByCreatedAtDesc(Collection statuses); diff --git a/src/main/java/com/son/soccerStreaming/admin/service/AdminService.java b/src/main/java/com/son/soccerStreaming/admin/service/AdminService.java index 6046dbd..6630212 100644 --- a/src/main/java/com/son/soccerStreaming/admin/service/AdminService.java +++ b/src/main/java/com/son/soccerStreaming/admin/service/AdminService.java @@ -6,6 +6,9 @@ import com.son.soccerStreaming.apifootball.service.ApiFootballPlayerSyncService; import com.son.soccerStreaming.apifootball.service.ApiFootballStandingSyncService; import com.son.soccerStreaming.apifootball.service.ApiFootballTeamSyncService; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncAlreadyRunningException; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; +import com.son.soccerStreaming.apifootball.scheduler.ApiFootballSyncFailureRetryScheduler; import com.son.soccerStreaming.apifootball.service.LeagueSeasonCoverageSyncService; import com.son.soccerStreaming.admin.dto.AdminDto; import com.son.soccerStreaming.admin.entity.AdminAuditLog; @@ -162,6 +165,8 @@ public class AdminService { private final LeagueSeasonCoverageSyncService leagueSeasonCoverageSyncService; private final AdminSyncTaskRunner adminSyncTaskRunner; private final AdminSyncJobService adminSyncJobService; + private final ApiFootballSyncExecutionGuard apiFootballSyncExecutionGuard; + private final ApiFootballSyncFailureRetryScheduler apiFootballSyncFailureRetryScheduler; private final MediaUrlService mediaUrlService; private final ConcurrentMap manualSyncStates = new ConcurrentHashMap<>(); @@ -643,9 +648,10 @@ public AdminDto.SyncStatusResponse getSyncStatuses(Integer season) { private AdminDto.SyncResponse runSync(Long adminUserId, String task, String targetType, Long targetId, String details, SyncTask syncTask) { AppUser admin = findUser(adminUserId); String syncKey = manualSyncKey(task, details); - acquireManualSync(syncKey); + ApiFootballSyncExecutionGuard.Lease reservation = acquireManualSync(syncKey); try { int count = syncTask.run(); + apiFootballSyncFailureRetryScheduler.cancelPendingByExecutionKey(syncKey); String message = task + " sync completed. " + details + "; count=" + count; adminAuditLogRepository.save(AdminAuditLog.of(admin, AdminAuditType.SYNC, targetType, targetId, message, details, true)); return AdminDto.SyncResponse.builder() @@ -665,7 +671,7 @@ private AdminDto.SyncResponse runSync(Long adminUserId, String task, String targ .message(message) .build(); } finally { - releaseManualSync(syncKey); + releaseManualSync(syncKey, reservation); } } @@ -677,7 +683,7 @@ public AdminDto.SyncCancelResponse cancelSyncJob(Long adminUserId, Long jobId) { AppUser admin = findUser(adminUserId); AdminSyncJobService.CancelResult result = adminSyncJobService.requestCancel(jobId); if (result.cancelledBeforeStart()) { - releaseManualSync(manualSyncKey(result.task(), result.details())); + releaseCurrentManualSync(manualSyncKey(result.task(), result.details())); } String message = result.cancelledBeforeStart() ? result.task() + " sync cancelled before start." @@ -698,11 +704,25 @@ private AdminDto.SyncResponse queueSync(Long adminUserId, String task, String ta // Keep the admin request short; the runner writes start/completion audit logs from a worker thread. findUser(adminUserId); String syncKey = manualSyncKey(task, details); - acquireManualSync(syncKey); - var job = adminSyncJobService.create(adminUserId, task, targetType, targetId, season, details); + if (adminSyncJobService.hasActiveJob(task, details)) { + throw new CustomException(ErrorCode.ADMIN_SYNC_ALREADY_RUNNING); + } + ApiFootballSyncExecutionGuard.Lease reservation = acquireManualSync(syncKey); + final com.son.soccerStreaming.admin.entity.AdminSyncJob job; + try { + if (adminSyncJobService.hasActiveJob(task, details)) { + throw new CustomException(ErrorCode.ADMIN_SYNC_ALREADY_RUNNING); + } + job = adminSyncJobService.create(adminUserId, task, targetType, targetId, season, details); + } catch (RuntimeException exception) { + releaseManualSync(syncKey, reservation); + throw exception; + } AdminSyncTaskRunner.SyncTask trackedSyncTask = progress -> { try { - return syncTask.run(progress); + int count = syncTask.run(progress); + apiFootballSyncFailureRetryScheduler.cancelPendingByExecutionKey(syncKey); + return count; } catch (Exception exception) { recordSyncFailure(task, season, exception); throw exception; @@ -710,10 +730,10 @@ private AdminDto.SyncResponse queueSync(Long adminUserId, String task, String ta }; try { adminSyncTaskRunner.run(job.getId(), adminUserId, task, targetType, targetId, details, - trackedSyncTask, () -> releaseManualSync(syncKey)); + trackedSyncTask, () -> releaseManualSync(syncKey, reservation)); } catch (RuntimeException exception) { adminSyncJobService.markFailed(job.getId(), task + " sync could not be queued: " + exception.getMessage()); - releaseManualSync(syncKey); + releaseManualSync(syncKey, reservation); throw exception; } return AdminDto.SyncResponse.builder() @@ -730,21 +750,44 @@ private String manualSyncKey(String task, String details) { return task + ":" + details; } - private void acquireManualSync(String syncKey) { + private ApiFootballSyncExecutionGuard.Lease acquireManualSync(String syncKey) { + final ApiFootballSyncExecutionGuard.Lease reservation; + try { + reservation = apiFootballSyncExecutionGuard.acquire(syncKey); + } catch (ApiFootballSyncAlreadyRunningException exception) { + throw new CustomException(ErrorCode.ADMIN_SYNC_ALREADY_RUNNING); + } Instant now = Instant.now(); - manualSyncStates.compute(syncKey, (key, current) -> { - if (current != null && current.running()) { - throw new CustomException(ErrorCode.ADMIN_SYNC_TOO_FREQUENT); - } - if (current != null && Duration.between(current.requestedAt(), now).compareTo(MANUAL_SYNC_COOLDOWN) < 0) { - throw new CustomException(ErrorCode.ADMIN_SYNC_TOO_FREQUENT); - } - return new ManualSyncState(now, true); - }); + try { + manualSyncStates.compute(syncKey, (key, current) -> { + if (current != null && current.running()) { + throw new CustomException(ErrorCode.ADMIN_SYNC_ALREADY_RUNNING); + } + if (current != null && Duration.between(current.requestedAt(), now).compareTo(MANUAL_SYNC_COOLDOWN) < 0) { + throw new CustomException(ErrorCode.ADMIN_SYNC_TOO_FREQUENT); + } + return new ManualSyncState(now, true, reservation); + }); + return reservation; + } catch (RuntimeException exception) { + apiFootballSyncExecutionGuard.release(reservation); + throw exception; + } + } + + private void releaseManualSync(String syncKey, ApiFootballSyncExecutionGuard.Lease reservation) { + manualSyncStates.computeIfPresent(syncKey, (key, current) -> + Objects.equals(current.reservation(), reservation) + ? new ManualSyncState(current.requestedAt(), false, current.reservation()) + : current); + apiFootballSyncExecutionGuard.release(reservation); } - private void releaseManualSync(String syncKey) { - manualSyncStates.computeIfPresent(syncKey, (key, current) -> new ManualSyncState(current.requestedAt(), false)); + private void releaseCurrentManualSync(String syncKey) { + ManualSyncState current = manualSyncStates.get(syncKey); + if (current != null) { + releaseManualSync(syncKey, current.reservation()); + } } private String syncDetails(Integer league, Integer season) { @@ -889,7 +932,8 @@ private String valueText(Object value) { private record FieldChange(String fieldName, Object previousValue, Object newValue) { } - private record ManualSyncState(Instant requestedAt, boolean running) { + private record ManualSyncState(Instant requestedAt, boolean running, + ApiFootballSyncExecutionGuard.Lease reservation) { } private Fixture findFixtureWithTeams(Long fixtureId) { diff --git a/src/main/java/com/son/soccerStreaming/admin/service/AdminSyncJobService.java b/src/main/java/com/son/soccerStreaming/admin/service/AdminSyncJobService.java index fcd1db1..8dad08a 100644 --- a/src/main/java/com/son/soccerStreaming/admin/service/AdminSyncJobService.java +++ b/src/main/java/com/son/soccerStreaming/admin/service/AdminSyncJobService.java @@ -28,6 +28,12 @@ @RequiredArgsConstructor public class AdminSyncJobService { + private static final List ACTIVE_STATUSES = List.of( + AdminSyncJobStatus.QUEUED, + AdminSyncJobStatus.RUNNING, + AdminSyncJobStatus.CANCEL_REQUESTED + ); + private final AdminSyncJobRepository jobRepository; private final AdminSyncJobErrorRepository errorRepository; private final AppUserRepository appUserRepository; @@ -40,6 +46,11 @@ public AdminSyncJob create(Long adminUserId, String task, String targetType, Lon return jobRepository.save(AdminSyncJob.queued(admin, task, targetType, targetId, season, details)); } + @Transactional(readOnly = true) + public boolean hasActiveJob(String task, String details) { + return jobRepository.existsByTaskAndDetailsAndStatusIn(task, details, ACTIVE_STATUSES); + } + @Transactional(propagation = Propagation.REQUIRES_NEW) public boolean markRunning(Long jobId) { return job(jobId).markRunning(); @@ -93,16 +104,14 @@ public CancelResult requestCancel(Long jobId) { @Transactional(readOnly = true) public AdminDto.SyncJobListResponse recentJobs(Integer limit) { int safeLimit = limit == null ? 10 : Math.min(Math.max(limit, 1), 50); - List activeStatuses = List.of( - AdminSyncJobStatus.QUEUED, AdminSyncJobStatus.RUNNING, AdminSyncJobStatus.CANCEL_REQUESTED); List activeJobs = new ArrayList<>( - jobRepository.findAllByStatusInOrderByCreatedAtDesc(activeStatuses)); + jobRepository.findAllByStatusInOrderByCreatedAtDesc(ACTIVE_STATUSES)); activeJobs.sort(Comparator .comparingInt((AdminSyncJob job) -> activeRank(job.getStatus())) .thenComparing(AdminSyncJob::getCreatedAt)); List jobs = new ArrayList<>(activeJobs); jobs.addAll(jobRepository.findAllByStatusNotInOrderByCompletedAtDesc( - activeStatuses, PageRequest.of(0, safeLimit))); + ACTIVE_STATUSES, PageRequest.of(0, safeLimit))); List ids = jobs.stream().map(AdminSyncJob::getId).toList(); Map> errorsByJob = ids.isEmpty() ? Map.of() diff --git a/src/main/java/com/son/soccerStreaming/apifootball/client/ApiFootballCircuitBreaker.java b/src/main/java/com/son/soccerStreaming/apifootball/client/ApiFootballCircuitBreaker.java index 094c474..8a9b47a 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/client/ApiFootballCircuitBreaker.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/client/ApiFootballCircuitBreaker.java @@ -1,6 +1,5 @@ package com.son.soccerStreaming.apifootball.client; -import com.son.soccerStreaming.global.externalapi.ExternalApiErrorCategory; import com.son.soccerStreaming.global.externalapi.ExternalApiException; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -49,9 +48,7 @@ public void recordSuccess() { } public void recordFailure(String operation, Exception exception) { - if (!enabled || exception instanceof ApiFootballCircuitOpenException - || exception instanceof ExternalApiException external - && external.getCategory() == ExternalApiErrorCategory.CIRCUIT_OPEN) { + if (!enabled || !countsAsProviderFailure(exception)) { return; } int failures = consecutiveFailures.incrementAndGet(); @@ -65,6 +62,19 @@ public void recordFailure(String operation, Exception exception) { operation, failures, openUntil); } + boolean countsAsProviderFailure(Exception exception) { + if (exception instanceof ApiFootballCircuitOpenException) { + return false; + } + if (!(exception instanceof ExternalApiException external)) { + return true; + } + return switch (external.getCategory()) { + case TIMEOUT, NETWORK, UPSTREAM_SERVER, INVALID_RESPONSE -> true; + default -> false; + }; + } + private int configuredFailureThreshold() { return Math.max(1, failureThreshold); } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/client/ApiFootballClient.java b/src/main/java/com/son/soccerStreaming/apifootball/client/ApiFootballClient.java index da17442..ff1b6bc 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/client/ApiFootballClient.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/client/ApiFootballClient.java @@ -6,6 +6,7 @@ import com.son.soccerStreaming.apifootball.dto.ApiFootballLineupDto; import com.son.soccerStreaming.apifootball.dto.ApiFootballLiveDto; import com.son.soccerStreaming.apifootball.dto.ApiFootballPlayerDto; +import com.son.soccerStreaming.apifootball.dto.ApiFootballResponseEnvelope; import com.son.soccerStreaming.apifootball.dto.ApiFootballStandingDto; import com.son.soccerStreaming.apifootball.dto.ApiFootballTeamDto; import lombok.RequiredArgsConstructor; @@ -267,18 +268,21 @@ public List getStandings(Integer league return standingResponseOf(body); } - private T get(String operation, String path, ParameterizedTypeReference responseType, Object... uriVariables) { + private > T get( + String operation, String path, ParameterizedTypeReference responseType, Object... uriVariables) { try { - return externalApiExecutor.execute(ExternalApiProvider.API_FOOTBALL, operation, () -> { + T response = externalApiExecutor.execute(ExternalApiProvider.API_FOOTBALL, operation, () -> { apiFootballCircuitBreaker.beforeRequest(operation); - T response = apiFootballRestClient.get() + var responseEntity = apiFootballRestClient.get() .uri(baseUrl + path, uriVariables) .headers(this::setApiHeaders) .retrieve() - .body(responseType); - apiFootballCircuitBreaker.recordSuccess(); - return response; + .toEntity(responseType); + return ApiFootballResponseValidator.validate( + operation, responseEntity.getStatusCode(), responseEntity.getBody()); }); + apiFootballCircuitBreaker.recordSuccess(); + return response; } catch (ExternalApiException exception) { apiFootballCircuitBreaker.recordFailure(operation, exception); throw exception; diff --git a/src/main/java/com/son/soccerStreaming/apifootball/client/ApiFootballResponseValidator.java b/src/main/java/com/son/soccerStreaming/apifootball/client/ApiFootballResponseValidator.java new file mode 100644 index 0000000..b9ffd2a --- /dev/null +++ b/src/main/java/com/son/soccerStreaming/apifootball/client/ApiFootballResponseValidator.java @@ -0,0 +1,59 @@ +package com.son.soccerStreaming.apifootball.client; + +import com.fasterxml.jackson.databind.JsonNode; +import com.son.soccerStreaming.apifootball.dto.ApiFootballResponseEnvelope; +import com.son.soccerStreaming.global.externalapi.ExternalApiErrorCategory; +import com.son.soccerStreaming.global.externalapi.ExternalApiException; +import com.son.soccerStreaming.global.externalapi.ExternalApiProvider; +import org.springframework.http.HttpStatus; +import org.springframework.http.HttpStatusCode; + +import java.util.List; + +final class ApiFootballResponseValidator { + + private ApiFootballResponseValidator() { + } + + static > T validate( + String operation, HttpStatusCode status, T body) { + if (status.value() == HttpStatus.NO_CONTENT.value()) { + return body; + } + if (body == null) { + throw invalidResponse(operation, status, "API-Football returned an empty response body"); + } + + JsonNode errors = body.getErrors(); + if (errors == null) { + throw invalidResponse(operation, status, "API-Football response did not contain errors metadata"); + } + if (!errors.isContainerNode() || !errors.isEmpty()) { + throw invalidResponse(operation, status, "API-Football returned an error response"); + } + + Integer results = body.getResults(); + List response = body.getResponse(); + if (results == null || results < 0 || response == null) { + throw invalidResponse(operation, status, "API-Football response envelope was incomplete"); + } + if (results != response.size()) { + throw invalidResponse(operation, status, "API-Football response result count was inconsistent"); + } + return body; + } + + private static ExternalApiException invalidResponse( + String operation, HttpStatusCode status, String message) { + return new ExternalApiException( + ExternalApiProvider.API_FOOTBALL, + operation, + ExternalApiErrorCategory.INVALID_RESPONSE, + status.value(), + false, + null, + message, + null + ); + } +} diff --git a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballFixtureStatisticsDto.java b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballFixtureStatisticsDto.java index dae89b0..e56499d 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballFixtureStatisticsDto.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballFixtureStatisticsDto.java @@ -14,8 +14,7 @@ private ApiFootballFixtureStatisticsDto() { @Getter @NoArgsConstructor @JsonIgnoreProperties(ignoreUnknown = true) - public static class ApiResponse { - private List response; + public static class ApiResponse extends ApiFootballResponseEnvelope { } @Getter diff --git a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballInjuryDto.java b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballInjuryDto.java index 3734e8a..dec013d 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballInjuryDto.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballInjuryDto.java @@ -14,8 +14,7 @@ private ApiFootballInjuryDto() { @Getter @NoArgsConstructor @JsonIgnoreProperties(ignoreUnknown = true) - public static class ApiResponse { - private List response; + public static class ApiResponse extends ApiFootballResponseEnvelope { } @Getter diff --git a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballLeagueDto.java b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballLeagueDto.java index 29604ea..c500ba5 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballLeagueDto.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballLeagueDto.java @@ -16,8 +16,7 @@ private ApiFootballLeagueDto() { @Getter @NoArgsConstructor @JsonIgnoreProperties(ignoreUnknown = true) - public static class ApiResponse { - private List response; + public static class ApiResponse extends ApiFootballResponseEnvelope { } @Getter diff --git a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballLineupDto.java b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballLineupDto.java index 0e0fc98..f637fd9 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballLineupDto.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballLineupDto.java @@ -15,8 +15,7 @@ private ApiFootballLineupDto() { @Getter @NoArgsConstructor @JsonIgnoreProperties(ignoreUnknown = true) - public static class ApiResponse { - private List response; + public static class ApiResponse extends ApiFootballResponseEnvelope { } @Getter diff --git a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballLiveDto.java b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballLiveDto.java index df980da..5b83a62 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballLiveDto.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballLiveDto.java @@ -15,8 +15,7 @@ private ApiFootballLiveDto() { @Getter @NoArgsConstructor @JsonIgnoreProperties(ignoreUnknown = true) - public static class ApiResponse { - private List response; + public static class ApiResponse extends ApiFootballResponseEnvelope { } @Getter diff --git a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballPlayerDto.java b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballPlayerDto.java index 9923463..07a2859 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballPlayerDto.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballPlayerDto.java @@ -14,8 +14,7 @@ private ApiFootballPlayerDto() { @Getter @NoArgsConstructor @JsonIgnoreProperties(ignoreUnknown = true) - public static class ApiResponse { - private List response; + public static class ApiResponse extends ApiFootballResponseEnvelope { private Paging paging; } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballResponseEnvelope.java b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballResponseEnvelope.java new file mode 100644 index 0000000..e344a2a --- /dev/null +++ b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballResponseEnvelope.java @@ -0,0 +1,18 @@ +package com.son.soccerStreaming.apifootball.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Getter +@NoArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +public class ApiFootballResponseEnvelope { + + private JsonNode errors; + private Integer results; + private List response; +} diff --git a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballStandingDto.java b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballStandingDto.java index 779554f..0b9dfcd 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballStandingDto.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballStandingDto.java @@ -15,8 +15,7 @@ private ApiFootballStandingDto() { @Getter @NoArgsConstructor @JsonIgnoreProperties(ignoreUnknown = true) - public static class ApiResponse { - private List response; + public static class ApiResponse extends ApiFootballResponseEnvelope { } @Getter diff --git a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballTeamDto.java b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballTeamDto.java index 6b01055..63517f4 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballTeamDto.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/dto/ApiFootballTeamDto.java @@ -14,8 +14,7 @@ private ApiFootballTeamDto() { @Getter @NoArgsConstructor @JsonIgnoreProperties(ignoreUnknown = true) - public static class ApiResponse { - private List response; + public static class ApiResponse extends ApiFootballResponseEnvelope { } @Getter diff --git a/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballFixtureDetailStartupSyncRunner.java b/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballFixtureDetailStartupSyncRunner.java index 403f068..b5170a2 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballFixtureDetailStartupSyncRunner.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballFixtureDetailStartupSyncRunner.java @@ -3,6 +3,7 @@ import com.son.soccerStreaming.apifootball.scheduler.ApiFootballSyncFailureRetryScheduler; import com.son.soccerStreaming.apifootball.service.ApiFootballFixtureDetailSyncException; import com.son.soccerStreaming.apifootball.service.ApiFootballFixtureDetailSyncService; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -28,17 +29,19 @@ public class ApiFootballFixtureDetailStartupSyncRunner implements CommandLineRun @Override public void run(String... args) { + String syncKey = ApiFootballSyncExecutionGuard.key("fixture-details", "season=" + season); log.info("API-Football startup fixture detail sync started. season={}", season); try { int syncedCount = apiFootballFixtureDetailSyncService.syncSeasonFixtureDetails(season, false); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); log.info("API-Football startup fixture detail sync completed. season={}, count={}", season, syncedCount); } catch (Exception e) { log.error("API-Football startup fixture detail sync failed. season={}", season, e); - scheduleRetry(e); + scheduleRetry(syncKey, e); } } - private void scheduleRetry(Exception exception) { + private void scheduleRetry(String syncKey, Exception exception) { if (!failureRetryScheduler.shouldRetry(exception)) { return; } @@ -48,7 +51,9 @@ private void scheduleRetry(Exception exception) { String fixtureIds = chunk.stream().map(String::valueOf).collect(java.util.stream.Collectors.joining("-")); failureRetryScheduler.schedule( "startup:fixture-details:%s:chunk:%s".formatted(season, fixtureIds), + syncKey, "startup fixture detail sync season=%s chunk=%s".formatted(season, index++), + exception, () -> apiFootballFixtureDetailSyncService.syncFixtureDetailsByIds(chunk, false) ); } @@ -57,7 +62,9 @@ private void scheduleRetry(Exception exception) { failureRetryScheduler.schedule( "startup:fixture-details:%s".formatted(season), + syncKey, "startup fixture detail sync season=%s".formatted(season), + exception, () -> apiFootballFixtureDetailSyncService.syncSeasonFixtureDetails(season, false) ); } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballFixtureStartupSyncRunner.java b/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballFixtureStartupSyncRunner.java index 3f4be9f..ac0e6b8 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballFixtureStartupSyncRunner.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballFixtureStartupSyncRunner.java @@ -1,6 +1,7 @@ package com.son.soccerStreaming.apifootball.runner; import com.son.soccerStreaming.apifootball.scheduler.ApiFootballSyncFailureRetryScheduler; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import com.son.soccerStreaming.apifootball.service.ApiFootballFixtureSyncService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -30,15 +31,20 @@ public class ApiFootballFixtureStartupSyncRunner implements CommandLineRunner { @Override public void run(String... args) { + String syncKey = ApiFootballSyncExecutionGuard.key( + "fixtures", "league=%s; season=%s".formatted(league, season)); try { log.info("API-Football startup fixture sync started."); apiFootballFixtureSyncService.syncSeasonFixtures(league, season); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); } catch (Exception e) { log.error("API-Football startup fixture sync failed. league={}, season={}", league, season, e); if (!failureRetryScheduler.shouldRetry(e)) return; failureRetryScheduler.schedule( "startup:fixtures:%s:%s".formatted(league, season), + syncKey, "startup fixture sync league=%s season=%s".formatted(league, season), + e, () -> apiFootballFixtureSyncService.syncSeasonFixtures(league, season) ); } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballInjuryStartupSyncRunner.java b/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballInjuryStartupSyncRunner.java index d45ddec..bce0609 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballInjuryStartupSyncRunner.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballInjuryStartupSyncRunner.java @@ -1,6 +1,7 @@ package com.son.soccerStreaming.apifootball.runner; import com.son.soccerStreaming.apifootball.scheduler.ApiFootballSyncFailureRetryScheduler; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import com.son.soccerStreaming.apifootball.service.ApiFootballInjurySyncService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -30,15 +31,20 @@ public class ApiFootballInjuryStartupSyncRunner implements CommandLineRunner { @Override public void run(String... args) { + String syncKey = ApiFootballSyncExecutionGuard.key( + "injuries", "league=%s; season=%s".formatted(league, season)); log.info("API-Football startup injury sync started."); try { apiFootballInjurySyncService.syncInjuries(league, season); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); } catch (Exception e) { log.error("API-Football startup injury sync failed. league={}, season={}", league, season, e); if (!failureRetryScheduler.shouldRetry(e)) return; failureRetryScheduler.schedule( "startup:injuries:%s:%s".formatted(league, season), + syncKey, "startup injury sync league=%s season=%s".formatted(league, season), + e, () -> apiFootballInjurySyncService.syncInjuries(league, season) ); } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballRegisteredPlayerStartupSyncRunner.java b/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballRegisteredPlayerStartupSyncRunner.java index 91138a6..abb2a2b 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballRegisteredPlayerStartupSyncRunner.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballRegisteredPlayerStartupSyncRunner.java @@ -3,6 +3,7 @@ import com.son.soccerStreaming.apifootball.scheduler.ApiFootballSyncFailureRetryScheduler; import com.son.soccerStreaming.apifootball.service.ApiFootballPlayerSyncService; import com.son.soccerStreaming.apifootball.service.ApiFootballRegisteredPlayerSyncException; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -34,16 +35,19 @@ public class ApiFootballRegisteredPlayerStartupSyncRunner implements CommandLine @Override public void run(String... args) { + String syncKey = ApiFootballSyncExecutionGuard.key( + "players", "league=%s; season=%s".formatted(league, season)); log.info("API-Football startup registered player sync started. league={}, season={}", league, season); try { apiFootballPlayerSyncService.syncRegisteredPlayers(league, season, delayMs); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); } catch (Exception e) { log.error("API-Football startup registered player sync failed. league={}, season={}", league, season, e); - scheduleRetry(e); + scheduleRetry(syncKey, e); } } - private void scheduleRetry(Exception exception) { + private void scheduleRetry(String syncKey, Exception exception) { if (!failureRetryScheduler.shouldRetry(exception)) { return; } @@ -51,7 +55,9 @@ private void scheduleRetry(Exception exception) { for (Long teamId : playerSyncException.getFailedTeamIds()) { failureRetryScheduler.schedule( "startup:registered-players:%s:%s:team:%s".formatted(league, season, teamId), + syncKey, "startup registered player sync league=%s season=%s teamId=%s".formatted(league, season, teamId), + exception, () -> apiFootballPlayerSyncService.syncRegisteredPlayersByTeamId(teamId, league, season, delayMs) ); } @@ -60,7 +66,9 @@ private void scheduleRetry(Exception exception) { failureRetryScheduler.schedule( "startup:registered-players:%s:%s".formatted(league, season), + syncKey, "startup registered player sync league=%s season=%s".formatted(league, season), + exception, () -> apiFootballPlayerSyncService.syncRegisteredPlayers(league, season, delayMs) ); } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballStandingStartupSyncRunner.java b/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballStandingStartupSyncRunner.java index 96053fc..d7ead3b 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballStandingStartupSyncRunner.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballStandingStartupSyncRunner.java @@ -1,6 +1,7 @@ package com.son.soccerStreaming.apifootball.runner; import com.son.soccerStreaming.apifootball.scheduler.ApiFootballSyncFailureRetryScheduler; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import com.son.soccerStreaming.apifootball.service.ApiFootballStandingSyncService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -30,15 +31,20 @@ public class ApiFootballStandingStartupSyncRunner implements CommandLineRunner { @Override public void run(String... args) { + String syncKey = ApiFootballSyncExecutionGuard.key( + "standings", "league=%s; season=%s".formatted(league, season)); log.info("API-Football startup standing sync started."); try { apiFootballStandingSyncService.syncStandings(league, season); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); } catch (Exception e) { log.error("API-Football startup standing sync failed. league={}, season={}", league, season, e); if (!failureRetryScheduler.shouldRetry(e)) return; failureRetryScheduler.schedule( "startup:standings:%s:%s".formatted(league, season), + syncKey, "startup standing sync league=%s season=%s".formatted(league, season), + e, () -> apiFootballStandingSyncService.syncStandings(league, season) ); } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballTeamStartupSyncRunner.java b/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballTeamStartupSyncRunner.java index 472b7e6..7efe7c1 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballTeamStartupSyncRunner.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/runner/ApiFootballTeamStartupSyncRunner.java @@ -1,6 +1,7 @@ package com.son.soccerStreaming.apifootball.runner; import com.son.soccerStreaming.apifootball.scheduler.ApiFootballSyncFailureRetryScheduler; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import com.son.soccerStreaming.apifootball.service.ApiFootballTeamSyncService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -30,15 +31,20 @@ public class ApiFootballTeamStartupSyncRunner implements CommandLineRunner { @Override public void run(String... args) { + String syncKey = ApiFootballSyncExecutionGuard.key( + "teams", "league=%s; season=%s".formatted(league, season)); log.info("API-Football startup team sync started."); try { apiFootballTeamSyncService.syncTeams(league, season); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); } catch (Exception e) { log.error("API-Football startup team sync failed. league={}, season={}", league, season, e); if (!failureRetryScheduler.shouldRetry(e)) return; failureRetryScheduler.schedule( "startup:teams:%s:%s".formatted(league, season), + syncKey, "startup team sync league=%s season=%s".formatted(league, season), + e, () -> apiFootballTeamSyncService.syncTeams(league, season) ); } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/runner/LeagueSeasonCoverageStartupSyncRunner.java b/src/main/java/com/son/soccerStreaming/apifootball/runner/LeagueSeasonCoverageStartupSyncRunner.java index 873796e..a5e77f4 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/runner/LeagueSeasonCoverageStartupSyncRunner.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/runner/LeagueSeasonCoverageStartupSyncRunner.java @@ -1,6 +1,7 @@ package com.son.soccerStreaming.apifootball.runner; import com.son.soccerStreaming.apifootball.scheduler.ApiFootballSyncFailureRetryScheduler; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import com.son.soccerStreaming.apifootball.service.LeagueSeasonCoverageSyncService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -25,15 +26,19 @@ public class LeagueSeasonCoverageStartupSyncRunner implements CommandLineRunner @Override public void run(String... args) { + String syncKey = ApiFootballSyncExecutionGuard.key("seasons", "league=" + league); log.info("API-Football startup league season coverage sync started. league={}", league); try { leagueSeasonCoverageSyncService.syncLeagueSeasons(league); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); } catch (Exception e) { log.error("API-Football startup league season coverage sync failed. league={}", league, e); if (!failureRetryScheduler.shouldRetry(e)) return; failureRetryScheduler.schedule( "startup:league-seasons:%s".formatted(league), + syncKey, "startup league season coverage sync league=%s".formatted(league), + e, () -> leagueSeasonCoverageSyncService.syncLeagueSeasons(league) ); } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballFixtureDetailSyncScheduler.java b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballFixtureDetailSyncScheduler.java index f06e925..baee3de 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballFixtureDetailSyncScheduler.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballFixtureDetailSyncScheduler.java @@ -2,6 +2,7 @@ import com.son.soccerStreaming.apifootball.service.ApiFootballFixtureDetailSyncException; import com.son.soccerStreaming.apifootball.service.ApiFootballFixtureDetailSyncService; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import com.son.soccerStreaming.fixture.entity.Fixture; import com.son.soccerStreaming.live.service.LiveFixtureBroadcastService; import com.son.soccerStreaming.fixture.repository.FixtureRepository; @@ -24,43 +25,63 @@ public class ApiFootballFixtureDetailSyncScheduler { private final LiveFixtureBroadcastService liveFixtureBroadcastService; private final FixtureRepository fixtureRepository; private final ApiFootballSyncFailureRetryScheduler failureRetryScheduler; + private final ApiFootballSyncExecutionGuard executionGuard; @Value("${api-football.sync.fixtures.season:2025}") private Integer season; @Scheduled(cron = "${api-football.sync.fixture-details.live-cron:0 * * * * *}") public void syncLiveFixtureDetails() { + String syncKey = ApiFootballSyncExecutionGuard.key("fixture-details-live", "live"); + if (!executionGuard.executeIfAvailable(syncKey, () -> syncLiveFixtureDetailsSafely(syncKey))) { + log.info("API-Football live fixture detail sync skipped because the same job is active. syncKey={}", syncKey); + } + } + + private void syncLiveFixtureDetailsSafely(String syncKey) { try { - syncLiveFixtureDetailsNow(); + if (syncLiveFixtureDetailsNow()) { + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); + } } catch (Exception e) { log.error("API-Football live fixture detail sync failed.", e); - scheduleFixtureDetailRetry("live", true, e); + scheduleFixtureDetailRetry(syncKey, "live", true, e); } } @Scheduled(cron = "${api-football.sync.fixture-details.daily-cron:0 55 4 * * *}") public void syncFixtureDetailsDaily() { + String syncKey = ApiFootballSyncExecutionGuard.key("fixture-details", "season=" + season); + if (!executionGuard.executeIfAvailable(syncKey, () -> syncFixtureDetailsDailyNow(syncKey))) { + log.info("API-Football daily fixture detail sync skipped because the same job is active. syncKey={}", syncKey); + } + } + + private void syncFixtureDetailsDailyNow(String syncKey) { try { apiFootballFixtureDetailSyncService.syncSeasonFixtureDetails(season, false); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); } catch (Exception e) { log.error("API-Football daily fixture detail sync failed.", e); - scheduleFixtureDetailRetry("daily:%s".formatted(season), false, e); + scheduleFixtureDetailRetry(syncKey, "daily:%s".formatted(season), false, e); } } - private void syncLiveFixtureDetailsNow() { + private boolean syncLiveFixtureDetailsNow() { List liveFixtures = fixtureRepository.findAllByFixtureStatus("LIVE"); if (liveFixtures.isEmpty()) { - return; + return false; } apiFootballFixtureDetailSyncService.syncFixtureDetailsWithResults( liveFixtures, true ).forEach(result -> liveFixtureBroadcastService.broadcastFixture(result.fixtureId(), result.latestEvent())); + return true; } - private void scheduleFixtureDetailRetry(String reason, boolean applyLiveStandingImpact, Exception exception) { + private void scheduleFixtureDetailRetry(String syncKey, String reason, + boolean applyLiveStandingImpact, Exception exception) { if (!failureRetryScheduler.shouldRetry(exception)) { return; } @@ -70,7 +91,9 @@ private void scheduleFixtureDetailRetry(String reason, boolean applyLiveStanding String fixtureIds = chunk.stream().map(String::valueOf).collect(java.util.stream.Collectors.joining("-")); failureRetryScheduler.schedule( "fixture-details:%s:chunk:%s".formatted(reason, fixtureIds), + syncKey, "fixture detail sync reason=%s chunk=%s".formatted(reason, index++), + exception, () -> apiFootballFixtureDetailSyncService.syncFixtureDetailsByIds(chunk, applyLiveStandingImpact) ); } @@ -79,7 +102,9 @@ private void scheduleFixtureDetailRetry(String reason, boolean applyLiveStanding failureRetryScheduler.schedule( "fixture-details:%s".formatted(reason), + syncKey, "fixture detail sync reason=%s".formatted(reason), + exception, applyLiveStandingImpact ? this::syncLiveFixtureDetailsNow : () -> apiFootballFixtureDetailSyncService.syncSeasonFixtureDetails(season, false) ); diff --git a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballFixtureSyncScheduler.java b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballFixtureSyncScheduler.java index 32875fb..ef4ae93 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballFixtureSyncScheduler.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballFixtureSyncScheduler.java @@ -1,6 +1,7 @@ package com.son.soccerStreaming.apifootball.scheduler; import com.son.soccerStreaming.apifootball.service.ApiFootballFixtureSyncService; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import com.son.soccerStreaming.fixture.repository.FixtureRepository; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -24,6 +25,7 @@ public class ApiFootballFixtureSyncScheduler { private final ApiFootballFixtureSyncService apiFootballFixtureSyncService; private final FixtureRepository fixtureRepository; private final ApiFootballSyncFailureRetryScheduler failureRetryScheduler; + private final ApiFootballSyncExecutionGuard executionGuard; @Value("${api-football.sync.fixtures.league:39}") private Integer league; @@ -48,28 +50,51 @@ public void syncLiveFixturesWhenMatchWindowOpen() { return; } + String syncKey = ApiFootballSyncExecutionGuard.key( + "fixtures-live", "league=%s; season=%s".formatted(league, season)); + if (!executionGuard.executeIfAvailable(syncKey, () -> syncLiveFixturesNow(syncKey))) { + log.info("API-Football live fixture sync skipped because the same job is active. syncKey={}", syncKey); + } + } + + private void syncLiveFixturesNow(String syncKey) { try { apiFootballFixtureSyncService.syncLiveFixtures(league, season); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); } catch (Exception e) { log.error("API-Football live fixture sync failed.", e); if (!failureRetryScheduler.shouldRetry(e)) return; failureRetryScheduler.schedule( "fixtures:live:%s:%s".formatted(league, season), + syncKey, "live fixture sync league=%s season=%s".formatted(league, season), + e, () -> apiFootballFixtureSyncService.syncLiveFixtures(league, season) ); } } private void syncSeasonFixtures(String reason) { + String syncKey = ApiFootballSyncExecutionGuard.key( + "fixtures", "league=%s; season=%s".formatted(league, season)); + if (!executionGuard.executeIfAvailable(syncKey, () -> syncSeasonFixturesNow(reason, syncKey))) { + log.info("API-Football season fixture sync skipped because the same job is active. syncKey={}, reason={}", + syncKey, reason); + } + } + + private void syncSeasonFixturesNow(String reason, String syncKey) { try { apiFootballFixtureSyncService.syncSeasonFixtures(league, season); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); } catch (Exception e) { log.error("API-Football season fixture sync failed. reason={}, league={}, season={}", reason, league, season, e); if (!failureRetryScheduler.shouldRetry(e)) return; failureRetryScheduler.schedule( "fixtures:%s:%s:%s".formatted(reason, league, season), + syncKey, "season fixture sync reason=%s league=%s season=%s".formatted(reason, league, season), + e, () -> apiFootballFixtureSyncService.syncSeasonFixtures(league, season) ); } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballInjurySyncScheduler.java b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballInjurySyncScheduler.java index 975796f..198c283 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballInjurySyncScheduler.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballInjurySyncScheduler.java @@ -1,6 +1,7 @@ package com.son.soccerStreaming.apifootball.scheduler; import com.son.soccerStreaming.apifootball.service.ApiFootballInjurySyncService; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -16,6 +17,7 @@ public class ApiFootballInjurySyncScheduler { private final ApiFootballInjurySyncService apiFootballInjurySyncService; private final ApiFootballSyncFailureRetryScheduler failureRetryScheduler; + private final ApiFootballSyncExecutionGuard executionGuard; @Value("${api-football.sync.injuries.league:39}") private Integer league; @@ -25,14 +27,25 @@ public class ApiFootballInjurySyncScheduler { @Scheduled(cron = "${api-football.sync.injuries.daily-cron:0 0 5 * * *}") public void syncInjuriesDaily() { + String syncKey = ApiFootballSyncExecutionGuard.key( + "injuries", "league=%s; season=%s".formatted(league, season)); + if (!executionGuard.executeIfAvailable(syncKey, () -> syncInjuriesNow(syncKey))) { + log.info("API-Football injury sync skipped because the same job is active. syncKey={}", syncKey); + } + } + + private void syncInjuriesNow(String syncKey) { try { apiFootballInjurySyncService.syncInjuries(league, season); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); } catch (Exception e) { log.error("API-Football injury sync failed. league={}, season={}", league, season, e); if (!failureRetryScheduler.shouldRetry(e)) return; failureRetryScheduler.schedule( "injuries:%s:%s".formatted(league, season), + syncKey, "injury sync league=%s season=%s".formatted(league, season), + e, () -> apiFootballInjurySyncService.syncInjuries(league, season) ); } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballRegisteredPlayerSyncScheduler.java b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballRegisteredPlayerSyncScheduler.java index 7b80525..e7de733 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballRegisteredPlayerSyncScheduler.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballRegisteredPlayerSyncScheduler.java @@ -2,6 +2,7 @@ import com.son.soccerStreaming.apifootball.service.ApiFootballPlayerSyncService; import com.son.soccerStreaming.apifootball.service.ApiFootballRegisteredPlayerSyncException; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -17,6 +18,7 @@ public class ApiFootballRegisteredPlayerSyncScheduler { private final ApiFootballPlayerSyncService apiFootballPlayerSyncService; private final ApiFootballSyncFailureRetryScheduler failureRetryScheduler; + private final ApiFootballSyncExecutionGuard executionGuard; @Value("${api-football.sync.players.registered.league:39}") private Integer league; @@ -29,15 +31,24 @@ public class ApiFootballRegisteredPlayerSyncScheduler { @Scheduled(cron = "${api-football.sync.players.registered.daily-cron:0 10 5 * * *}") public void syncRegisteredPlayersDaily() { + String syncKey = ApiFootballSyncExecutionGuard.key( + "players", "league=%s; season=%s".formatted(league, season)); + if (!executionGuard.executeIfAvailable(syncKey, () -> syncRegisteredPlayersNow(syncKey))) { + log.info("API-Football registered player sync skipped because the same job is active. syncKey={}", syncKey); + } + } + + private void syncRegisteredPlayersNow(String syncKey) { try { apiFootballPlayerSyncService.syncRegisteredPlayers(league, season, delayMs); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); } catch (Exception e) { log.error("API-Football registered player sync failed. league={}, season={}", league, season, e); - scheduleRetry(e); + scheduleRetry(syncKey, e); } } - private void scheduleRetry(Exception exception) { + private void scheduleRetry(String syncKey, Exception exception) { if (!failureRetryScheduler.shouldRetry(exception)) { return; } @@ -45,7 +56,9 @@ private void scheduleRetry(Exception exception) { for (Long teamId : playerSyncException.getFailedTeamIds()) { failureRetryScheduler.schedule( "registered-players:%s:%s:team:%s".formatted(league, season, teamId), + syncKey, "registered player sync league=%s season=%s teamId=%s".formatted(league, season, teamId), + exception, () -> apiFootballPlayerSyncService.syncRegisteredPlayersByTeamId(teamId, league, season, delayMs) ); } @@ -54,7 +67,9 @@ private void scheduleRetry(Exception exception) { failureRetryScheduler.schedule( "registered-players:%s:%s".formatted(league, season), + syncKey, "registered player sync league=%s season=%s".formatted(league, season), + exception, () -> apiFootballPlayerSyncService.syncRegisteredPlayers(league, season, delayMs) ); } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballStandingSyncScheduler.java b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballStandingSyncScheduler.java index 9eab486..50b1942 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballStandingSyncScheduler.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballStandingSyncScheduler.java @@ -1,6 +1,7 @@ package com.son.soccerStreaming.apifootball.scheduler; import com.son.soccerStreaming.apifootball.service.ApiFootballStandingSyncService; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import com.son.soccerStreaming.fixture.repository.FixtureRepository; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -18,6 +19,7 @@ public class ApiFootballStandingSyncScheduler { private final ApiFootballStandingSyncService apiFootballStandingSyncService; private final FixtureRepository fixtureRepository; private final ApiFootballSyncFailureRetryScheduler failureRetryScheduler; + private final ApiFootballSyncExecutionGuard executionGuard; @Value("${api-football.sync.standings.league:39}") private Integer league; @@ -39,14 +41,26 @@ public void syncStandingsHourlyWhenLive() { } private void syncStandings(String reason) { + String syncKey = ApiFootballSyncExecutionGuard.key( + "standings", "league=%s; season=%s".formatted(league, season)); + if (!executionGuard.executeIfAvailable(syncKey, () -> syncStandingsNow(reason, syncKey))) { + log.info("API-Football standing sync skipped because the same job is active. syncKey={}, reason={}", + syncKey, reason); + } + } + + private void syncStandingsNow(String reason, String syncKey) { try { apiFootballStandingSyncService.syncStandings(league, season); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); } catch (Exception e) { log.error("API-Football standing sync failed. reason={}, league={}, season={}", reason, league, season, e); if (!failureRetryScheduler.shouldRetry(e)) return; failureRetryScheduler.schedule( "standings:%s:%s:%s".formatted(reason, league, season), + syncKey, "standing sync reason=%s league=%s season=%s".formatted(reason, league, season), + e, () -> apiFootballStandingSyncService.syncStandings(league, season) ); } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballSyncFailureRetryScheduler.java b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballSyncFailureRetryScheduler.java index e7f3f6a..9b55830 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballSyncFailureRetryScheduler.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballSyncFailureRetryScheduler.java @@ -1,31 +1,36 @@ package com.son.soccerStreaming.apifootball.scheduler; import jakarta.annotation.PreDestroy; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import com.son.soccerStreaming.apifootball.service.ApiFootballSyncStatusService; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import java.time.Duration; +import java.time.Instant; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.atomic.AtomicReference; import com.son.soccerStreaming.global.externalapi.ExternalApiException; @Slf4j @Component -@RequiredArgsConstructor public class ApiFootballSyncFailureRetryScheduler { private final ApiFootballSyncStatusService syncStatusService; - private final ScheduledExecutorService retryExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> { - Thread thread = new Thread(runnable, "api-football-sync-failure-retry"); - thread.setDaemon(true); - return thread; - }); + private final ApiFootballSyncExecutionGuard executionGuard; + private final ScheduledExecutorService retryExecutor; private final Map retryStates = new ConcurrentHashMap<>(); + private final Map terminalFailedRetries = new ConcurrentHashMap<>(); @Value("${api-football.sync.failure-retry.enabled:true}") private boolean enabled; @@ -33,143 +38,325 @@ public class ApiFootballSyncFailureRetryScheduler { @Value("${api-football.sync.failure-retry.max-attempts:2}") private int maxAttempts; - @Value("${api-football.sync.failure-retry.delay-minutes:30}") - private long delayMinutes; + @Value("${api-football.sync.failure-retry.initial-delay-minutes:1}") + private long initialDelayMinutes; - public void schedule(String key, String description, Runnable retryAction) { + @Value("${api-football.sync.failure-retry.delay-multiplier:5}") + private long delayMultiplier; + + @Value("${api-football.sync.failure-retry.max-delay-minutes:30}") + private long maxDelayMinutes; + + @Autowired + public ApiFootballSyncFailureRetryScheduler(ApiFootballSyncStatusService syncStatusService, + ApiFootballSyncExecutionGuard executionGuard) { + this(syncStatusService, executionGuard, newRetryExecutor()); + } + + ApiFootballSyncFailureRetryScheduler(ApiFootballSyncStatusService syncStatusService, + ApiFootballSyncExecutionGuard executionGuard, + ScheduledExecutorService retryExecutor) { + this.syncStatusService = syncStatusService; + this.executionGuard = executionGuard; + this.retryExecutor = retryExecutor; + } + + private static ScheduledExecutorService newRetryExecutor() { + return Executors.newSingleThreadScheduledExecutor(runnable -> { + Thread thread = new Thread(runnable, "api-football-sync-failure-retry"); + thread.setDaemon(true); + return thread; + }); + } + + public void schedule(String retryKey, String executionKey, String description, + Exception failure, Runnable retryAction) { if (!enabled) { - log.warn("API-Football sync failure retry skipped because it is disabled. key={}, description={}", - key, description); + log.warn("API-Football sync failure retry skipped because it is disabled. retryKey={}, executionKey={}, description={}", + retryKey, executionKey, description); return; } int configuredMaxAttempts = Math.max(0, maxAttempts); if (configuredMaxAttempts == 0) { - log.error("API-Football sync failure retry exhausted. alert=api-football-sync-retry-exhausted, key={}, description={}, maxAttempts=0", - key, description); + log.error("API-Football sync failure retry exhausted. alert=api-football-sync-retry-exhausted, retryKey={}, executionKey={}, description={}, maxAttempts=0", + retryKey, executionKey, description); return; } - RetryState newState = new RetryState(description, retryAction, configuredMaxAttempts); - RetryState existingState = retryStates.putIfAbsent(key, newState); + RetryState newState = new RetryState(executionKey, description, retryAction, configuredMaxAttempts); + // A new automatic/startup synchronization supersedes terminal failures from its previous run. + // Partial retries use scheduleNext directly, so they do not clear sibling terminal failures here. + terminalFailedRetries.entrySet().removeIf(entry -> entry.getValue().equals(executionKey)); + RetryState existingState = retryStates.putIfAbsent(retryKey, newState); if (existingState != null) { - log.warn("API-Football sync failure retry already pending. key={}, description={}, currentAttempt={}/{}", - key, existingState.description(), existingState.attempt(), existingState.maxAttempts()); + log.warn("API-Football sync failure retry already pending. retryKey={}, executionKey={}, description={}, currentAttempt={}/{}", + retryKey, existingState.executionKey(), existingState.description(), + existingState.attempt(), existingState.maxAttempts()); return; } - syncStatusOfRetryKey(key).ifPresent(status -> - syncStatusService.recordRetryPendingByKey(status.syncKey(), status.displayName(), - "Retry scheduled. " + description)); - scheduleNext(key, newState); + scheduleNext(retryKey, newState, failure, false); } public boolean shouldRetry(Exception exception) { - Throwable current = exception; - while (current != null) { - if (current instanceof ExternalApiException externalApiException) { - return externalApiException.isRetryable(); + return externalApiException(exception) + .map(ExternalApiException::isRetryable) + .orElse(true); + } + + public int cancelPendingByExecutionKey(String executionKey) { + int cancelledCount = 0; + for (Map.Entry entry : retryStates.entrySet()) { + RetryState state = entry.getValue(); + if (!state.executionKey().equals(executionKey)) { + continue; + } + if (retryStates.remove(entry.getKey(), state)) { + state.cancel(); + cancelledCount++; + log.info("API-Football pending retry cancelled after synchronization success. retryKey={}, executionKey={}, attempt={}/{}", + entry.getKey(), executionKey, state.attempt(), state.maxAttempts()); } - current = current.getCause(); } - return true; + terminalFailedRetries.entrySet().removeIf(entry -> entry.getValue().equals(executionKey)); + return cancelledCount; } @PreDestroy public void shutdown() { + retryStates.values().forEach(RetryState::cancel); + retryStates.clear(); + terminalFailedRetries.clear(); retryExecutor.shutdownNow(); } - private void scheduleNext(String key, RetryState state) { - long delay = Math.max(0, delayMinutes); + private void scheduleNext(String retryKey, RetryState state, Exception failure, boolean activeJobDeferred) { + if (!isCurrent(retryKey, state)) { + return; + } + + Duration delay = activeJobDeferred + ? Duration.ofMinutes(Math.max(0, initialDelayMinutes)) + : retryDelay(state.attempt(), failure); + long delayMillis = safeMillis(delay); int nextAttempt = state.attempt() + 1; - log.warn("API-Football sync failure retry scheduled. key={}, description={}, attempt={}/{}, delayMinutes={}", - key, state.description(), nextAttempt, state.maxAttempts(), delay); + Instant nextAttemptAt = Instant.now().plusMillis(delayMillis); + boolean retryAfterApplied = !activeJobDeferred && retryAfter(failure).isPresent(); + + log.atWarn() + .addKeyValue("event.action", "api-football-sync-retry") + .addKeyValue("event.outcome", "scheduled") + .addKeyValue("api_football.retry_key", retryKey) + .addKeyValue("api_football.execution_key", state.executionKey()) + .addKeyValue("api_football.retry_attempt", nextAttempt) + .addKeyValue("api_football.retry_max_attempts", state.maxAttempts()) + .addKeyValue("api_football.retry_delay_ms", delayMillis) + .addKeyValue("api_football.retry_at", nextAttemptAt) + .addKeyValue("api_football.retry_after_applied", retryAfterApplied) + .log("API-Football sync failure retry scheduled."); + + if (!activeJobDeferred) { + syncStatusOfRetryKey(retryKey).ifPresent(status -> + syncStatusService.recordRetryPendingByKey(status.syncKey(), status.displayName(), + "Retry scheduled for %s (attempt %d/%d, delay=%dms, retryAfter=%s). %s" + .formatted(nextAttemptAt, nextAttempt, state.maxAttempts(), delayMillis, + retryAfterApplied, state.description()))); + } - retryExecutor.schedule(() -> runRetry(key), delay, TimeUnit.MINUTES); + ScheduledFuture future = retryExecutor.schedule( + () -> runRetry(retryKey, state), delayMillis, TimeUnit.MILLISECONDS); + state.replaceScheduledFuture(future); + if (!isCurrent(retryKey, state)) { + state.cancel(); + } + } + + private void runRetry(String retryKey, RetryState state) { + if (!isCurrent(retryKey, state)) { + return; + } + + if (!executionGuard.executeIfAvailable(state.executionKey(), () -> runRetryNow(retryKey, state))) { + if (!isCurrent(retryKey, state)) { + return; + } + log.info("API-Football retry deferred because the same synchronization is active. retryKey={}, executionKey={}", + retryKey, state.executionKey()); + scheduleNext(retryKey, state, null, true); + } } - private void runRetry(String key) { - RetryState state = retryStates.get(key); - if (state == null) { + private void runRetryNow(String retryKey, RetryState state) { + if (!isCurrent(retryKey, state)) { return; } int attempt = state.incrementAttempt(); try { - log.info("API-Football sync failure retry started. key={}, description={}, attempt={}/{}", - key, state.description(), attempt, state.maxAttempts()); + log.info("API-Football sync failure retry started. retryKey={}, executionKey={}, description={}, attempt={}/{}", + retryKey, state.executionKey(), state.description(), attempt, state.maxAttempts()); state.retryAction().run(); - retryStates.remove(key); - log.info("API-Football sync failure retry succeeded. key={}, description={}, attempt={}/{}", - key, state.description(), attempt, state.maxAttempts()); - } catch (Exception e) { - if (!shouldRetry(e)) { - retryStates.remove(key); - syncStatusOfRetryKey(key).ifPresent(status -> - syncStatusService.recordFailureByKey(status.syncKey(), status.displayName(), e)); - log.error("API-Football sync failure retry stopped for a non-retryable error. key={}, description={}, attempt={}/{}", - key, state.description(), attempt, state.maxAttempts(), e); + if (retryStates.remove(retryKey, state)) { + state.cancel(); + recordSuccessWhenNoRelatedRetryRemains(retryKey); + } + log.info("API-Football sync failure retry succeeded. retryKey={}, executionKey={}, description={}, attempt={}/{}", + retryKey, state.executionKey(), state.description(), attempt, state.maxAttempts()); + } catch (Exception exception) { + if (!isCurrent(retryKey, state)) { + return; + } + if (!shouldRetry(exception)) { + removeAndRecordFailure(retryKey, state, exception); + log.error("API-Football sync failure retry stopped for a non-retryable error. retryKey={}, executionKey={}, description={}, attempt={}/{}", + retryKey, state.executionKey(), state.description(), attempt, state.maxAttempts(), exception); return; } if (attempt >= state.maxAttempts()) { - retryStates.remove(key); - syncStatusOfRetryKey(key).ifPresent(status -> - syncStatusService.recordFailureByKey(status.syncKey(), status.displayName(), e)); - log.error("API-Football sync failure retry exhausted. alert=api-football-sync-retry-exhausted, key={}, description={}, attempts={}", - key, state.description(), attempt, e); + removeAndRecordFailure(retryKey, state, exception); + log.error("API-Football sync failure retry exhausted. alert=api-football-sync-retry-exhausted, retryKey={}, executionKey={}, description={}, attempts={}", + retryKey, state.executionKey(), state.description(), attempt, exception); return; } - log.error("API-Football sync failure retry failed. key={}, description={}, attempt={}/{}", - key, state.description(), attempt, state.maxAttempts(), e); - syncStatusOfRetryKey(key).ifPresent(status -> - syncStatusService.recordRetryPendingByKey(status.syncKey(), status.displayName(), - "Retry failed and was rescheduled. " + state.description())); - scheduleNext(key, state); + log.error("API-Football sync failure retry failed. retryKey={}, executionKey={}, description={}, attempt={}/{}", + retryKey, state.executionKey(), state.description(), attempt, state.maxAttempts(), exception); + scheduleNext(retryKey, state, exception, false); + } + } + + private void removeAndRecordFailure(String retryKey, RetryState state, Exception exception) { + if (retryStates.remove(retryKey, state)) { + state.cancel(); + } + terminalFailedRetries.put(retryKey, state.executionKey()); + syncStatusOfRetryKey(retryKey).ifPresent(status -> + syncStatusService.recordFailureByKey(status.syncKey(), status.displayName(), exception)); + } + + private void recordSuccessWhenNoRelatedRetryRemains(String completedRetryKey) { + if (!completedRetryKey.contains(":team:") && !completedRetryKey.contains(":chunk:")) { + return; + } + Optional completedTarget = syncStatusOfRetryKey(completedRetryKey); + if (completedTarget.isEmpty()) { + return; + } + SyncStatusTarget target = completedTarget.get(); + boolean relatedRetryRemains = retryStates.keySet().stream() + .map(this::syncStatusOfRetryKey) + .flatMap(Optional::stream) + .anyMatch(target::equals); + boolean relatedRetryFailed = terminalFailedRetries.keySet().stream() + .map(this::syncStatusOfRetryKey) + .flatMap(Optional::stream) + .anyMatch(target::equals); + if (!relatedRetryRemains && !relatedRetryFailed) { + syncStatusService.recordSuccessByKey(target.syncKey(), target.displayName()); + } + } + + Duration retryDelay(int completedAttempts, Exception failure) { + Optional retryAfter = retryAfter(failure); + if (retryAfter.isPresent()) { + return retryAfter.get(); + } + + long initial = Math.max(0, initialDelayMinutes); + long maximum = Math.max(initial, maxDelayMinutes); + long multiplier = Math.max(1, delayMultiplier); + long delay = initial; + for (int index = 0; index < completedAttempts && delay < maximum; index++) { + if (delay > maximum / multiplier) { + delay = maximum; + } else { + delay = Math.min(maximum, delay * multiplier); + } + } + return Duration.ofMinutes(delay); + } + + private Optional retryAfter(Exception exception) { + return externalApiException(exception) + .map(ExternalApiException::getRetryAfter) + .filter(delay -> !delay.isNegative()); + } + + private Optional externalApiException(Throwable exception) { + Throwable current = exception; + while (current != null) { + if (current instanceof ExternalApiException externalApiException) { + return Optional.of(externalApiException); + } + current = current.getCause(); } + return Optional.empty(); } - private java.util.Optional syncStatusOfRetryKey(String key) { + private boolean isCurrent(String retryKey, RetryState state) { + return !state.isCancelled() && retryStates.get(retryKey) == state; + } + + private long safeMillis(Duration duration) { + try { + return Math.max(0, duration.toMillis()); + } catch (ArithmeticException overflow) { + return Long.MAX_VALUE; + } + } + + int pendingRetryCount() { + return retryStates.size(); + } + + private Optional syncStatusOfRetryKey(String key) { String[] parts = key.split(":"); if (parts.length == 0) { - return java.util.Optional.empty(); + return Optional.empty(); } return switch (parts[0]) { case "teams" -> seasonTarget("teams", "Teams", parts, 2); case "standings" -> seasonTarget("standings", "Standings", parts, parts.length - 1); case "fixtures" -> seasonTarget("fixtures", "Fixtures", parts, parts.length - 1); - case "fixture-details" -> seasonTarget("fixture-details", "Season Details", parts, 1); + case "fixture-details" -> parts.length > 2 && "daily".equals(parts[1]) + ? seasonTarget("fixture-details", "Season Details", parts, 2) + : Optional.empty(); case "registered-players" -> seasonTarget("players", "Players", parts, 2); case "injuries" -> seasonTarget("injuries", "Injuries", parts, 2); case "league-seasons" -> parts.length > 1 - ? java.util.Optional.of(new SyncStatusTarget("league-seasons:" + parts[1], "League Seasons")) - : java.util.Optional.empty(); + ? Optional.of(new SyncStatusTarget("league-seasons:" + parts[1], "League Seasons")) + : Optional.empty(); case "startup" -> startupTarget(parts); - default -> java.util.Optional.empty(); + default -> Optional.empty(); }; } - private java.util.Optional startupTarget(String[] parts) { + private Optional startupTarget(String[] parts) { if (parts.length < 3) { - return java.util.Optional.empty(); + return Optional.empty(); } return switch (parts[1]) { + case "league-seasons" -> Optional.of(new SyncStatusTarget("league-seasons:" + parts[2], "League Seasons")); + case "teams" -> seasonTarget("teams", "Teams", parts, 3); + case "standings" -> seasonTarget("standings", "Standings", parts, 3); + case "fixtures" -> seasonTarget("fixtures", "Fixtures", parts, 3); case "fixture-details" -> seasonTarget("fixture-details", "Season Details", parts, 2); case "registered-players" -> seasonTarget("players", "Players", parts, 3); - default -> java.util.Optional.empty(); + case "injuries" -> seasonTarget("injuries", "Injuries", parts, 3); + default -> Optional.empty(); }; } - private java.util.Optional seasonTarget(String task, String displayName, String[] parts, int seasonIndex) { + private Optional seasonTarget(String task, String displayName, String[] parts, int seasonIndex) { if (seasonIndex < 0 || seasonIndex >= parts.length) { - return java.util.Optional.empty(); + return Optional.empty(); } try { int season = Integer.parseInt(parts[seasonIndex]); - return java.util.Optional.of(new SyncStatusTarget("%s:%d".formatted(task, season), "%s %d".formatted(displayName, season))); + return Optional.of(new SyncStatusTarget("%s:%d".formatted(task, season), "%s %d".formatted(displayName, season))); } catch (NumberFormatException exception) { - return java.util.Optional.empty(); + return Optional.empty(); } } @@ -178,22 +365,30 @@ private record SyncStatusTarget(String syncKey, String displayName) { private static class RetryState { - private static final java.util.concurrent.atomic.AtomicIntegerFieldUpdater ATTEMPT_UPDATER = - java.util.concurrent.atomic.AtomicIntegerFieldUpdater.newUpdater(RetryState.class, "attempt"); + private static final AtomicIntegerFieldUpdater ATTEMPT_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(RetryState.class, "attempt"); + private final String executionKey; private final String description; private final Runnable retryAction; private final int maxAttempts; + private final AtomicBoolean cancelled = new AtomicBoolean(); + private final AtomicReference> scheduledFuture = new AtomicReference<>(); @SuppressWarnings("unused") private volatile int attempt; - private RetryState(String description, Runnable retryAction, int maxAttempts) { + private RetryState(String executionKey, String description, Runnable retryAction, int maxAttempts) { + this.executionKey = executionKey; this.description = description; this.retryAction = retryAction; this.maxAttempts = maxAttempts; } + String executionKey() { + return executionKey; + } + String description() { return description; } @@ -213,5 +408,27 @@ int attempt() { int incrementAttempt() { return ATTEMPT_UPDATER.incrementAndGet(this); } + + boolean isCancelled() { + return cancelled.get(); + } + + void replaceScheduledFuture(ScheduledFuture future) { + ScheduledFuture previous = scheduledFuture.getAndSet(future); + if (previous != null && !previous.isDone()) { + previous.cancel(false); + } + if (cancelled.get()) { + future.cancel(false); + } + } + + void cancel() { + cancelled.set(true); + ScheduledFuture future = scheduledFuture.getAndSet(null); + if (future != null) { + future.cancel(false); + } + } } } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballTeamSyncScheduler.java b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballTeamSyncScheduler.java index cfaa8f6..6589b5e 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballTeamSyncScheduler.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballTeamSyncScheduler.java @@ -1,6 +1,7 @@ package com.son.soccerStreaming.apifootball.scheduler; import com.son.soccerStreaming.apifootball.service.ApiFootballTeamSyncService; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -16,6 +17,7 @@ public class ApiFootballTeamSyncScheduler { private final ApiFootballTeamSyncService apiFootballTeamSyncService; private final ApiFootballSyncFailureRetryScheduler failureRetryScheduler; + private final ApiFootballSyncExecutionGuard executionGuard; @Value("${api-football.sync.teams.league:39}") private Integer league; @@ -25,14 +27,25 @@ public class ApiFootballTeamSyncScheduler { @Scheduled(cron = "${api-football.sync.teams.cron:0 0 4 * * *}") public void syncTeams() { + String syncKey = ApiFootballSyncExecutionGuard.key( + "teams", "league=%s; season=%s".formatted(league, season)); + if (!executionGuard.executeIfAvailable(syncKey, () -> syncTeamsNow(syncKey))) { + log.info("API-Football team sync skipped because the same job is active. syncKey={}", syncKey); + } + } + + private void syncTeamsNow(String syncKey) { try { apiFootballTeamSyncService.syncTeams(league, season); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); } catch (Exception e) { log.error("API-Football team sync failed. league={}, season={}", league, season, e); if (!failureRetryScheduler.shouldRetry(e)) return; failureRetryScheduler.schedule( "teams:%s:%s".formatted(league, season), + syncKey, "team sync league=%s season=%s".formatted(league, season), + e, () -> apiFootballTeamSyncService.syncTeams(league, season) ); } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/LeagueSeasonCoverageSyncScheduler.java b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/LeagueSeasonCoverageSyncScheduler.java index f1be2f9..ae50d08 100644 --- a/src/main/java/com/son/soccerStreaming/apifootball/scheduler/LeagueSeasonCoverageSyncScheduler.java +++ b/src/main/java/com/son/soccerStreaming/apifootball/scheduler/LeagueSeasonCoverageSyncScheduler.java @@ -1,6 +1,7 @@ package com.son.soccerStreaming.apifootball.scheduler; import com.son.soccerStreaming.apifootball.service.LeagueSeasonCoverageSyncService; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -16,20 +17,31 @@ public class LeagueSeasonCoverageSyncScheduler { private final LeagueSeasonCoverageSyncService leagueSeasonCoverageSyncService; private final ApiFootballSyncFailureRetryScheduler failureRetryScheduler; + private final ApiFootballSyncExecutionGuard executionGuard; @Value("${api-football.sync.league-seasons.league:39}") private Integer league; @Scheduled(cron = "${api-football.sync.league-seasons.daily-cron:0 0 3 * * *}") public void syncLeagueSeasons() { + String syncKey = ApiFootballSyncExecutionGuard.key("seasons", "league=" + league); + if (!executionGuard.executeIfAvailable(syncKey, () -> syncLeagueSeasonsNow(syncKey))) { + log.info("API-Football league season sync skipped because the same job is active. syncKey={}", syncKey); + } + } + + private void syncLeagueSeasonsNow(String syncKey) { try { leagueSeasonCoverageSyncService.syncLeagueSeasons(league); + failureRetryScheduler.cancelPendingByExecutionKey(syncKey); } catch (Exception e) { log.error("API-Football league season coverage sync failed. league={}", league, e); if (!failureRetryScheduler.shouldRetry(e)) return; failureRetryScheduler.schedule( "league-seasons:%s".formatted(league), + syncKey, "league season coverage sync league=%s".formatted(league), + e, () -> leagueSeasonCoverageSyncService.syncLeagueSeasons(league) ); } diff --git a/src/main/java/com/son/soccerStreaming/apifootball/service/ApiFootballSyncAlreadyRunningException.java b/src/main/java/com/son/soccerStreaming/apifootball/service/ApiFootballSyncAlreadyRunningException.java new file mode 100644 index 0000000..9977ec2 --- /dev/null +++ b/src/main/java/com/son/soccerStreaming/apifootball/service/ApiFootballSyncAlreadyRunningException.java @@ -0,0 +1,8 @@ +package com.son.soccerStreaming.apifootball.service; + +public class ApiFootballSyncAlreadyRunningException extends RuntimeException { + + public ApiFootballSyncAlreadyRunningException(String syncKey) { + super("API-Football synchronization is already active. syncKey=" + syncKey); + } +} diff --git a/src/main/java/com/son/soccerStreaming/apifootball/service/ApiFootballSyncExecutionGuard.java b/src/main/java/com/son/soccerStreaming/apifootball/service/ApiFootballSyncExecutionGuard.java new file mode 100644 index 0000000..ee9457d --- /dev/null +++ b/src/main/java/com/son/soccerStreaming/apifootball/service/ApiFootballSyncExecutionGuard.java @@ -0,0 +1,49 @@ +package com.son.soccerStreaming.apifootball.service; + +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +@Component +public class ApiFootballSyncExecutionGuard { + + private final Map activeSyncKeys = new ConcurrentHashMap<>(); + + public Lease acquire(String syncKey) { + UUID ownerId = UUID.randomUUID(); + if (activeSyncKeys.putIfAbsent(syncKey, ownerId) != null) { + throw new ApiFootballSyncAlreadyRunningException(syncKey); + } + return new Lease(syncKey, ownerId); + } + + public void release(Lease lease) { + if (lease != null) { + activeSyncKeys.remove(lease.syncKey(), lease.ownerId()); + } + } + + public boolean executeIfAvailable(String syncKey, Runnable action) { + final Lease lease; + try { + lease = acquire(syncKey); + } catch (ApiFootballSyncAlreadyRunningException exception) { + return false; + } + try { + action.run(); + return true; + } finally { + release(lease); + } + } + + public static String key(String task, String details) { + return task + ":" + details; + } + + public record Lease(String syncKey, UUID ownerId) { + } +} diff --git a/src/main/java/com/son/soccerStreaming/global/exception/ErrorCode.java b/src/main/java/com/son/soccerStreaming/global/exception/ErrorCode.java index 2582244..ebd4b09 100644 --- a/src/main/java/com/son/soccerStreaming/global/exception/ErrorCode.java +++ b/src/main/java/com/son/soccerStreaming/global/exception/ErrorCode.java @@ -43,6 +43,7 @@ public enum ErrorCode { INTERNAL_SERVER_ERROR(HttpStatus.INTERNAL_SERVER_ERROR, "예기치 못한 서버 오류가 발생했습니다."), // 409 CONFLICT + ADMIN_SYNC_ALREADY_RUNNING(HttpStatus.CONFLICT, "동일한 동기화 작업이 이미 대기 중이거나 실행 중입니다."), EMAIL_ALREADY_EXISTS(HttpStatus.CONFLICT, "이미 가입된 이메일입니다."); private final HttpStatus status; diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index b22aef2..17ece0f 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -100,7 +100,9 @@ api-football: failure-retry: enabled: ${API_FOOTBALL_SYNC_FAILURE_RETRY_ENABLED:true} max-attempts: ${API_FOOTBALL_SYNC_FAILURE_RETRY_MAX_ATTEMPTS:2} - delay-minutes: ${API_FOOTBALL_SYNC_FAILURE_RETRY_DELAY_MINUTES:1} + initial-delay-minutes: ${API_FOOTBALL_SYNC_FAILURE_RETRY_INITIAL_DELAY_MINUTES:${API_FOOTBALL_SYNC_FAILURE_RETRY_DELAY_MINUTES:1}} + delay-multiplier: ${API_FOOTBALL_SYNC_FAILURE_RETRY_DELAY_MULTIPLIER:5} + max-delay-minutes: ${API_FOOTBALL_SYNC_FAILURE_RETRY_MAX_DELAY_MINUTES:30} league-seasons: enabled: ${API_FOOTBALL_LEAGUE_SEASONS_SYNC_ENABLED:true} run-on-startup: ${API_FOOTBALL_LEAGUE_SEASONS_SYNC_RUN_ON_STARTUP:true} diff --git a/src/test/java/com/son/soccerStreaming/admin/service/AdminServiceTest.java b/src/test/java/com/son/soccerStreaming/admin/service/AdminServiceTest.java index b599aeb..89c3493 100644 --- a/src/test/java/com/son/soccerStreaming/admin/service/AdminServiceTest.java +++ b/src/test/java/com/son/soccerStreaming/admin/service/AdminServiceTest.java @@ -6,6 +6,10 @@ import com.son.soccerStreaming.apifootball.service.ApiFootballPlayerSyncService; import com.son.soccerStreaming.apifootball.service.ApiFootballStandingSyncService; import com.son.soccerStreaming.apifootball.service.ApiFootballTeamSyncService; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncStatusService; +import com.son.soccerStreaming.apifootball.scheduler.ApiFootballSyncFailureRetryScheduler; +import com.son.soccerStreaming.apifootball.service.SyncProgressReporter; import com.son.soccerStreaming.apifootball.service.LeagueSeasonCoverageSyncService; import com.son.soccerStreaming.admin.dto.AdminDto; import com.son.soccerStreaming.admin.entity.AdminAuditLog; @@ -16,6 +20,7 @@ import com.son.soccerStreaming.apifootball.repository.ApiFootballSyncStatusRepository; import com.son.soccerStreaming.auth.entity.AppUser; import com.son.soccerStreaming.global.exception.CustomException; +import com.son.soccerStreaming.global.exception.ErrorCode; import com.son.soccerStreaming.fixture.entity.Fixture; import com.son.soccerStreaming.fixture.repository.FixtureEventRepository; import com.son.soccerStreaming.fixture.repository.FixtureLineupRepository; @@ -53,7 +58,9 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.mock; @ExtendWith(MockitoExtension.class) class AdminServiceTest { @@ -87,6 +94,8 @@ class AdminServiceTest { @Mock private ApiFootballSyncStatusRepository apiFootballSyncStatusRepository; @Mock + private ApiFootballSyncStatusService apiFootballSyncStatusService; + @Mock private ApiFootballTeamSyncService apiFootballTeamSyncService; @Mock private ApiFootballStandingSyncService apiFootballStandingSyncService; @@ -105,6 +114,10 @@ class AdminServiceTest { @Mock private AdminSyncJobService adminSyncJobService; @Mock + private ApiFootballSyncExecutionGuard apiFootballSyncExecutionGuard; + @Mock + private ApiFootballSyncFailureRetryScheduler apiFootballSyncFailureRetryScheduler; + @Mock private MediaUrlService mediaUrlService; @InjectMocks @@ -261,6 +274,88 @@ void syncPlayersQueuesBackgroundTask() { ); } + @Test + void successfulSynchronousAdminSyncCancelsPendingAutomaticRetry() { + when(appUserRepository.findById(1L)).thenReturn(Optional.of(adminUser())); + when(leagueSeasonCoverageSyncService.syncLeagueSeasons(39)).thenReturn(3); + + AdminDto.SyncResponse response = adminService.syncLeagueSeasons(1L, 39); + + assertThat(response.isSuccess()).isTrue(); + verify(apiFootballSyncFailureRetryScheduler) + .cancelPendingByExecutionKey("seasons:league=39"); + } + + @Test + void failedAdminSyncDoesNotScheduleSlowRetry() { + when(appUserRepository.findById(1L)).thenReturn(Optional.of(adminUser())); + when(leagueSeasonCoverageSyncService.syncLeagueSeasons(39)) + .thenThrow(new RuntimeException("upstream failed")); + + AdminDto.SyncResponse response = adminService.syncLeagueSeasons(1L, 39); + + assertThat(response.isSuccess()).isFalse(); + verifyNoInteractions(apiFootballSyncFailureRetryScheduler); + } + + @Test + void queuedAdminSyncCancelsPendingRetryOnlyAfterTheWorkerTaskSucceeds() throws Exception { + when(appUserRepository.findById(1L)).thenReturn(Optional.of(adminUser())); + when(adminSyncJobService.create( + 1L, "players", "PLAYER", null, 2025, "league=39; season=2025" + )).thenReturn(AdminSyncJob.builder().id(99L).build()); + when(leagueSeasonCoverageRepository.findByLeagueIdAndSeasonYear(39, 2025)).thenReturn(Optional.of( + LeagueSeasonCoverage.builder() + .leagueId(39) + .seasonYear(2025) + .players(true) + .build() + )); + when(teamStandingRepository.existsByLeagueIdAndSeason(39, 2025)).thenReturn(true); + when(apiFootballPlayerSyncService.syncRegisteredPlayers( + eq(39), eq(2025), eq(7000L), any(SyncProgressReporter.class))) + .thenReturn(20); + + adminService.syncPlayers(1L, 39, 2025, 7000L); + + verify(apiFootballSyncFailureRetryScheduler, org.mockito.Mockito.never()) + .cancelPendingByExecutionKey(any()); + ArgumentCaptor taskCaptor = + ArgumentCaptor.forClass(AdminSyncTaskRunner.SyncTask.class); + verify(adminSyncTaskRunner).run( + eq(99L), eq(1L), eq("players"), eq("PLAYER"), eq((Long) null), + eq("league=39; season=2025"), taskCaptor.capture(), any(Runnable.class)); + + taskCaptor.getValue().run(mock(SyncProgressReporter.class)); + + verify(apiFootballSyncFailureRetryScheduler) + .cancelPendingByExecutionKey("players:league=39; season=2025"); + } + + @Test + void syncPlayersDoesNotQueueWhenSameJobIsAlreadyActive() { + when(leagueSeasonCoverageRepository.findByLeagueIdAndSeasonYear(39, 2025)).thenReturn(Optional.of( + LeagueSeasonCoverage.builder() + .leagueId(39) + .seasonYear(2025) + .players(true) + .build() + )); + when(teamStandingRepository.existsByLeagueIdAndSeason(39, 2025)).thenReturn(true); + when(appUserRepository.findById(1L)).thenReturn(Optional.of(adminUser())); + when(adminSyncJobService.hasActiveJob("players", "league=39; season=2025")).thenReturn(true); + + assertThatThrownBy(() -> adminService.syncPlayers(1L, 39, 2025, 7000L)) + .isInstanceOfSatisfying(CustomException.class, + exception -> assertThat(exception.getErrorCode()) + .isEqualTo(ErrorCode.ADMIN_SYNC_ALREADY_RUNNING)); + + verify(adminSyncJobService, org.mockito.Mockito.never()) + .create(any(), any(), any(), any(), any(), any()); + verify(adminSyncTaskRunner, org.mockito.Mockito.never()) + .run(any(), any(), any(), any(), any(), any(), any(), any()); + } + @Test void syncPlayersRequiresStandingsBeforeCreatingJob() { when(leagueSeasonCoverageRepository.findByLeagueIdAndSeasonYear(39, 2025)).thenReturn(Optional.of( diff --git a/src/test/java/com/son/soccerStreaming/admin/service/AdminSyncJobServiceTest.java b/src/test/java/com/son/soccerStreaming/admin/service/AdminSyncJobServiceTest.java index b5995bb..1ce4c10 100644 --- a/src/test/java/com/son/soccerStreaming/admin/service/AdminSyncJobServiceTest.java +++ b/src/test/java/com/son/soccerStreaming/admin/service/AdminSyncJobServiceTest.java @@ -49,6 +49,19 @@ void createsAndCompletesPersistentJob() { assertThat(job.getCompletedAt()).isNotNull(); } + @Test + void detectsQueuedRunningAndCancellationRequestedAsActiveJobs() { + List activeStatuses = List.of( + AdminSyncJobStatus.QUEUED, + AdminSyncJobStatus.RUNNING, + AdminSyncJobStatus.CANCEL_REQUESTED + ); + when(jobRepository.existsByTaskAndDetailsAndStatusIn( + "players", "league=39; season=2025", activeStatuses)).thenReturn(true); + + assertThat(service.hasActiveJob("players", "league=39; season=2025")).isTrue(); + } + @Test void failureAfterSuccessfulUnitsBecomesPartialFailure() { AdminSyncJob job = AdminSyncJob.queued(AppUser.builder().email("admin@example.com").build(), diff --git a/src/test/java/com/son/soccerStreaming/apifootball/client/ApiFootballCircuitBreakerTest.java b/src/test/java/com/son/soccerStreaming/apifootball/client/ApiFootballCircuitBreakerTest.java index 3f7eb79..24c3302 100644 --- a/src/test/java/com/son/soccerStreaming/apifootball/client/ApiFootballCircuitBreakerTest.java +++ b/src/test/java/com/son/soccerStreaming/apifootball/client/ApiFootballCircuitBreakerTest.java @@ -1,9 +1,14 @@ package com.son.soccerStreaming.apifootball.client; +import com.son.soccerStreaming.global.externalapi.ExternalApiErrorCategory; +import com.son.soccerStreaming.global.externalapi.ExternalApiException; +import com.son.soccerStreaming.global.externalapi.ExternalApiProvider; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.test.util.ReflectionTestUtils; +import java.time.Duration; + import static org.assertj.core.api.Assertions.assertThatThrownBy; class ApiFootballCircuitBreakerTest { @@ -33,4 +38,34 @@ void opensAfterThresholdAndRecoversOnSuccess() { circuitBreaker.beforeRequest("players"); } + + @Test + void ignoresRequestAndRateLimitFailures() { + circuitBreaker.recordFailure("fixtures", failure(ExternalApiErrorCategory.BAD_REQUEST)); + circuitBreaker.recordFailure("fixtures", failure(ExternalApiErrorCategory.RATE_LIMITED)); + + circuitBreaker.beforeRequest("players"); + } + + @Test + void opensForProviderAvailabilityFailures() { + circuitBreaker.recordFailure("fixtures", failure(ExternalApiErrorCategory.TIMEOUT)); + circuitBreaker.recordFailure("standings", failure(ExternalApiErrorCategory.UPSTREAM_SERVER)); + + assertThatThrownBy(() -> circuitBreaker.beforeRequest("players")) + .isInstanceOf(ApiFootballCircuitOpenException.class); + } + + private ExternalApiException failure(ExternalApiErrorCategory category) { + return new ExternalApiException( + ExternalApiProvider.API_FOOTBALL, + "test", + category, + null, + false, + Duration.ZERO, + "test failure", + null + ); + } } diff --git a/src/test/java/com/son/soccerStreaming/apifootball/client/ApiFootballResponseValidatorTest.java b/src/test/java/com/son/soccerStreaming/apifootball/client/ApiFootballResponseValidatorTest.java new file mode 100644 index 0000000..0b61da8 --- /dev/null +++ b/src/test/java/com/son/soccerStreaming/apifootball/client/ApiFootballResponseValidatorTest.java @@ -0,0 +1,76 @@ +package com.son.soccerStreaming.apifootball.client; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.son.soccerStreaming.apifootball.dto.ApiFootballLiveDto; +import com.son.soccerStreaming.global.externalapi.ExternalApiErrorCategory; +import com.son.soccerStreaming.global.externalapi.ExternalApiException; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpStatus; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ApiFootballResponseValidatorTest { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Test + void acceptsAValidEmptyResponse() throws Exception { + ApiFootballLiveDto.ApiResponse body = response(""" + {"errors": [], "results": 0, "response": []} + """); + + assertThat(ApiFootballResponseValidator.validate("getEvents", HttpStatus.OK, body)) + .isSameAs(body); + } + + @Test + void acceptsNoContentAsAValidEmptyResult() { + ApiFootballLiveDto.ApiResponse result = ApiFootballResponseValidator.validate( + "getEvents", HttpStatus.NO_CONTENT, (ApiFootballLiveDto.ApiResponse) null); + + assertThat(result).isNull(); + } + + @Test + void rejectsAnEmptyBodyForARegularSuccessResponse() { + assertThatThrownBy(() -> + ApiFootballResponseValidator.validate("getEvents", HttpStatus.OK, null)) + .isInstanceOfSatisfying(ExternalApiException.class, exception -> { + assertThat(exception.getCategory()).isEqualTo(ExternalApiErrorCategory.INVALID_RESPONSE); + assertThat(exception.isRetryable()).isFalse(); + }); + } + + @Test + void rejectsAResponseContainingProviderErrors() throws Exception { + ApiFootballLiveDto.ApiResponse body = response(""" + { + "errors": {"requests": "Invalid season"}, + "results": 0, + "response": [] + } + """); + + assertThatThrownBy(() -> + ApiFootballResponseValidator.validate("getFixtures", HttpStatus.OK, body)) + .isInstanceOfSatisfying(ExternalApiException.class, exception -> + assertThat(exception.getCategory()).isEqualTo(ExternalApiErrorCategory.INVALID_RESPONSE)); + } + + @Test + void rejectsAnInconsistentResultCount() throws Exception { + ApiFootballLiveDto.ApiResponse body = response(""" + {"errors": [], "results": 1, "response": []} + """); + + assertThatThrownBy(() -> + ApiFootballResponseValidator.validate("getFixtures", HttpStatus.OK, body)) + .isInstanceOf(ExternalApiException.class); + } + + @SuppressWarnings("unchecked") + private ApiFootballLiveDto.ApiResponse response(String json) throws Exception { + return objectMapper.readValue(json, ApiFootballLiveDto.ApiResponse.class); + } +} diff --git a/src/test/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballFixtureDetailSyncSchedulerTest.java b/src/test/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballFixtureDetailSyncSchedulerTest.java index d9c92a0..ee74d04 100644 --- a/src/test/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballFixtureDetailSyncSchedulerTest.java +++ b/src/test/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballFixtureDetailSyncSchedulerTest.java @@ -1,6 +1,7 @@ package com.son.soccerStreaming.apifootball.scheduler; import com.son.soccerStreaming.apifootball.service.ApiFootballFixtureDetailSyncService; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; import com.son.soccerStreaming.fixture.repository.FixtureRepository; import com.son.soccerStreaming.live.service.LiveFixtureBroadcastService; import org.junit.jupiter.api.Test; @@ -23,7 +24,8 @@ void skipsDetailSyncWhenThereAreNoLiveFixtures() { detailSyncService, broadcastService, fixtureRepository, - retryScheduler + retryScheduler, + new ApiFootballSyncExecutionGuard() ); when(fixtureRepository.findAllByFixtureStatus("LIVE")).thenReturn(List.of()); @@ -31,4 +33,25 @@ void skipsDetailSyncWhenThereAreNoLiveFixtures() { verifyNoInteractions(detailSyncService, broadcastService, retryScheduler); } + + @Test + void skipsDetailSyncWhenTheSameJobIsAlreadyReserved() { + ApiFootballFixtureDetailSyncService detailSyncService = mock(ApiFootballFixtureDetailSyncService.class); + LiveFixtureBroadcastService broadcastService = mock(LiveFixtureBroadcastService.class); + FixtureRepository fixtureRepository = mock(FixtureRepository.class); + ApiFootballSyncFailureRetryScheduler retryScheduler = mock(ApiFootballSyncFailureRetryScheduler.class); + ApiFootballSyncExecutionGuard guard = new ApiFootballSyncExecutionGuard(); + guard.acquire(ApiFootballSyncExecutionGuard.key("fixture-details-live", "live")); + ApiFootballFixtureDetailSyncScheduler scheduler = new ApiFootballFixtureDetailSyncScheduler( + detailSyncService, + broadcastService, + fixtureRepository, + retryScheduler, + guard + ); + + scheduler.syncLiveFixtureDetails(); + + verifyNoInteractions(detailSyncService, broadcastService, fixtureRepository, retryScheduler); + } } diff --git a/src/test/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballSyncFailureRetrySchedulerTest.java b/src/test/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballSyncFailureRetrySchedulerTest.java new file mode 100644 index 0000000..2efa008 --- /dev/null +++ b/src/test/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballSyncFailureRetrySchedulerTest.java @@ -0,0 +1,226 @@ +package com.son.soccerStreaming.apifootball.scheduler; + +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncStatusService; +import com.son.soccerStreaming.global.externalapi.ExternalApiErrorCategory; +import com.son.soccerStreaming.global.externalapi.ExternalApiException; +import com.son.soccerStreaming.global.externalapi.ExternalApiProvider; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.Duration; +import java.util.List; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class ApiFootballSyncFailureRetrySchedulerTest { + + private final ApiFootballSyncStatusService syncStatusService = mock(ApiFootballSyncStatusService.class); + private final ApiFootballSyncExecutionGuard executionGuard = new ApiFootballSyncExecutionGuard(); + private final ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + private final ApiFootballSyncFailureRetryScheduler scheduler = + new ApiFootballSyncFailureRetryScheduler(syncStatusService, executionGuard, executor); + + ApiFootballSyncFailureRetrySchedulerTest() { + ReflectionTestUtils.setField(scheduler, "enabled", true); + ReflectionTestUtils.setField(scheduler, "maxAttempts", 2); + ReflectionTestUtils.setField(scheduler, "initialDelayMinutes", 1L); + ReflectionTestUtils.setField(scheduler, "delayMultiplier", 5L); + ReflectionTestUtils.setField(scheduler, "maxDelayMinutes", 30L); + when(executor.schedule(any(Runnable.class), anyLong(), eq(TimeUnit.MILLISECONDS))) + .thenAnswer(invocation -> mock(ScheduledFuture.class)); + } + + @AfterEach + void tearDown() { + scheduler.shutdown(); + } + + @Test + void increasesFallbackDelayAndCapsItAtConfiguredMaximum() { + assertThat(scheduler.retryDelay(0, new RuntimeException())).isEqualTo(Duration.ofMinutes(1)); + assertThat(scheduler.retryDelay(1, new RuntimeException())).isEqualTo(Duration.ofMinutes(5)); + assertThat(scheduler.retryDelay(2, new RuntimeException())).isEqualTo(Duration.ofMinutes(25)); + assertThat(scheduler.retryDelay(3, new RuntimeException())).isEqualTo(Duration.ofMinutes(30)); + } + + @Test + void retryAfterOverridesFallbackDelayWithoutSlowRetryCap() { + ExternalApiException failure = retryableFailure(Duration.ofHours(2)); + + assertThat(scheduler.retryDelay(1, failure)).isEqualTo(Duration.ofHours(2)); + assertThat(scheduler.retryDelay(1, new RuntimeException(failure))).isEqualTo(Duration.ofHours(2)); + } + + @Test + void retryFailureUsesTheLatestRetryAfterForTheNextSchedule() { + ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(Runnable.class); + ArgumentCaptor delayCaptor = ArgumentCaptor.forClass(Long.class); + ExternalApiException latestFailure = retryableFailure(Duration.ofMinutes(7)); + + scheduler.schedule("fixtures:daily:39:2025", "fixtures:league=39; season=2025", + "fixture sync", new RuntimeException(), () -> { + throw latestFailure; + }); + verify(executor).schedule(taskCaptor.capture(), delayCaptor.capture(), eq(TimeUnit.MILLISECONDS)); + taskCaptor.getValue().run(); + + verify(executor, times(2)).schedule(any(Runnable.class), delayCaptor.capture(), eq(TimeUnit.MILLISECONDS)); + assertThat(delayCaptor.getAllValues()).contains(Duration.ofMinutes(7).toMillis()); + } + + @Test + void doesNotRegisterTheSameRetryKeyTwice() { + scheduler.schedule("teams:39:2025", "teams:league=39; season=2025", + "team sync", new RuntimeException(), () -> { }); + scheduler.schedule("teams:39:2025", "teams:league=39; season=2025", + "team sync", new RuntimeException(), () -> { }); + + assertThat(scheduler.pendingRetryCount()).isEqualTo(1); + verify(executor, times(1)).schedule(any(Runnable.class), anyLong(), eq(TimeUnit.MILLISECONDS)); + } + + @Test + void cancellingAnExecutionScopePreventsCapturedStaleTasksFromRunning() { + ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(Runnable.class); + Runnable firstAction = mock(Runnable.class); + Runnable secondAction = mock(Runnable.class); + Runnable unrelatedAction = mock(Runnable.class); + + scheduler.schedule("teams:39:2025", "teams:league=39; season=2025", + "scheduled teams", new RuntimeException(), firstAction); + scheduler.schedule("startup:teams:39:2025", "teams:league=39; season=2025", + "startup teams", new RuntimeException(), secondAction); + scheduler.schedule("teams:40:2025", "teams:league=40; season=2025", + "other teams", new RuntimeException(), unrelatedAction); + verify(executor, times(3)).schedule(taskCaptor.capture(), anyLong(), eq(TimeUnit.MILLISECONDS)); + + assertThat(scheduler.cancelPendingByExecutionKey("teams:league=39; season=2025")).isEqualTo(2); + taskCaptor.getAllValues().get(0).run(); + taskCaptor.getAllValues().get(1).run(); + + assertThat(scheduler.pendingRetryCount()).isEqualTo(1); + verify(firstAction, never()).run(); + verify(secondAction, never()).run(); + verify(unrelatedAction, never()).run(); + verify(executor, times(3)).schedule(any(Runnable.class), anyLong(), eq(TimeUnit.MILLISECONDS)); + } + + @Test + void partialRetrySuccessKeepsSiblingsAndMarksSuccessAfterTheLastOne() { + ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(Runnable.class); + String executionKey = "players:league=39; season=2025"; + + scheduler.schedule("registered-players:39:2025:team:1", executionKey, + "team 1", new RuntimeException(), () -> { }); + scheduler.schedule("registered-players:39:2025:team:2", executionKey, + "team 2", new RuntimeException(), () -> { }); + verify(executor, times(2)).schedule(taskCaptor.capture(), anyLong(), eq(TimeUnit.MILLISECONDS)); + + taskCaptor.getAllValues().get(0).run(); + assertThat(scheduler.pendingRetryCount()).isEqualTo(1); + verify(syncStatusService, never()).recordSuccessByKey(any(), any()); + + taskCaptor.getAllValues().get(1).run(); + assertThat(scheduler.pendingRetryCount()).isZero(); + verify(syncStatusService).recordSuccessByKey("players:2025", "Players 2025"); + } + + @Test + void successfulSiblingDoesNotHideATerminalPartialRetryFailure() { + ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(Runnable.class); + String executionKey = "players:league=39; season=2025"; + + scheduler.schedule("registered-players:39:2025:team:1", executionKey, + "team 1", new RuntimeException(), () -> { + throw nonRetryableFailure(); + }); + scheduler.schedule("registered-players:39:2025:team:2", executionKey, + "team 2", new RuntimeException(), () -> { }); + verify(executor, times(2)).schedule(taskCaptor.capture(), anyLong(), eq(TimeUnit.MILLISECONDS)); + + taskCaptor.getAllValues().forEach(Runnable::run); + + assertThat(scheduler.pendingRetryCount()).isZero(); + verify(syncStatusService, never()).recordSuccessByKey(any(), any()); + verify(syncStatusService).recordFailureByKey(eq("players:2025"), eq("Players 2025"), any()); + } + + @Test + void aNewAutomaticRetryBatchSupersedesTerminalFailuresFromThePreviousRun() { + ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(Runnable.class); + String executionKey = "players:league=39; season=2025"; + + scheduler.schedule("registered-players:39:2025:team:1", executionKey, + "previous team failure", new RuntimeException(), () -> { + throw nonRetryableFailure(); + }); + verify(executor).schedule(taskCaptor.capture(), anyLong(), eq(TimeUnit.MILLISECONDS)); + taskCaptor.getValue().run(); + + scheduler.schedule("registered-players:39:2025:team:2", executionKey, + "current team failure", new RuntimeException(), () -> { }); + verify(executor, times(2)).schedule(taskCaptor.capture(), anyLong(), eq(TimeUnit.MILLISECONDS)); + taskCaptor.getValue().run(); + + verify(syncStatusService).recordSuccessByKey("players:2025", "Players 2025"); + } + + @Test + void activeSynchronizationDefersWithoutRunningTheRetryAction() { + ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(Runnable.class); + String executionKey = "fixtures:league=39; season=2025"; + Runnable action = mock(Runnable.class); + ApiFootballSyncExecutionGuard.Lease lease = executionGuard.acquire(executionKey); + + scheduler.schedule("fixtures:daily:39:2025", executionKey, + "fixture sync", new RuntimeException(), action); + verify(executor).schedule(taskCaptor.capture(), anyLong(), eq(TimeUnit.MILLISECONDS)); + taskCaptor.getValue().run(); + + assertThat(scheduler.pendingRetryCount()).isEqualTo(1); + verify(action, never()).run(); + verify(executor, times(2)).schedule(any(Runnable.class), anyLong(), eq(TimeUnit.MILLISECONDS)); + verify(syncStatusService, times(1)).recordRetryPendingByKey(any(), any(), any()); + executionGuard.release(lease); + } + + private ExternalApiException retryableFailure(Duration retryAfter) { + return new ExternalApiException( + ExternalApiProvider.API_FOOTBALL, + "fixtures", + ExternalApiErrorCategory.RATE_LIMITED, + 429, + true, + retryAfter, + "rate limited", + null + ); + } + + private ExternalApiException nonRetryableFailure() { + return new ExternalApiException( + ExternalApiProvider.API_FOOTBALL, + "players", + ExternalApiErrorCategory.BAD_REQUEST, + 400, + false, + null, + "bad request", + null + ); + } +} diff --git a/src/test/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballTeamSyncSchedulerTest.java b/src/test/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballTeamSyncSchedulerTest.java new file mode 100644 index 0000000..fe935ff --- /dev/null +++ b/src/test/java/com/son/soccerStreaming/apifootball/scheduler/ApiFootballTeamSyncSchedulerTest.java @@ -0,0 +1,58 @@ +package com.son.soccerStreaming.apifootball.scheduler; + +import com.son.soccerStreaming.apifootball.service.ApiFootballSyncExecutionGuard; +import com.son.soccerStreaming.apifootball.service.ApiFootballTeamSyncService; +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class ApiFootballTeamSyncSchedulerTest { + + @Test + void successfulAutomaticSyncCancelsPendingRetriesForTheSameScope() { + ApiFootballTeamSyncService syncService = mock(ApiFootballTeamSyncService.class); + ApiFootballSyncFailureRetryScheduler retryScheduler = mock(ApiFootballSyncFailureRetryScheduler.class); + ApiFootballTeamSyncScheduler scheduler = scheduler(syncService, retryScheduler); + + scheduler.syncTeams(); + + verify(retryScheduler).cancelPendingByExecutionKey("teams:league=39; season=2025"); + verify(retryScheduler, never()).schedule(any(), any(), any(), any(), any()); + } + + @Test + void failedAutomaticSyncSchedulesSlowRetryWithTheOriginalFailure() { + ApiFootballTeamSyncService syncService = mock(ApiFootballTeamSyncService.class); + ApiFootballSyncFailureRetryScheduler retryScheduler = mock(ApiFootballSyncFailureRetryScheduler.class); + ApiFootballTeamSyncScheduler scheduler = scheduler(syncService, retryScheduler); + RuntimeException failure = new RuntimeException("upstream failed"); + when(syncService.syncTeams(39, 2025)).thenThrow(failure); + when(retryScheduler.shouldRetry(failure)).thenReturn(true); + + scheduler.syncTeams(); + + verify(retryScheduler).schedule( + eq("teams:39:2025"), + eq("teams:league=39; season=2025"), + eq("team sync league=39 season=2025"), + eq(failure), + any(Runnable.class) + ); + verify(retryScheduler, never()).cancelPendingByExecutionKey(any()); + } + + private ApiFootballTeamSyncScheduler scheduler(ApiFootballTeamSyncService syncService, + ApiFootballSyncFailureRetryScheduler retryScheduler) { + ApiFootballTeamSyncScheduler scheduler = new ApiFootballTeamSyncScheduler( + syncService, retryScheduler, new ApiFootballSyncExecutionGuard()); + ReflectionTestUtils.setField(scheduler, "league", 39); + ReflectionTestUtils.setField(scheduler, "season", 2025); + return scheduler; + } +} diff --git a/src/test/java/com/son/soccerStreaming/apifootball/service/ApiFootballSyncExecutionGuardTest.java b/src/test/java/com/son/soccerStreaming/apifootball/service/ApiFootballSyncExecutionGuardTest.java new file mode 100644 index 0000000..866345c --- /dev/null +++ b/src/test/java/com/son/soccerStreaming/apifootball/service/ApiFootballSyncExecutionGuardTest.java @@ -0,0 +1,49 @@ +package com.son.soccerStreaming.apifootball.service; + +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ApiFootballSyncExecutionGuardTest { + + private final ApiFootballSyncExecutionGuard guard = new ApiFootballSyncExecutionGuard(); + + @Test + void rejectsTheSameKeyUntilItIsReleased() { + ApiFootballSyncExecutionGuard.Lease lease = guard.acquire("players:league=39; season=2025"); + + assertThatThrownBy(() -> guard.acquire("players:league=39; season=2025")) + .isInstanceOf(ApiFootballSyncAlreadyRunningException.class); + + guard.release(lease); + guard.acquire("players:league=39; season=2025"); + } + + @Test + void anOldLeaseCannotReleaseANewerReservation() { + ApiFootballSyncExecutionGuard.Lease oldLease = guard.acquire("fixtures:2025"); + guard.release(oldLease); + guard.acquire("fixtures:2025"); + + guard.release(oldLease); + + assertThatThrownBy(() -> guard.acquire("fixtures:2025")) + .isInstanceOf(ApiFootballSyncAlreadyRunningException.class); + } + + @Test + void releasesTheKeyAfterScheduledWorkFails() { + AtomicInteger calls = new AtomicInteger(); + + assertThatThrownBy(() -> guard.executeIfAvailable("injuries:2025", () -> { + calls.incrementAndGet(); + throw new IllegalStateException("failed"); + })).isInstanceOf(IllegalStateException.class); + + assertThat(guard.executeIfAvailable("injuries:2025", calls::incrementAndGet)).isTrue(); + assertThat(calls).hasValue(2); + } +}