From cc93552aeef883bfc0d6c69e25f1f5a508d27cda Mon Sep 17 00:00:00 2001 From: Sunny Aggarwal Date: Mon, 26 Oct 2026 17:26:25 +0530 Subject: [PATCH] OpenConceptLab/ocl_issues#2830 | expire Auto Match runs left running for over 24 hours --- core/celery.py | 4 +++ core/common/tasks.py | 6 +++++ core/map_projects/models.py | 13 +++++++++ core/map_projects/tests/tests.py | 46 +++++++++++++++++++++++++++++++- 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/core/celery.py b/core/celery.py index 8495b8f4c..26b34e95c 100644 --- a/core/celery.py +++ b/core/celery.py @@ -33,6 +33,10 @@ 'task': 'core.common.tasks.expire_old_celery_tasks', 'schedule': crontab(0, 1), # Run at 1 am }, + 'expire-stale-auto-match-runs': { + 'task': 'core.common.tasks.expire_stale_auto_match_runs', + 'schedule': timedelta(hours=1), + }, 'rerun-indexing-job': { 'task': 'core.common.tasks.rerun_indexing_job', 'schedule': timedelta(minutes=15), diff --git a/core/common/tasks.py b/core/common/tasks.py index 558fac965..c33510288 100644 --- a/core/common/tasks.py +++ b/core/common/tasks.py @@ -1192,6 +1192,12 @@ def expire_old_celery_tasks(): Task.objects.filter(updated_at__lt=timezone.now() - timezone.timedelta(days=7)).delete() +@app.task(ignore_result=True) +def expire_stale_auto_match_runs(): + from core.map_projects.models import AutomatchRun + AutomatchRun.expire_stale() + + def get_live_indexing_task_ids(): """ Task ids currently held by the indexing workers -- executing, prefetched or scheduled. diff --git a/core/map_projects/models.py b/core/map_projects/models.py index f5aad709a..84ad34eb5 100644 --- a/core/map_projects/models.py +++ b/core/map_projects/models.py @@ -6,6 +6,7 @@ from django.core.exceptions import ValidationError from django.db import models, IntegrityError from django.db.models import Sum +from django.utils import timezone from pydash import get from core.common.constants import PERSIST_NEW_ERROR_MESSAGE, ACCESS_TYPE_CHOICES, ACCESS_TYPE_NONE @@ -347,3 +348,15 @@ class Meta: def calculate_uri(self): return f"/auto-match-runs/{self.id or generate_temp_version()}/" + + # A run the client never closed, e.g. its tab closed before the create + # response arrived. The Mapper PATCHes a run only when it ends, so this must + # outlast the longest run. + STALE_AFTER = timezone.timedelta(hours=24) + + @classmethod + def expire_stale(cls, older_than=None): + now = timezone.now() + return cls.objects.filter( + completion_status=cls.RUNNING, updated_at__lt=now - (older_than or cls.STALE_AFTER) + ).update(completion_status=cls.CANCELLED, completed_at=now, updated_at=now) diff --git a/core/map_projects/tests/tests.py b/core/map_projects/tests/tests.py index be34e7d92..a85ebb955 100644 --- a/core/map_projects/tests/tests.py +++ b/core/map_projects/tests/tests.py @@ -3,12 +3,13 @@ from django.contrib.auth.models import Group from django.core.files.uploadedfile import SimpleUploadedFile from django.db import IntegrityError +from django.utils import timezone from mock import patch, ANY, Mock from rest_framework.test import APIRequestFactory from core.common.constants import PERSIST_NEW_ERROR_MESSAGE from core.common.tests import OCLAPITestCase, OCLTestCase, PREVIEW_GROUP_NAME -from core.map_projects.models import MapProject +from core.map_projects.models import MapProject, AutomatchRun from core.map_projects.views import AutomatchRunListView from core.map_projects.tests.factories import MapProjectFactory, AutomatchRunFactory from core.orgs.tests.factories import OrganizationFactory @@ -797,3 +798,46 @@ def test_get_404_for_missing_run(self): response = self.client.get( '/auto-match-runs/99999999/', HTTP_AUTHORIZATION='Token ' + self.user.get_token()) self.assertEqual(response.status_code, 404) + + +class AutomatchRunExpireStaleTest(OCLTestCase): + @staticmethod + def make_run(age, **kwargs): + run = AutomatchRunFactory(**kwargs) + AutomatchRun.objects.filter(id=run.id).update(updated_at=timezone.now() - age) + return run + + def test_expire_stale_cancels_only_old_running_runs(self): + stale = self.make_run(timezone.timedelta(hours=25)) + recent = self.make_run(timezone.timedelta(hours=1)) + finished = self.make_run(timezone.timedelta(hours=25), completion_status=AutomatchRun.COMPLETED) + + self.assertEqual(AutomatchRun.expire_stale(), 1) + + stale.refresh_from_db() + recent.refresh_from_db() + finished.refresh_from_db() + self.assertEqual(stale.completion_status, AutomatchRun.CANCELLED) + self.assertIsNotNone(stale.completed_at) + self.assertGreater(stale.updated_at, timezone.now() - timezone.timedelta(minutes=1)) + self.assertEqual(recent.completion_status, AutomatchRun.RUNNING) + self.assertIsNone(recent.completed_at) + self.assertEqual(finished.completion_status, AutomatchRun.COMPLETED) + self.assertIsNone(finished.completed_at) + + def test_expire_stale_with_custom_age(self): + run = self.make_run(timezone.timedelta(hours=2)) + + self.assertEqual(AutomatchRun.expire_stale(older_than=timezone.timedelta(hours=1)), 1) + + run.refresh_from_db() + self.assertEqual(run.completion_status, AutomatchRun.CANCELLED) + + def test_task(self): + from core.common.tasks import expire_stale_auto_match_runs + run = self.make_run(timezone.timedelta(days=2)) + + expire_stale_auto_match_runs() + + run.refresh_from_db() + self.assertEqual(run.completion_status, AutomatchRun.CANCELLED)