OpenConceptLab/ocl_issues#2830 | expire Auto Match runs left running for over 24 hours

core/celery.py

@@ -33,6 +33,10 @@
         'task': 'core.common.tasks.expire_old_celery_tasks',
         'schedule': crontab(0, 1),  # Run at 1 am
     },
+    'expire-stale-auto-match-runs': {
+        'task': 'core.common.tasks.expire_stale_auto_match_runs',
+        'schedule': timedelta(hours=1),
+    },
     'rerun-indexing-job': {
         'task': 'core.common.tasks.rerun_indexing_job',
         'schedule': timedelta(minutes=15),

core/common/tasks.py

@@ -1192,6 +1192,12 @@ def expire_old_celery_tasks():
     Task.objects.filter(updated_at__lt=timezone.now() - timezone.timedelta(days=7)).delete()
 
 
+@app.task(ignore_result=True)
+def expire_stale_auto_match_runs():
+    from core.map_projects.models import AutomatchRun
+    AutomatchRun.expire_stale()
+
+
 def get_live_indexing_task_ids():
     """
     Task ids currently held by the indexing workers -- executing, prefetched or scheduled.

core/map_projects/models.py

@@ -6,6 +6,7 @@
 from django.core.exceptions import ValidationError
 from django.db import models, IntegrityError
 from django.db.models import Sum
+from django.utils import timezone
 from pydash import get
 
 from core.common.constants import PERSIST_NEW_ERROR_MESSAGE, ACCESS_TYPE_CHOICES, ACCESS_TYPE_NONE
@@ -347,3 +348,15 @@ class Meta:
 
     def calculate_uri(self):
         return f"/auto-match-runs/{self.id or generate_temp_version()}/"
+
+    # A run the client never closed, e.g. its tab closed before the create
+    # response arrived. The Mapper PATCHes a run only when it ends, so this must
+    # outlast the longest run.
+    STALE_AFTER = timezone.timedelta(hours=24)
+
+    @classmethod
+    def expire_stale(cls, older_than=None):
+        now = timezone.now()
+        return cls.objects.filter(
+            completion_status=cls.RUNNING, updated_at__lt=now - (older_than or cls.STALE_AFTER)
+        ).update(completion_status=cls.CANCELLED, completed_at=now, updated_at=now)

core/map_projects/tests/tests.py

@@ -3,12 +3,13 @@
 from django.contrib.auth.models import Group
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db import IntegrityError
+from django.utils import timezone
 from mock import patch, ANY, Mock
 from rest_framework.test import APIRequestFactory
 
 from core.common.constants import PERSIST_NEW_ERROR_MESSAGE
 from core.common.tests import OCLAPITestCase, OCLTestCase, PREVIEW_GROUP_NAME
-from core.map_projects.models import MapProject
+from core.map_projects.models import MapProject, AutomatchRun
 from core.map_projects.views import AutomatchRunListView
 from core.map_projects.tests.factories import MapProjectFactory, AutomatchRunFactory
 from core.orgs.tests.factories import OrganizationFactory
@@ -797,3 +798,46 @@ def test_get_404_for_missing_run(self):
         response = self.client.get(
             '/auto-match-runs/99999999/', HTTP_AUTHORIZATION='Token ' + self.user.get_token())
         self.assertEqual(response.status_code, 404)
+
+
+class AutomatchRunExpireStaleTest(OCLTestCase):
+    @staticmethod
+    def make_run(age, **kwargs):
+        run = AutomatchRunFactory(**kwargs)
+        AutomatchRun.objects.filter(id=run.id).update(updated_at=timezone.now() - age)
+        return run
+
+    def test_expire_stale_cancels_only_old_running_runs(self):
+        stale = self.make_run(timezone.timedelta(hours=25))
+        recent = self.make_run(timezone.timedelta(hours=1))
+        finished = self.make_run(timezone.timedelta(hours=25), completion_status=AutomatchRun.COMPLETED)
+
+        self.assertEqual(AutomatchRun.expire_stale(), 1)
+
+        stale.refresh_from_db()
+        recent.refresh_from_db()
+        finished.refresh_from_db()
+        self.assertEqual(stale.completion_status, AutomatchRun.CANCELLED)
+        self.assertIsNotNone(stale.completed_at)
+        self.assertGreater(stale.updated_at, timezone.now() - timezone.timedelta(minutes=1))
+        self.assertEqual(recent.completion_status, AutomatchRun.RUNNING)
+        self.assertIsNone(recent.completed_at)
+        self.assertEqual(finished.completion_status, AutomatchRun.COMPLETED)
+        self.assertIsNone(finished.completed_at)
+
+    def test_expire_stale_with_custom_age(self):
+        run = self.make_run(timezone.timedelta(hours=2))
+
+        self.assertEqual(AutomatchRun.expire_stale(older_than=timezone.timedelta(hours=1)), 1)
+
+        run.refresh_from_db()
+        self.assertEqual(run.completion_status, AutomatchRun.CANCELLED)
+
+    def test_task(self):
+        from core.common.tasks import expire_stale_auto_match_runs
+        run = self.make_run(timezone.timedelta(days=2))
+
+        expire_stale_auto_match_runs()
+
+        run.refresh_from_db()
+        self.assertEqual(run.completion_status, AutomatchRun.CANCELLED)