From 4e44f9985209f7fb3d26132038cb565a8105d1da Mon Sep 17 00:00:00 2001 From: Harshil-Malisetty Date: Sat, 21 Mar 2026 14:03:19 +0530 Subject: [PATCH 1/2] feat: derive killed status for externally terminated tasks --- .../ui_backend_service/data/refiner/task_refiner.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/services/ui_backend_service/data/refiner/task_refiner.py b/services/ui_backend_service/data/refiner/task_refiner.py index c5732c538..2ce16d0ef 100644 --- a/services/ui_backend_service/data/refiner/task_refiner.py +++ b/services/ui_backend_service/data/refiner/task_refiner.py @@ -4,9 +4,7 @@ class TaskRefiner(Refinery): """ Refiner class for postprocessing Task rows. - Uses Metaflow Client API to refine Task's actual status from Metaflow Service and Datastore. - Parameters ----------- cache : AsyncCacheClient @@ -36,11 +34,12 @@ async def refine_record(self, record, values): record['status'] = 'failed' elif value is True: record['status'] = 'completed' - + # If task is failed and _task_ok key exists in values but is None, + # the process was killed externally (SIGKILL bypasses finally in task.py) + elif record['status'] == 'failed' and '_task_ok' in values and values['_task_ok'] is None: + record['status'] = 'killed' if values.get('_foreach_stack'): value = values['_foreach_stack'] if len(value) > 0 and len(value[0]) >= 4: - # The third one in the tuple is the foreach index. We access this way for backwards compatibility. record['foreach_label'] = "{}[{}]".format(record['task_id'], value[0][3]) - - return record + return record \ No newline at end of file From 418da8edd3df7266e20a0a1911d89bf31ccaa854 Mon Sep 17 00:00:00 2001 From: Harshil-Malisetty Date: Sat, 21 Mar 2026 15:55:22 +0530 Subject: [PATCH 2/2] feat: derive killed status for externally terminated tasks in SQL --- services/ui_backend_service/data/db/tables/task.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/ui_backend_service/data/db/tables/task.py b/services/ui_backend_service/data/db/tables/task.py index 996a177e9..26af66f95 100644 --- a/services/ui_backend_service/data/db/tables/task.py +++ b/services/ui_backend_service/data/db/tables/task.py @@ -174,6 +174,12 @@ def select_columns(self): THEN 'completed' WHEN attempt.attempt_ok IS FALSE THEN 'failed' + WHEN attempt.attempt_ok IS NULL + AND attempt.task_ok_location IS NULL + AND {table_name}.last_heartbeat_ts IS NOT NULL + AND @(extract(epoch from now())-{table_name}.last_heartbeat_ts)>{heartbeat_threshold} + AND {finished_at_column} IS NULL + THEN 'killed' WHEN COALESCE(attempt.attempt_finished_at, attempt.task_ok_finished_at) IS NOT NULL AND attempt_ok IS NULL THEN 'unknown'