From 1fa910806ad6660f3ab3d7e7ada9b3816c185a62 Mon Sep 17 00:00:00 2001 From: Brian Geuther Date: Mon, 28 Jul 2025 14:08:45 -0400 Subject: [PATCH 1/2] Adding in ignore condition for unsuccessful tasks --- nextflow/configs/profiles/sumner2.config | 40 ++++++++++++------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config index 348af65..f733415 100644 --- a/nextflow/configs/profiles/sumner2.config +++ b/nextflow/configs/profiles/sumner2.config @@ -231,7 +231,7 @@ process { // 0.5 * t_min + 10.5 time = { ((0.5 * params.clip_duration / 30 / 60 + 10.5) * 1.5).toInteger() + '.sec' * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_fboli_predict" { @@ -240,7 +240,7 @@ process { // 2.5 * t_min + 5 time = { ((2.5 * params.clip_duration / 30 / 60 + 5) * 1.5).toInteger() + '.sec' * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } // Frailty Resources @@ -251,7 +251,7 @@ process { // 0.6 * t_min + 30 time = { ((0.6 * params.clip_duration / 30 / 60 + 30) * 1.5).toInteger() + '.sec' * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_rearpaw" { @@ -260,7 +260,7 @@ process { memory = { ((0.0047 * params.clip_duration / 30 / 60 + 0.0801) * 1.5).toInteger() + '.GB' * task.attempt } time = { 10.min * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } // Gait Resources @@ -270,7 +270,7 @@ process { memory = { ((0.0009 * params.clip_duration / 30 / 60 + 0.727) * 1.5).toInteger() + '.GB' * task.attempt } time = { 10.min * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_gait_bin" { @@ -279,7 +279,7 @@ process { memory = { ((0.000005 * params.clip_duration / 30 / 60 + 0.0095) * 1.5).toInteger() + '.GB' * task.attempt } time = { 5.min * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } // JABS Resources @@ -293,7 +293,7 @@ process { // 162 * t_min + 4.6 for 6 windows time = { ((162 * params.clip_duration / 30 / 60 + 4.6) * 2.5).toInteger() + '.sec' * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_jabs_classify" { @@ -305,7 +305,7 @@ process { // 30 * t_min + 127 for 10 classifiers time = { ((30 * params.clip_duration / 30 / 60 + 127) * 2.5).toInteger() + '.sec' * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_jabs_tablegen" { @@ -313,7 +313,7 @@ process { memory = { 1.GB * task.attempt } time = { 5.min * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_jabs_heuristic" { @@ -324,7 +324,7 @@ process { // 0.111 * t_min + 37 for 6 classifiers time = { ((0.111 * params.clip_duration / 30 / 60 + 37) * 1.5).toInteger() + '.sec' * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_jabs_table_convert" { @@ -334,7 +334,7 @@ process { // High value in experiments was 1-minute for a 1-hr video time = { 20.min * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } // Single Mouse Resources @@ -348,7 +348,7 @@ process { // 15 * t_min - 63 time = { ((15 * params.clip_duration / 30 / 60 + 0) * 2.5).toInteger() + '.sec' * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_single_keypoints" { @@ -359,7 +359,7 @@ process { memory = { 64.GB * task.attempt } time = { 6.hours * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_single_qc" { @@ -367,7 +367,7 @@ process { cpus = 1 memory = { 4.GB * task.attempt } time = { 20.min * task.attempt } - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_clip_video" { @@ -377,7 +377,7 @@ process { // 19.8 * t_min + 52 time = { ((19.8 * params.clip_duration / 30 / 60 + 52) * 2.5).toInteger() + '.sec' * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } // Static Object Resources @@ -387,7 +387,7 @@ process { memory = { 4.GB * task.attempt } time = { 10.min * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_food_hopper" { @@ -395,7 +395,7 @@ process { memory = { 4.GB * task.attempt } time = { 10.min * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_lixit" { @@ -403,7 +403,7 @@ process { memory = { 4.GB * task.attempt } time = { 10.min * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } // Utility Resources @@ -412,14 +412,14 @@ process { memory = { 1.GB * task.attempt } time = { 5.min * task.attempt } array = 500 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_gen_vid" { cpus = 1 memory = { 1.GB * task.attempt } time = { 1.hours * task.attempt } - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } } From 8b2cde90996e4ca00ef3b819a323ee69e028d765 Mon Sep 17 00:00:00 2001 From: Brian Geuther Date: Thu, 4 Sep 2025 15:22:21 -0400 Subject: [PATCH 2/2] Ignoring failures in multimouse too Enabling retry and array on tracklet resources --- nextflow/configs/profiles/sumner2.config | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config index 9f6d47b..7c35754 100644 --- a/nextflow/configs/profiles/sumner2.config +++ b/nextflow/configs/profiles/sumner2.config @@ -456,7 +456,7 @@ process { } time = { 5.hours * task.attempt } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_multi_keypoints" { @@ -472,7 +472,7 @@ process { return r_value + '.hours' } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_multi_identity" { @@ -484,7 +484,7 @@ process { return r_value + '.hours' } array = 200 - errorStrategy = 'retry' + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } maxRetries = 3 } withLabel: "r_multi_tracklets" { @@ -493,6 +493,9 @@ process { // First video suggested ~16GB and ~30min for 3 mice memory = { 16.GB * task.attempt } time = { 45.min * task.attempt } + array = 200 + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } + maxRetries = 3 } // Static Object Resources // These do not scale with clip duration