From 4688a3e48dfbbd54d9ceab360f32165f346f234f Mon Sep 17 00:00:00 2001
From: Brian Geuther <Brian.Geuther@jax.org>
Date: Tue, 5 Aug 2025 14:48:35 -0400
Subject: [PATCH 1/8] Adding resource estimations for multimouse tasks

---
 nextflow/configs/profiles/sumner2.config | 40 +++++++++++++++++++++++-
 nextflow/modules/multi_mouse.nf          |  7 +++--
 2 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config
index b9505f2..79fac91 100644
--- a/nextflow/configs/profiles/sumner2.config
+++ b/nextflow/configs/profiles/sumner2.config
@@ -144,7 +144,15 @@ process {
         // gpu_training: 14day limit, 16gpu/user
         clusterOptions = '-q gpu_inference --gres gpu:1 --nice=64000000'
         containerOptions = "--nv"
-        resourceLimits = [ cpus: 6, memory: 63.GB, time: 6.h ]
+        // Fair sharing of memory should be 63GB, but full node has 2TB
+        resourceLimits = [ cpus: 6, memory: 128.GB, time: 6.h ]
+    }
+    withLabel: "gpu_long" {
+        queue = "gpu_a100_mig"
+        clusterOptions = '-q gpu_training --gres gpu:1 --nice=64000000'
+        containerOptions = "--nv"
+        // Fair sharing of memory should be 63GB, but full node has 2TB
+        resourceLimits = [ cpus: 6, memory: 128.GB, time: 14.d ]
     }
     // Alternative GPU profiles available on the cluster
     withLabel: "gpu_a100" {
@@ -389,6 +397,36 @@ process {
         errorStrategy = 'retry'
         maxRetries = 3
     }
+    // Multi Mouse Resources
+    // These processes assume the input video is 1hr and scale with number of animals
+    withLabel: "r_multi_seg" {
+        cpus = 2
+        // 8.53 GB * num_animals
+        memory = { (8.53 * task.num_animals * 1.5).toInteger() + '.GB' * task.attempt }
+        time = { 5.hr * task.attempt }
+        array = 200
+        errorStrategy = 'retry'
+        maxRetries = 3
+    }
+    withLabel: "r_multi_keypoints" {
+        cpus = 2
+        // 2.26 GB * num_animals + 2.47 GB
+        memory = { ((2.26 * task.num_animals + 2.47) * 1.5).toInteger() + '.GB' * task.attempt }
+        // 6 * num_animals + 3
+        time = { ((6 * task.num_animals + 3) * 1.5).toInteger() + '.hr' * task.attempt }
+        array = 200
+        errorStrategy = 'retry'
+        maxRetries = 3
+    }
+    withLabel: "r_multi_identity" {
+        cpus = 2
+        memory = { 2.GB * task.attempt }
+        // 0.1 * num_animals + 0.16
+        time = { ((0.1 * task.num_animals + 0.16) * 1.5).toInteger() + '.hr' * task.attempt }
+        array = 200
+        errorStrategy = 'retry'
+        maxRetries = 3
+    }
     // Static Object Resources
     // These do not scale with clip duration
     withLabel: "r_arena_corners" {
diff --git a/nextflow/modules/multi_mouse.nf b/nextflow/modules/multi_mouse.nf
index f8c155b..947df8d 100644
--- a/nextflow/modules/multi_mouse.nf
+++ b/nextflow/modules/multi_mouse.nf
@@ -1,6 +1,7 @@
 process PREDICT_MULTI_MOUSE_SEGMENTATION {
-    label "gpu"
+    label "gpu_long"
     label "tracking"
+    label "r_multi_seg"
     
     input:
     tuple path(video_file), path(in_pose)
@@ -16,8 +17,9 @@ process PREDICT_MULTI_MOUSE_SEGMENTATION {
 }
 
 process PREDICT_MULTI_MOUSE_KEYPOINTS {
-    label "gpu"
+    label "gpu_long"
     label "tracking"
+    label "r_multi_keypoints"
     
     input:
     tuple path(video_file), path(in_pose)
@@ -35,6 +37,7 @@ process PREDICT_MULTI_MOUSE_KEYPOINTS {
 process PREDICT_MULTI_MOUSE_IDENTITY {
     label "gpu"
     label "tracking"
+    label "r_multi_identity"
     
     input:
     tuple path(video_file), path(in_pose)

From ff2ab3cc1eacf0e2ab5c8ebc10fc175080e8537e Mon Sep 17 00:00:00 2001
From: Brian Geuther <Brian.Geuther@jax.org>
Date: Wed, 27 Aug 2025 14:53:29 -0400
Subject: [PATCH 2/8] Adjusting containers to benchmark tracklet generation

---
 nextflow/configs/profiles/sumner2.config | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config
index 0f09418..022fab9 100644
--- a/nextflow/configs/profiles/sumner2.config
+++ b/nextflow/configs/profiles/sumner2.config
@@ -191,7 +191,7 @@ process {
      * Runtime options
      */
     withLabel: "tracking" {
-        container = "/projects/kumar-lab/multimouse-pipeline/deployment-runtime_2025-03-05.sif"
+        container = "/projects/kumar-lab/multimouse-pipeline/deployment-runtime_2025-08-26.sif"
     }
     withLabel: "jabs_classify" {
         container = "/projects/kumar-lab/multimouse-pipeline/nextflow-containers/JABS-GUI_2025-02-12_v0.18.1.sif"
@@ -216,7 +216,7 @@ process {
     }
     withLabel: "rclone" {
         // executor.queueSize = 1
-        container = "/projects/kumar-lab/multimouse-pipeline/rclone.sif"
+        container = "/projects/kumar-lab/multimouse-pipeline/rclone_2025-08-18.sif"
     }
 
     /* 

From 6d88e57c6b5ed2a3f2017b71b81917bf9d5214de Mon Sep 17 00:00:00 2001
From: Brian Geuther <Brian.Geuther@jax.org>
Date: Wed, 27 Aug 2025 15:18:29 -0400
Subject: [PATCH 3/8] Dynamic resources referencing wrong var

---
 nextflow/configs/profiles/sumner2.config | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config
index 022fab9..a30e78b 100644
--- a/nextflow/configs/profiles/sumner2.config
+++ b/nextflow/configs/profiles/sumner2.config
@@ -407,8 +407,8 @@ process {
     // These processes assume the input video is 1hr and scale with number of animals
     withLabel: "r_multi_seg" {
         cpus = 2
-        // 8.53 GB * num_animals
-        memory = { (8.53 * task.num_animals * 1.5).toInteger() + '.GB' * task.attempt }
+        // 8.53 GB * num_mice
+        memory = { (8.53 * params.num_mice * 1.5).toInteger() + '.GB' * task.attempt }
         time = { 5.hr * task.attempt }
         array = 200
         errorStrategy = 'retry'
@@ -416,10 +416,10 @@ process {
     }
     withLabel: "r_multi_keypoints" {
         cpus = 2
-        // 2.26 GB * num_animals + 2.47 GB
-        memory = { ((2.26 * task.num_animals + 2.47) * 1.5).toInteger() + '.GB' * task.attempt }
-        // 6 * num_animals + 3
-        time = { ((6 * task.num_animals + 3) * 1.5).toInteger() + '.hr' * task.attempt }
+        // 2.26 GB * num_mice + 2.47 GB
+        memory = { ((2.26 * params.num_mice + 2.47) * 1.5).toInteger() + '.GB' * task.attempt }
+        // 6 * num_mice + 3
+        time = { ((6 * params.num_mice + 3) * 1.5).toInteger() + '.hr' * task.attempt }
         array = 200
         errorStrategy = 'retry'
         maxRetries = 3
@@ -427,8 +427,8 @@ process {
     withLabel: "r_multi_identity" {
         cpus = 2
         memory = { 2.GB * task.attempt }
-        // 0.1 * num_animals + 0.16
-        time = { ((0.1 * task.num_animals + 0.16) * 1.5).toInteger() + '.hr' * task.attempt }
+        // 0.1 * num_mice + 0.16
+        time = { ((0.1 * params.num_mice + 0.16) * 1.5).toInteger() + '.hr' * task.attempt }
         array = 200
         errorStrategy = 'retry'
         maxRetries = 3

From 05db4c0b8d665e84efe0ab884ddf85a7b6295986 Mon Sep 17 00:00:00 2001
From: Brian Geuther <Brian.Geuther@jax.org>
Date: Wed, 27 Aug 2025 15:30:06 -0400
Subject: [PATCH 4/8] .hr -> .hours

---
 nextflow/configs/profiles/sumner2.config | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config
index a30e78b..3f2a3ca 100644
--- a/nextflow/configs/profiles/sumner2.config
+++ b/nextflow/configs/profiles/sumner2.config
@@ -409,7 +409,7 @@ process {
         cpus = 2
         // 8.53 GB * num_mice
         memory = { (8.53 * params.num_mice * 1.5).toInteger() + '.GB' * task.attempt }
-        time = { 5.hr * task.attempt }
+        time = { 5.hours * task.attempt }
         array = 200
         errorStrategy = 'retry'
         maxRetries = 3
@@ -419,7 +419,7 @@ process {
         // 2.26 GB * num_mice + 2.47 GB
         memory = { ((2.26 * params.num_mice + 2.47) * 1.5).toInteger() + '.GB' * task.attempt }
         // 6 * num_mice + 3
-        time = { ((6 * params.num_mice + 3) * 1.5).toInteger() + '.hr' * task.attempt }
+        time = { ((6 * params.num_mice + 3) * 1.5).toInteger() + '.hours' * task.attempt }
         array = 200
         errorStrategy = 'retry'
         maxRetries = 3
@@ -428,7 +428,7 @@ process {
         cpus = 2
         memory = { 2.GB * task.attempt }
         // 0.1 * num_mice + 0.16
-        time = { ((0.1 * params.num_mice + 0.16) * 1.5).toInteger() + '.hr' * task.attempt }
+        time = { ((0.1 * params.num_mice + 0.16) * 1.5).toInteger() + '.hours' * task.attempt }
         array = 200
         errorStrategy = 'retry'
         maxRetries = 3

From cec2feb942842a487c68f8a08e3582e2fe85faea Mon Sep 17 00:00:00 2001
From: Brian Geuther <Brian.Geuther@jax.org>
Date: Tue, 2 Sep 2025 10:47:25 -0400
Subject: [PATCH 5/8] Making dynamic resource equations simpler to read.

---
 nextflow/configs/profiles/sumner2.config | 229 ++++++++++++++++++++---
 nextflow/modules/multi_mouse.nf          |   9 +-
 2 files changed, 211 insertions(+), 27 deletions(-)

diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config
index 3f2a3ca..dd9ee15 100644
--- a/nextflow/configs/profiles/sumner2.config
+++ b/nextflow/configs/profiles/sumner2.config
@@ -2,6 +2,23 @@
 
 workDir = "/flashscratch/${USER}/nextflow-work"
 
+/**
+ * Get dynamic resource allocation based on parameters
+ *
+ * @param minimum Minimum resource allocation
+ * @param m Slope of the dynamic resource
+ * @param x Var that scales resources
+ * @param b Constant added to the request
+ * @param safety Safety margin for the request
+ * @param attempt Current attempt number (1-based)
+ *
+ * @return integer of the resources requested
+ */
+def get_dynamic_resource(float minimum, float m, float x, float b, float safety=1.5, int attempt=1) {
+    def base = Math.max(minimum, (b + m * x) * safety).toInteger()
+    return base * attempt
+}
+
 params {
     /*
      * Additional Parameters for the sumner2 profile
@@ -191,7 +208,7 @@ process {
      * Runtime options
      */
     withLabel: "tracking" {
-        container = "/projects/kumar-lab/multimouse-pipeline/deployment-runtime_2025-08-26.sif"
+        container = "/projects/kumar-lab/multimouse-pipeline/nextflow-containers/deployment-runtime_2025-08-26.sif"
     }
     withLabel: "jabs_classify" {
         container = "/projects/kumar-lab/multimouse-pipeline/nextflow-containers/JABS-GUI_2025-02-12_v0.18.1.sif"
@@ -246,7 +263,15 @@ process {
         cpus = 1
         memory = { 1.GB * task.attempt }
         // 0.5 * t_min + 10.5
-        time = { ((0.5 * params.clip_duration / 30 / 60 + 10.5) * 1.5).toInteger() + '.sec' * task.attempt }
+        time = {
+            get_dynamic_resource(
+                min: 60,
+                m: 0.5,
+                x: params.clip_duration / 30 / 60,
+                b: 10.5,
+                attempt: task.attempt
+            ) + '.sec'
+        }
         array = 200
         errorStrategy = 'retry'
         maxRetries = 3
@@ -255,7 +280,15 @@ process {
         cpus = 2
         memory = { 4.GB * task.attempt }
         // 2.5 * t_min + 5
-        time = { ((2.5 * params.clip_duration / 30 / 60 + 5) * 1.5).toInteger() + '.sec' * task.attempt }
+        time = {
+            get_dynamic_resource(
+            min: 60,
+            m: 2.5,
+            x: params.clip_duration / 30 / 60,
+            b: 5,
+            attempt: task.attempt
+            ) + '.sec'
+        }
         array = 200
         errorStrategy = 'retry'
         maxRetries = 3
@@ -265,13 +298,21 @@ process {
         cpus = 1
         // 0.0048 * t_min + 0.080
         memory = {
-            def base_mb = Math.max(1024, ((0.0048 * params.clip_duration / 30 / 60 + 0.08) * 1.5 * 1024).toInteger())
-            return (base_mb * task.attempt).MB
+            get_dynamic_resource(min: 1,
+                m: 0.0048,
+                x: params.clip_duration / 30 / 60,
+                b: 0.08,
+                attempt: task.attempt
+            ) + '.GB'
         }
         // 0.6 * t_min + 30
         time = {
-            def base_min = Math.max(1, ((0.6 * params.clip_duration / 30 / 60 + 30) * 1.5 / 60).toInteger())
-            return (base_min * task.attempt).min
+            get_dynamic_resource(min: 1,
+                m: 0.6,
+                x: params.clip_duration / 30 / 60,
+                b: 30,
+                attempt: task.attempt
+            ) + '.min'
         }
         array = 200
         errorStrategy = 'retry'
@@ -280,7 +321,15 @@ process {
     withLabel: "r_rearpaw" {
         cpus = 1
         // 0.0047 * t_min + 0.0801
-        memory = { ((0.0047 * params.clip_duration / 30 / 60 + 0.0801) * 1.5).toInteger() + '.GB' * task.attempt }
+        memory = {
+            get_dynamic_resource(
+                min: 1,
+                m: 0.0047,
+                x: params.clip_duration / 30 / 60,
+                b: 0.0801,
+                attempt: task.attempt
+            ) + '.GB'
+        }
         time = { 10.min * task.attempt }
         array = 200
         errorStrategy = 'retry'
@@ -290,7 +339,15 @@ process {
     withLabel: "r_gait_h5" {
         cpus = 1
         // 0.0009 * t_min + 0.727
-        memory = { ((0.0009 * params.clip_duration / 30 / 60 + 0.727) * 1.5).toInteger() + '.GB' * task.attempt }
+        memory = {
+            get_dynamic_resource(
+                min: 1,
+                m: 0.0009,
+                x: params.clip_duration / 30 / 60,
+                b: 0.727,
+                attempt: task.attempt
+            ) + '.GB'
+        }
         time = { 10.min * task.attempt }
         array = 200
         errorStrategy = 'retry'
@@ -299,7 +356,15 @@ process {
     withLabel: "r_gait_bin" {
         cpus = 1
         // 0.000005 * t_min + 0.0095
-        memory = { ((0.000005 * params.clip_duration / 30 / 60 + 0.0095) * 1.5).toInteger() + '.GB' * task.attempt }
+        memory = {
+            get_dynamic_resource(
+                min: 1,
+                m: 0.000005,
+                x: params.clip_duration / 30 / 60,
+                b: 0.0095,
+                attempt: task.attempt
+            ) + '.GB'
+        }
         time = { 5.min * task.attempt }
         array = 200
         errorStrategy = 'retry'
@@ -312,9 +377,27 @@ process {
         // are more computationally expensive than per-frame features (calculated once per pose file)
         cpus = 1
         // 0.182 * t_min - 0.273 for 6 windows
-        memory = { ((0.182 * params.clip_duration / 30 / 60) * 2.5).toInteger() + '.GB' * task.attempt }
+        memory = {
+            get_dynamic_resource(
+                min: 1,
+                m: 0.182,
+                x: params.clip_duration / 30 / 60,
+                b: 0,
+                safety: 2.5,
+                attempt: task.attempt
+            ) + '.GB'
+        }
         // 162 * t_min + 4.6 for 6 windows
-        time = { ((162 * params.clip_duration / 30 / 60 + 4.6) * 2.5).toInteger() + '.sec' * task.attempt }
+        time = {
+            get_dynamic_resource(
+                min: 1,
+                m: 162,
+                x: params.clip_duration / 30 / 60,
+                b: 4.6,
+                safety: 2.5,
+                attempt: task.attempt
+            ) + '.sec'
+        }
         array = 200
         errorStrategy = 'retry'
         maxRetries = 3
@@ -324,9 +407,27 @@ process {
         // Adding more classifiers will scale linearly
         cpus = 1
         // 0.429 * t_min + 0.3 for 10 classifiers
-        memory = { ((0.429 * params.clip_duration / 30 / 60 + 0.3) * 2.5).toInteger() + '.GB' * task.attempt }
+        memory = {
+            get_dynamic_resource(
+                min: 1,
+                m: 0.429,
+                x: params.clip_duration / 30 / 60,
+                b: 0.3,
+                safety: 2.5,
+                attempt: task.attempt
+            ) + '.GB'
+        }
         // 30 * t_min + 127 for 10 classifiers
-        time = { ((30 * params.clip_duration / 30 / 60 + 127) * 2.5).toInteger() + '.sec' * task.attempt }
+        time = {
+            get_dynamic_resource(
+                min: 1,
+                m: 30,
+                x: params.clip_duration / 30 / 60,
+                b: 127,
+                safety: 2.5,
+                attempt: task.attempt
+            ) + '.sec'
+        }
         array = 200
         errorStrategy = 'retry'
         maxRetries = 3
@@ -345,7 +446,15 @@ process {
         cpus = 1
         memory = { 1.GB * task.attempt }
         // 0.111 * t_min + 37 for 6 classifiers
-        time = { ((0.111 * params.clip_duration / 30 / 60 + 37) * 1.5).toInteger() + '.sec' * task.attempt }
+        time = {
+            get_dynamic_resource(
+                min: 1,
+                m: 0.111,
+                x: params.clip_duration / 30 / 60,
+                b: 37,
+                attempt: task.attempt
+            ) + '.sec'
+        }
         array = 200
         errorStrategy = 'retry'
         maxRetries = 3
@@ -367,9 +476,27 @@ process {
         // because the shape of matrices grows by the prediction data
         // Equation provided, but numbers were manually increased
         // 0.06 * t_min + 1.86
-        memory = { ((0.06 * params.clip_duration / 30 / 60 + 2) * 2.5).toInteger() + '.GB' * task.attempt }
+        memory = {
+            get_dynamic_resource(
+                min: 1,
+                m: 0.06,
+                x: params.clip_duration / 30 / 60,
+                b: 2,
+                safety: 2.5,
+                attempt: task.attempt
+            ) + '.GB'
+        }
         // 15 * t_min - 63
-        time = { ((15 * params.clip_duration / 30 / 60 + 0) * 2.5).toInteger() + '.sec' * task.attempt }
+        time = {
+            get_dynamic_resource(
+                min: 1,
+                m: 15,
+                x: params.clip_duration / 30 / 60,
+                b: 0,
+                safety: 2.5,
+                attempt: task.attempt
+            ) + '.sec'
+        }
         array = 200
         errorStrategy = 'retry'
         maxRetries = 3
@@ -396,9 +523,26 @@ process {
     withLabel: "r_clip_video" {
         cpus = 2
         // 0.0000003 * t_min + 0.35
-        memory = { ((0.0000003 * params.clip_duration / 30 / 60 + 0.35) * 1.5).toInteger() + '.GB' * task.attempt }
+        memory = {
+            get_dynamic_resource(
+                min: 1,
+                m: 0.0000003,
+                x: params.clip_duration / 30 / 60,
+                b: 0.35,
+                attempt: task.attempt
+            ) + '.GB'
+        }
         // 19.8 * t_min + 52
-        time = { ((19.8 * params.clip_duration / 30 / 60 + 52) * 2.5).toInteger() + '.sec' * task.attempt }
+        time = {
+            get_dynamic_resource(
+                min: 1,
+                m: 19.8,
+                x: params.clip_duration / 30 / 60,
+                b: 52,
+                safety: 2.5,
+                attempt: task.attempt
+            ) + '.sec'
+        }
         array = 200
         errorStrategy = 'retry'
         maxRetries = 3
@@ -408,7 +552,15 @@ process {
     withLabel: "r_multi_seg" {
         cpus = 2
         // 8.53 GB * num_mice
-        memory = { (8.53 * params.num_mice * 1.5).toInteger() + '.GB' * task.attempt }
+        memory = {
+            get_dynamic_resource(
+                min: 1,
+                m: 8.53,
+                x: params.num_mice,
+                b: 0,
+                attempt: task.attempt
+            ) + '.GB'
+        }
         time = { 5.hours * task.attempt }
         array = 200
         errorStrategy = 'retry'
@@ -417,9 +569,25 @@ process {
     withLabel: "r_multi_keypoints" {
         cpus = 2
         // 2.26 GB * num_mice + 2.47 GB
-        memory = { ((2.26 * params.num_mice + 2.47) * 1.5).toInteger() + '.GB' * task.attempt }
+        memory = {
+            get_dynamic_resource(
+                min: 1,
+                m: 2.26,
+                x: params.num_mice,
+                b: 2.47,
+                attempt: task.attempt
+            ) + '.GB'
+        }
         // 6 * num_mice + 3
-        time = { ((6 * params.num_mice + 3) * 1.5).toInteger() + '.hours' * task.attempt }
+        time = {
+            get_dynamic_resource(
+                min: 1,
+                m: 6,
+                x: params.num_mice,
+                b: 3,
+                attempt: task.attempt
+            ) + '.hours'
+        }
         array = 200
         errorStrategy = 'retry'
         maxRetries = 3
@@ -428,11 +596,26 @@ process {
         cpus = 2
         memory = { 2.GB * task.attempt }
         // 0.1 * num_mice + 0.16
-        time = { ((0.1 * params.num_mice + 0.16) * 1.5).toInteger() + '.hours' * task.attempt }
+        time = {
+            get_dynamic_resource(
+                min: 1,
+                m: 0.1,
+                x: params.num_mice,
+                b: 0.16,
+                attempt: task.attempt
+            ) + '.hours'
+        }
         array = 200
         errorStrategy = 'retry'
         maxRetries = 3
     }
+    withLabel: "r_multi_tracklets" {
+        cpus = 1
+        // TODO: Tune these numbers based on number of mice
+        // First video suggested ~7GB and ~30min for 3 mice
+        memory = { 10.GB * task.attempt }
+        time = { 45.min * task.attempt }
+    }
     // Static Object Resources
     // These do not scale with clip duration
     withLabel: "r_arena_corners" {
diff --git a/nextflow/modules/multi_mouse.nf b/nextflow/modules/multi_mouse.nf
index 7c215cd..e29b258 100644
--- a/nextflow/modules/multi_mouse.nf
+++ b/nextflow/modules/multi_mouse.nf
@@ -2,7 +2,7 @@ process PREDICT_MULTI_MOUSE_SEGMENTATION {
     label "gpu_long"
     label "tracking"
     label "r_multi_seg"
-    
+
     input:
     tuple path(video_file), path(in_pose)
 
@@ -20,7 +20,7 @@ process PREDICT_MULTI_MOUSE_KEYPOINTS {
     label "gpu_long"
     label "tracking"
     label "r_multi_keypoints"
-    
+
     input:
     tuple path(video_file), path(in_pose)
 
@@ -38,7 +38,7 @@ process PREDICT_MULTI_MOUSE_IDENTITY {
     label "gpu"
     label "tracking"
     label "r_multi_identity"
-    
+
     input:
     tuple path(video_file), path(in_pose)
 
@@ -55,7 +55,8 @@ process PREDICT_MULTI_MOUSE_IDENTITY {
 process GENERATE_MULTI_MOUSE_TRACKLETS {
     label "cpu"
     label "tracking"
-    
+    label "r_multi_tracklets"
+
     input:
     tuple path(video_file), path(in_pose)
     val num_animals

From d21102738312480d11fde5b723ccbc6fe8d9ba94 Mon Sep 17 00:00:00 2001
From: Brian Geuther <Brian.Geuther@jax.org>
Date: Tue, 2 Sep 2025 14:36:41 -0400
Subject: [PATCH 6/8] Reverting pretty format because nextflow seems to not
 parse the function def

---
 nextflow/configs/profiles/sumner2.config | 202 +++++------------------
 1 file changed, 40 insertions(+), 162 deletions(-)

diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config
index dd9ee15..606cfac 100644
--- a/nextflow/configs/profiles/sumner2.config
+++ b/nextflow/configs/profiles/sumner2.config
@@ -2,23 +2,6 @@
 
 workDir = "/flashscratch/${USER}/nextflow-work"
 
-/**
- * Get dynamic resource allocation based on parameters
- *
- * @param minimum Minimum resource allocation
- * @param m Slope of the dynamic resource
- * @param x Var that scales resources
- * @param b Constant added to the request
- * @param safety Safety margin for the request
- * @param attempt Current attempt number (1-based)
- *
- * @return integer of the resources requested
- */
-def get_dynamic_resource(float minimum, float m, float x, float b, float safety=1.5, int attempt=1) {
-    def base = Math.max(minimum, (b + m * x) * safety).toInteger()
-    return base * attempt
-}
-
 params {
     /*
      * Additional Parameters for the sumner2 profile
@@ -264,13 +247,8 @@ process {
         memory = { 1.GB * task.attempt }
         // 0.5 * t_min + 10.5
         time = {
-            get_dynamic_resource(
-                min: 60,
-                m: 0.5,
-                x: params.clip_duration / 30 / 60,
-                b: 10.5,
-                attempt: task.attempt
-            ) + '.sec'
+            def r_value = Math.max(60, (0.5 * (params.clip_duration / 30 / 60) + 10.5) * 1.5).toInteger() * task.attempt
+            return r_value + '.sec'
         }
         array = 200
         errorStrategy = 'retry'
@@ -281,13 +259,8 @@ process {
         memory = { 4.GB * task.attempt }
         // 2.5 * t_min + 5
         time = {
-            get_dynamic_resource(
-            min: 60,
-            m: 2.5,
-            x: params.clip_duration / 30 / 60,
-            b: 5,
-            attempt: task.attempt
-            ) + '.sec'
+            def r_value = Math.max(60, (2.5 * (params.clip_duration / 30 / 60) + 5) * 1.5).toInteger() * task.attempt
+            return r_value + '.sec'
         }
         array = 200
         errorStrategy = 'retry'
@@ -298,21 +271,13 @@ process {
         cpus = 1
         // 0.0048 * t_min + 0.080
         memory = {
-            get_dynamic_resource(min: 1,
-                m: 0.0048,
-                x: params.clip_duration / 30 / 60,
-                b: 0.08,
-                attempt: task.attempt
-            ) + '.GB'
+            def r_value = Math.max(1, (0.0048 * (params.clip_duration / 30 / 60) + 0.08) * 1.5).toInteger() * task.attempt
+            return r_value + '.GB'
         }
         // 0.6 * t_min + 30
         time = {
-            get_dynamic_resource(min: 1,
-                m: 0.6,
-                x: params.clip_duration / 30 / 60,
-                b: 30,
-                attempt: task.attempt
-            ) + '.min'
+            def r_value = Math.max(10, (0.6 * (params.clip_duration / 30 / 60) + 30) * 1.5).toInteger() * task.attempt
+            return r_value + '.min'
         }
         array = 200
         errorStrategy = 'retry'
@@ -322,13 +287,8 @@ process {
         cpus = 1
         // 0.0047 * t_min + 0.0801
         memory = {
-            get_dynamic_resource(
-                min: 1,
-                m: 0.0047,
-                x: params.clip_duration / 30 / 60,
-                b: 0.0801,
-                attempt: task.attempt
-            ) + '.GB'
+            def r_value = Math.max(1, (0.0047 * (params.clip_duration / 30 / 60) + 0.0801) * 1.5).toInteger() * task.attempt
+            return r_value + '.GB'
         }
         time = { 10.min * task.attempt }
         array = 200
@@ -340,13 +300,8 @@ process {
         cpus = 1
         // 0.0009 * t_min + 0.727
         memory = {
-            get_dynamic_resource(
-                min: 1,
-                m: 0.0009,
-                x: params.clip_duration / 30 / 60,
-                b: 0.727,
-                attempt: task.attempt
-            ) + '.GB'
+            def r_value = Math.max(1, (0.0009 * (params.clip_duration / 30 / 60) + 0.727) * 1.5).toInteger() * task.attempt
+            return r_value + '.GB'
         }
         time = { 10.min * task.attempt }
         array = 200
@@ -357,13 +312,8 @@ process {
         cpus = 1
         // 0.000005 * t_min + 0.0095
         memory = {
-            get_dynamic_resource(
-                min: 1,
-                m: 0.000005,
-                x: params.clip_duration / 30 / 60,
-                b: 0.0095,
-                attempt: task.attempt
-            ) + '.GB'
+            def r_value = Math.max(1, (0.000005 * (params.clip_duration / 30 / 60) + 0.0095) * 1.5).toInteger() * task.attempt
+            return r_value + '.GB'
         }
         time = { 5.min * task.attempt }
         array = 200
@@ -378,25 +328,13 @@ process {
         cpus = 1
         // 0.182 * t_min - 0.273 for 6 windows
         memory = {
-            get_dynamic_resource(
-                min: 1,
-                m: 0.182,
-                x: params.clip_duration / 30 / 60,
-                b: 0,
-                safety: 2.5,
-                attempt: task.attempt
-            ) + '.GB'
+            def r_value = Math.max(1, (0.182 * (params.clip_duration / 30 / 60) - 0.273) * 2.5).toInteger() * task.attempt
+            return r_value + '.GB'
         }
         // 162 * t_min + 4.6 for 6 windows
         time = {
-            get_dynamic_resource(
-                min: 1,
-                m: 162,
-                x: params.clip_duration / 30 / 60,
-                b: 4.6,
-                safety: 2.5,
-                attempt: task.attempt
-            ) + '.sec'
+            def r_value = Math.max(1, (162 * (params.clip_duration / 30 / 60) + 4.6) * 2.5).toInteger() * task.attempt
+            return r_value + '.sec'
         }
         array = 200
         errorStrategy = 'retry'
@@ -408,25 +346,13 @@ process {
         cpus = 1
         // 0.429 * t_min + 0.3 for 10 classifiers
         memory = {
-            get_dynamic_resource(
-                min: 1,
-                m: 0.429,
-                x: params.clip_duration / 30 / 60,
-                b: 0.3,
-                safety: 2.5,
-                attempt: task.attempt
-            ) + '.GB'
+            def r_value = Math.max(1, (0.429 * (params.clip_duration / 30 / 60) + 0.3) * 2.5).toInteger() * task.attempt
+            return r_value + '.GB'
         }
         // 30 * t_min + 127 for 10 classifiers
         time = {
-            get_dynamic_resource(
-                min: 1,
-                m: 30,
-                x: params.clip_duration / 30 / 60,
-                b: 127,
-                safety: 2.5,
-                attempt: task.attempt
-            ) + '.sec'
+            def r_value = Math.max(1, (127 + 30 * (params.clip_duration / 30 / 60)) * 2.5).toInteger() * task.attempt
+            return r_value + '.sec'
         }
         array = 200
         errorStrategy = 'retry'
@@ -447,13 +373,8 @@ process {
         memory = { 1.GB * task.attempt }
         // 0.111 * t_min + 37 for 6 classifiers
         time = {
-            get_dynamic_resource(
-                min: 1,
-                m: 0.111,
-                x: params.clip_duration / 30 / 60,
-                b: 37,
-                attempt: task.attempt
-            ) + '.sec'
+            def r_value = Math.max(1, (0.111 * (params.clip_duration / 30 / 60) + 37) * 2.5).toInteger() * task.attempt
+            return r_value + '.sec'
         }
         array = 200
         errorStrategy = 'retry'
@@ -477,25 +398,13 @@ process {
         // Equation provided, but numbers were manually increased
         // 0.06 * t_min + 1.86
         memory = {
-            get_dynamic_resource(
-                min: 1,
-                m: 0.06,
-                x: params.clip_duration / 30 / 60,
-                b: 2,
-                safety: 2.5,
-                attempt: task.attempt
-            ) + '.GB'
+            def r_value = Math.max(1, (0.06 * (params.clip_duration / 30 / 60) + 1.86) * 2.5).toInteger() * task.attempt
+            return r_value + '.GB'
         }
         // 15 * t_min - 63
         time = {
-            get_dynamic_resource(
-                min: 1,
-                m: 15,
-                x: params.clip_duration / 30 / 60,
-                b: 0,
-                safety: 2.5,
-                attempt: task.attempt
-            ) + '.sec'
+            def r_value = Math.max(1, (15 * (params.clip_duration / 30 / 60) - 63) * 2.5).toInteger() * task.attempt
+            return r_value + '.sec'
         }
         array = 200
         errorStrategy = 'retry'
@@ -524,24 +433,13 @@ process {
         cpus = 2
         // 0.0000003 * t_min + 0.35
         memory = {
-            get_dynamic_resource(
-                min: 1,
-                m: 0.0000003,
-                x: params.clip_duration / 30 / 60,
-                b: 0.35,
-                attempt: task.attempt
-            ) + '.GB'
+            def r_value = Math.max(1, (0.0000003 * (params.clip_duration / 30 / 60) + 0.35) * 2.5).toInteger() * task.attempt
+            return r_value + '.GB'
         }
         // 19.8 * t_min + 52
         time = {
-            get_dynamic_resource(
-                min: 1,
-                m: 19.8,
-                x: params.clip_duration / 30 / 60,
-                b: 52,
-                safety: 2.5,
-                attempt: task.attempt
-            ) + '.sec'
+            def r_value = Math.max(1, (52 + 19.8 * (params.clip_duration / 30 / 60)) * 2.5).toInteger() * task.attempt
+            return r_value + '.sec'
         }
         array = 200
         errorStrategy = 'retry'
@@ -553,13 +451,8 @@ process {
         cpus = 2
         // 8.53 GB * num_mice
         memory = {
-            get_dynamic_resource(
-                min: 1,
-                m: 8.53,
-                x: params.num_mice,
-                b: 0,
-                attempt: task.attempt
-            ) + '.GB'
+            def r_value = Math.max(1, (8.53 * params.num_mice) * 2.5).toInteger() * task.attempt
+            return r_value + '.GB'
         }
         time = { 5.hours * task.attempt }
         array = 200
@@ -570,23 +463,13 @@ process {
         cpus = 2
         // 2.26 GB * num_mice + 2.47 GB
         memory = {
-            get_dynamic_resource(
-                min: 1,
-                m: 2.26,
-                x: params.num_mice,
-                b: 2.47,
-                attempt: task.attempt
-            ) + '.GB'
+            def r_value = Math.max(1, (2.26 * params.num_mice + 2.47) * 2.5).toInteger() * task.attempt
+            return r_value + '.GB'
         }
         // 6 * num_mice + 3
         time = {
-            get_dynamic_resource(
-                min: 1,
-                m: 6,
-                x: params.num_mice,
-                b: 3,
-                attempt: task.attempt
-            ) + '.hours'
+            def r_value = Math.max(1, (3 + 6 * params.num_mice) * 2.5).toInteger() * task.attempt
+            return r_value + '.hours'
         }
         array = 200
         errorStrategy = 'retry'
@@ -597,13 +480,8 @@ process {
         memory = { 2.GB * task.attempt }
         // 0.1 * num_mice + 0.16
         time = {
-            get_dynamic_resource(
-                min: 1,
-                m: 0.1,
-                x: params.num_mice,
-                b: 0.16,
-                attempt: task.attempt
-            ) + '.hours'
+            def r_value = Math.max(1, (0.1 * params.num_mice + 0.16) * 2.5).toInteger() * task.attempt
+            return r_value + '.hours'
         }
         array = 200
         errorStrategy = 'retry'

From 28cf1b64c8e223bda0131764c9e92adc17dddca4 Mon Sep 17 00:00:00 2001
From: Brian Geuther <Brian.Geuther@jax.org>
Date: Tue, 2 Sep 2025 15:08:29 -0400
Subject: [PATCH 7/8] Max with only ints, not floats or doubles.

---
 nextflow/configs/profiles/sumner2.config | 40 ++++++++++++------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config
index 606cfac..ccf1fcd 100644
--- a/nextflow/configs/profiles/sumner2.config
+++ b/nextflow/configs/profiles/sumner2.config
@@ -247,7 +247,7 @@ process {
         memory = { 1.GB * task.attempt }
         // 0.5 * t_min + 10.5
         time = {
-            def r_value = Math.max(60, (0.5 * (params.clip_duration / 30 / 60) + 10.5) * 1.5).toInteger() * task.attempt
+            def r_value = Math.max(60, ((0.5 * (params.clip_duration / 30 / 60) + 10.5) * 1.5 * task.attempt).toInteger())
             return r_value + '.sec'
         }
         array = 200
@@ -259,7 +259,7 @@ process {
         memory = { 4.GB * task.attempt }
         // 2.5 * t_min + 5
         time = {
-            def r_value = Math.max(60, (2.5 * (params.clip_duration / 30 / 60) + 5) * 1.5).toInteger() * task.attempt
+            def r_value = Math.max(60, ((2.5 * (params.clip_duration / 30 / 60) + 5) * 1.5 * task.attempt).toInteger())
             return r_value + '.sec'
         }
         array = 200
@@ -271,12 +271,12 @@ process {
         cpus = 1
         // 0.0048 * t_min + 0.080
         memory = {
-            def r_value = Math.max(1, (0.0048 * (params.clip_duration / 30 / 60) + 0.08) * 1.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((0.0048 * (params.clip_duration / 30 / 60) + 0.08) * 1.5 * task.attempt).toInteger())
             return r_value + '.GB'
         }
         // 0.6 * t_min + 30
         time = {
-            def r_value = Math.max(10, (0.6 * (params.clip_duration / 30 / 60) + 30) * 1.5).toInteger() * task.attempt
+            def r_value = Math.max(10, ((0.6 * (params.clip_duration / 30 / 60) + 30) * 1.5 * task.attempt).toInteger())
             return r_value + '.min'
         }
         array = 200
@@ -287,7 +287,7 @@ process {
         cpus = 1
         // 0.0047 * t_min + 0.0801
         memory = {
-            def r_value = Math.max(1, (0.0047 * (params.clip_duration / 30 / 60) + 0.0801) * 1.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((0.0047 * (params.clip_duration / 30 / 60) + 0.0801) * 1.5 * task.attempt).toInteger())
             return r_value + '.GB'
         }
         time = { 10.min * task.attempt }
@@ -300,7 +300,7 @@ process {
         cpus = 1
         // 0.0009 * t_min + 0.727
         memory = {
-            def r_value = Math.max(1, (0.0009 * (params.clip_duration / 30 / 60) + 0.727) * 1.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((0.0009 * (params.clip_duration / 30 / 60) + 0.727) * 1.5 * task.attempt).toInteger())
             return r_value + '.GB'
         }
         time = { 10.min * task.attempt }
@@ -312,7 +312,7 @@ process {
         cpus = 1
         // 0.000005 * t_min + 0.0095
         memory = {
-            def r_value = Math.max(1, (0.000005 * (params.clip_duration / 30 / 60) + 0.0095) * 1.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((0.000005 * (params.clip_duration / 30 / 60) + 0.0095) * 1.5 * task.attempt).toInteger())
             return r_value + '.GB'
         }
         time = { 5.min * task.attempt }
@@ -328,12 +328,12 @@ process {
         cpus = 1
         // 0.182 * t_min - 0.273 for 6 windows
         memory = {
-            def r_value = Math.max(1, (0.182 * (params.clip_duration / 30 / 60) - 0.273) * 2.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((0.182 * (params.clip_duration / 30 / 60) - 0.273) * 2.5 * task.attempt).toInteger())
             return r_value + '.GB'
         }
         // 162 * t_min + 4.6 for 6 windows
         time = {
-            def r_value = Math.max(1, (162 * (params.clip_duration / 30 / 60) + 4.6) * 2.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((162 * (params.clip_duration / 30 / 60) + 4.6) * 2.5 * task.attempt).toInteger())
             return r_value + '.sec'
         }
         array = 200
@@ -346,12 +346,12 @@ process {
         cpus = 1
         // 0.429 * t_min + 0.3 for 10 classifiers
         memory = {
-            def r_value = Math.max(1, (0.429 * (params.clip_duration / 30 / 60) + 0.3) * 2.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((0.429 * (params.clip_duration / 30 / 60) + 0.3) * 2.5 * task.attempt).toInteger())
             return r_value + '.GB'
         }
         // 30 * t_min + 127 for 10 classifiers
         time = {
-            def r_value = Math.max(1, (127 + 30 * (params.clip_duration / 30 / 60)) * 2.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((127 + 30 * (params.clip_duration / 30 / 60)) * 2.5 * task.attempt).toInteger())
             return r_value + '.sec'
         }
         array = 200
@@ -373,7 +373,7 @@ process {
         memory = { 1.GB * task.attempt }
         // 0.111 * t_min + 37 for 6 classifiers
         time = {
-            def r_value = Math.max(1, (0.111 * (params.clip_duration / 30 / 60) + 37) * 2.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((0.111 * (params.clip_duration / 30 / 60) + 37) * 2.5 * task.attempt).toInteger())
             return r_value + '.sec'
         }
         array = 200
@@ -398,12 +398,12 @@ process {
         // Equation provided, but numbers were manually increased
         // 0.06 * t_min + 1.86
         memory = {
-            def r_value = Math.max(1, (0.06 * (params.clip_duration / 30 / 60) + 1.86) * 2.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((0.06 * (params.clip_duration / 30 / 60) + 1.86) * 2.5 * task.attempt).toInteger())
             return r_value + '.GB'
         }
         // 15 * t_min - 63
         time = {
-            def r_value = Math.max(1, (15 * (params.clip_duration / 30 / 60) - 63) * 2.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((15 * (params.clip_duration / 30 / 60) - 63) * 2.5 * task.attempt).toInteger())
             return r_value + '.sec'
         }
         array = 200
@@ -433,12 +433,12 @@ process {
         cpus = 2
         // 0.0000003 * t_min + 0.35
         memory = {
-            def r_value = Math.max(1, (0.0000003 * (params.clip_duration / 30 / 60) + 0.35) * 2.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((0.0000003 * (params.clip_duration / 30 / 60) + 0.35) * 2.5 * task.attempt).toInteger())
             return r_value + '.GB'
         }
         // 19.8 * t_min + 52
         time = {
-            def r_value = Math.max(1, (52 + 19.8 * (params.clip_duration / 30 / 60)) * 2.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((52 + 19.8 * (params.clip_duration / 30 / 60)) * 2.5 * task.attempt).toInteger())
             return r_value + '.sec'
         }
         array = 200
@@ -451,7 +451,7 @@ process {
         cpus = 2
         // 8.53 GB * num_mice
         memory = {
-            def r_value = Math.max(1, (8.53 * params.num_mice) * 2.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((8.53 * params.num_mice) * 2.5 * task.attempt).toInteger())
             return r_value + '.GB'
         }
         time = { 5.hours * task.attempt }
@@ -463,12 +463,12 @@ process {
         cpus = 2
         // 2.26 GB * num_mice + 2.47 GB
         memory = {
-            def r_value = Math.max(1, (2.26 * params.num_mice + 2.47) * 2.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((2.26 * params.num_mice + 2.47) * 2.5 * task.attempt).toInteger())
             return r_value + '.GB'
         }
         // 6 * num_mice + 3
         time = {
-            def r_value = Math.max(1, (3 + 6 * params.num_mice) * 2.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((3 + 6 * params.num_mice) * 2.5 * task.attempt).toInteger())
             return r_value + '.hours'
         }
         array = 200
@@ -480,7 +480,7 @@ process {
         memory = { 2.GB * task.attempt }
         // 0.1 * num_mice + 0.16
         time = {
-            def r_value = Math.max(1, (0.1 * params.num_mice + 0.16) * 2.5).toInteger() * task.attempt
+            def r_value = Math.max(1, ((0.1 * params.num_mice + 0.16) * 2.5 * task.attempt).toInteger())
             return r_value + '.hours'
         }
         array = 200

From 2a8585e340d13d0e236bc8dfc3e183aaa7011ab5 Mon Sep 17 00:00:00 2001
From: Brian Geuther <Brian.Geuther@jax.org>
Date: Wed, 3 Sep 2025 15:21:30 -0400
Subject: [PATCH 8/8] Adjusting to 16GB start

---
 nextflow/configs/profiles/sumner2.config | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config
index ccf1fcd..cceacfc 100644
--- a/nextflow/configs/profiles/sumner2.config
+++ b/nextflow/configs/profiles/sumner2.config
@@ -490,8 +490,8 @@ process {
     withLabel: "r_multi_tracklets" {
         cpus = 1
         // TODO: Tune these numbers based on number of mice
-        // First video suggested ~7GB and ~30min for 3 mice
-        memory = { 10.GB * task.attempt }
+        // First video suggested ~16GB and ~30min for 3 mice
+        memory = { 16.GB * task.attempt }
         time = { 45.min * task.attempt }
     }
     // Static Object Resources