diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config index 43bceab..cceacfc 100644 --- a/nextflow/configs/profiles/sumner2.config +++ b/nextflow/configs/profiles/sumner2.config @@ -144,7 +144,15 @@ process { // gpu_training: 14day limit, 16gpu/user clusterOptions = '-q gpu_inference --gres gpu:1 --nice=64000000' containerOptions = "--nv" - resourceLimits = [ cpus: 6, memory: 63.GB, time: 6.h ] + // Fair sharing of memory should be 63GB, but full node has 2TB + resourceLimits = [ cpus: 6, memory: 128.GB, time: 6.h ] + } + withLabel: "gpu_long" { + queue = "gpu_a100_mig" + clusterOptions = '-q gpu_training --gres gpu:1 --nice=64000000' + containerOptions = "--nv" + // Fair sharing of memory should be 63GB, but full node has 2TB + resourceLimits = [ cpus: 6, memory: 128.GB, time: 14.d ] } // Alternative GPU profiles available on the cluster withLabel: "gpu_a100" { @@ -183,7 +191,7 @@ process { * Runtime options */ withLabel: "tracking" { - container = "/projects/kumar-lab/multimouse-pipeline/deployment-runtime_2025-03-05.sif" + container = "/projects/kumar-lab/multimouse-pipeline/nextflow-containers/deployment-runtime_2025-08-26.sif" } withLabel: "jabs_classify" { container = "/projects/kumar-lab/multimouse-pipeline/nextflow-containers/JABS-GUI_2025-02-12_v0.18.1.sif" @@ -208,7 +216,7 @@ process { } withLabel: "rclone" { // executor.queueSize = 1 - container = "/projects/kumar-lab/multimouse-pipeline/rclone.sif" + container = "/projects/kumar-lab/multimouse-pipeline/rclone_2025-08-18.sif" } /* @@ -238,7 +246,10 @@ process { cpus = 1 memory = { 1.GB * task.attempt } // 0.5 * t_min + 10.5 - time = { ((0.5 * params.clip_duration / 30 / 60 + 10.5) * 1.5).toInteger() + '.sec' * task.attempt } + time = { + def r_value = Math.max(60, ((0.5 * (params.clip_duration / 30 / 60) + 10.5) * 1.5 * task.attempt).toInteger()) + return r_value + '.sec' + } array = 200 errorStrategy = 'retry' maxRetries = 3 @@ -247,7 +258,10 @@ process { cpus = 2 memory = { 4.GB * task.attempt } // 2.5 * t_min + 5 - time = { ((2.5 * params.clip_duration / 30 / 60 + 5) * 1.5).toInteger() + '.sec' * task.attempt } + time = { + def r_value = Math.max(60, ((2.5 * (params.clip_duration / 30 / 60) + 5) * 1.5 * task.attempt).toInteger()) + return r_value + '.sec' + } array = 200 errorStrategy = 'retry' maxRetries = 3 @@ -257,13 +271,13 @@ process { cpus = 1 // 0.0048 * t_min + 0.080 memory = { - def base_mb = Math.max(1024, ((0.0048 * params.clip_duration / 30 / 60 + 0.08) * 1.5 * 1024).toInteger()) - return (base_mb * task.attempt).MB + def r_value = Math.max(1, ((0.0048 * (params.clip_duration / 30 / 60) + 0.08) * 1.5 * task.attempt).toInteger()) + return r_value + '.GB' } // 0.6 * t_min + 30 time = { - def base_min = Math.max(1, ((0.6 * params.clip_duration / 30 / 60 + 30) * 1.5 / 60).toInteger()) - return (base_min * task.attempt).min + def r_value = Math.max(10, ((0.6 * (params.clip_duration / 30 / 60) + 30) * 1.5 * task.attempt).toInteger()) + return r_value + '.min' } array = 200 errorStrategy = 'retry' @@ -272,7 +286,10 @@ process { withLabel: "r_rearpaw" { cpus = 1 // 0.0047 * t_min + 0.0801 - memory = { ((0.0047 * params.clip_duration / 30 / 60 + 0.0801) * 1.5).toInteger() + '.GB' * task.attempt } + memory = { + def r_value = Math.max(1, ((0.0047 * (params.clip_duration / 30 / 60) + 0.0801) * 1.5 * task.attempt).toInteger()) + return r_value + '.GB' + } time = { 10.min * task.attempt } array = 200 errorStrategy = 'retry' @@ -282,7 +299,10 @@ process { withLabel: "r_gait_h5" { cpus = 1 // 0.0009 * t_min + 0.727 - memory = { ((0.0009 * params.clip_duration / 30 / 60 + 0.727) * 1.5).toInteger() + '.GB' * task.attempt } + memory = { + def r_value = Math.max(1, ((0.0009 * (params.clip_duration / 30 / 60) + 0.727) * 1.5 * task.attempt).toInteger()) + return r_value + '.GB' + } time = { 10.min * task.attempt } array = 200 errorStrategy = 'retry' @@ -291,7 +311,10 @@ process { withLabel: "r_gait_bin" { cpus = 1 // 0.000005 * t_min + 0.0095 - memory = { ((0.000005 * params.clip_duration / 30 / 60 + 0.0095) * 1.5).toInteger() + '.GB' * task.attempt } + memory = { + def r_value = Math.max(1, ((0.000005 * (params.clip_duration / 30 / 60) + 0.0095) * 1.5 * task.attempt).toInteger()) + return r_value + '.GB' + } time = { 5.min * task.attempt } array = 200 errorStrategy = 'retry' @@ -304,9 +327,15 @@ process { // are more computationally expensive than per-frame features (calculated once per pose file) cpus = 1 // 0.182 * t_min - 0.273 for 6 windows - memory = { ((0.182 * params.clip_duration / 30 / 60) * 2.5).toInteger() + '.GB' * task.attempt } + memory = { + def r_value = Math.max(1, ((0.182 * (params.clip_duration / 30 / 60) - 0.273) * 2.5 * task.attempt).toInteger()) + return r_value + '.GB' + } // 162 * t_min + 4.6 for 6 windows - time = { ((162 * params.clip_duration / 30 / 60 + 4.6) * 2.5).toInteger() + '.sec' * task.attempt } + time = { + def r_value = Math.max(1, ((162 * (params.clip_duration / 30 / 60) + 4.6) * 2.5 * task.attempt).toInteger()) + return r_value + '.sec' + } array = 200 errorStrategy = 'retry' maxRetries = 3 @@ -316,9 +345,15 @@ process { // Adding more classifiers will scale linearly cpus = 1 // 0.429 * t_min + 0.3 for 10 classifiers - memory = { ((0.429 * params.clip_duration / 30 / 60 + 0.3) * 2.5).toInteger() + '.GB' * task.attempt } + memory = { + def r_value = Math.max(1, ((0.429 * (params.clip_duration / 30 / 60) + 0.3) * 2.5 * task.attempt).toInteger()) + return r_value + '.GB' + } // 30 * t_min + 127 for 10 classifiers - time = { ((30 * params.clip_duration / 30 / 60 + 127) * 2.5).toInteger() + '.sec' * task.attempt } + time = { + def r_value = Math.max(1, ((127 + 30 * (params.clip_duration / 30 / 60)) * 2.5 * task.attempt).toInteger()) + return r_value + '.sec' + } array = 200 errorStrategy = 'retry' maxRetries = 3 @@ -337,7 +372,10 @@ process { cpus = 1 memory = { 1.GB * task.attempt } // 0.111 * t_min + 37 for 6 classifiers - time = { ((0.111 * params.clip_duration / 30 / 60 + 37) * 1.5).toInteger() + '.sec' * task.attempt } + time = { + def r_value = Math.max(1, ((0.111 * (params.clip_duration / 30 / 60) + 37) * 2.5 * task.attempt).toInteger()) + return r_value + '.sec' + } array = 200 errorStrategy = 'retry' maxRetries = 3 @@ -359,9 +397,15 @@ process { // because the shape of matrices grows by the prediction data // Equation provided, but numbers were manually increased // 0.06 * t_min + 1.86 - memory = { ((0.06 * params.clip_duration / 30 / 60 + 2) * 2.5).toInteger() + '.GB' * task.attempt } + memory = { + def r_value = Math.max(1, ((0.06 * (params.clip_duration / 30 / 60) + 1.86) * 2.5 * task.attempt).toInteger()) + return r_value + '.GB' + } // 15 * t_min - 63 - time = { ((15 * params.clip_duration / 30 / 60 + 0) * 2.5).toInteger() + '.sec' * task.attempt } + time = { + def r_value = Math.max(1, ((15 * (params.clip_duration / 30 / 60) - 63) * 2.5 * task.attempt).toInteger()) + return r_value + '.sec' + } array = 200 errorStrategy = 'retry' maxRetries = 3 @@ -388,13 +432,68 @@ process { withLabel: "r_clip_video" { cpus = 2 // 0.0000003 * t_min + 0.35 - memory = { ((0.0000003 * params.clip_duration / 30 / 60 + 0.35) * 1.5).toInteger() + '.GB' * task.attempt } + memory = { + def r_value = Math.max(1, ((0.0000003 * (params.clip_duration / 30 / 60) + 0.35) * 2.5 * task.attempt).toInteger()) + return r_value + '.GB' + } // 19.8 * t_min + 52 - time = { ((19.8 * params.clip_duration / 30 / 60 + 52) * 2.5).toInteger() + '.sec' * task.attempt } + time = { + def r_value = Math.max(1, ((52 + 19.8 * (params.clip_duration / 30 / 60)) * 2.5 * task.attempt).toInteger()) + return r_value + '.sec' + } + array = 200 + errorStrategy = 'retry' + maxRetries = 3 + } + // Multi Mouse Resources + // These processes assume the input video is 1hr and scale with number of animals + withLabel: "r_multi_seg" { + cpus = 2 + // 8.53 GB * num_mice + memory = { + def r_value = Math.max(1, ((8.53 * params.num_mice) * 2.5 * task.attempt).toInteger()) + return r_value + '.GB' + } + time = { 5.hours * task.attempt } + array = 200 + errorStrategy = 'retry' + maxRetries = 3 + } + withLabel: "r_multi_keypoints" { + cpus = 2 + // 2.26 GB * num_mice + 2.47 GB + memory = { + def r_value = Math.max(1, ((2.26 * params.num_mice + 2.47) * 2.5 * task.attempt).toInteger()) + return r_value + '.GB' + } + // 6 * num_mice + 3 + time = { + def r_value = Math.max(1, ((3 + 6 * params.num_mice) * 2.5 * task.attempt).toInteger()) + return r_value + '.hours' + } + array = 200 + errorStrategy = 'retry' + maxRetries = 3 + } + withLabel: "r_multi_identity" { + cpus = 2 + memory = { 2.GB * task.attempt } + // 0.1 * num_mice + 0.16 + time = { + def r_value = Math.max(1, ((0.1 * params.num_mice + 0.16) * 2.5 * task.attempt).toInteger()) + return r_value + '.hours' + } array = 200 errorStrategy = 'retry' maxRetries = 3 } + withLabel: "r_multi_tracklets" { + cpus = 1 + // TODO: Tune these numbers based on number of mice + // First video suggested ~16GB and ~30min for 3 mice + memory = { 16.GB * task.attempt } + time = { 45.min * task.attempt } + } // Static Object Resources // These do not scale with clip duration withLabel: "r_arena_corners" { diff --git a/nextflow/modules/multi_mouse.nf b/nextflow/modules/multi_mouse.nf index a0c4431..e29b258 100644 --- a/nextflow/modules/multi_mouse.nf +++ b/nextflow/modules/multi_mouse.nf @@ -1,7 +1,8 @@ process PREDICT_MULTI_MOUSE_SEGMENTATION { - label "gpu" + label "gpu_long" label "tracking" - + label "r_multi_seg" + input: tuple path(video_file), path(in_pose) @@ -16,9 +17,10 @@ process PREDICT_MULTI_MOUSE_SEGMENTATION { } process PREDICT_MULTI_MOUSE_KEYPOINTS { - label "gpu" + label "gpu_long" label "tracking" - + label "r_multi_keypoints" + input: tuple path(video_file), path(in_pose) @@ -35,7 +37,8 @@ process PREDICT_MULTI_MOUSE_KEYPOINTS { process PREDICT_MULTI_MOUSE_IDENTITY { label "gpu" label "tracking" - + label "r_multi_identity" + input: tuple path(video_file), path(in_pose) @@ -52,7 +55,8 @@ process PREDICT_MULTI_MOUSE_IDENTITY { process GENERATE_MULTI_MOUSE_TRACKLETS { label "cpu" label "tracking" - + label "r_multi_tracklets" + input: tuple path(video_file), path(in_pose) val num_animals