From a19bd8e688f611e703713c823801dcd79785bf27 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Tue, 30 Sep 2025 07:22:24 -0400 Subject: [PATCH 1/9] First draft of classifier generation pipeline --- nextflow/bootstrap/classifier_source.config | 96 ++++++++ nextflow/bootstrap/main.nf | 233 ++++++++++++++++++++ 2 files changed, 329 insertions(+) create mode 100644 nextflow/bootstrap/classifier_source.config create mode 100644 nextflow/bootstrap/main.nf diff --git a/nextflow/bootstrap/classifier_source.config b/nextflow/bootstrap/classifier_source.config new file mode 100644 index 0000000..0e59308 --- /dev/null +++ b/nextflow/bootstrap/classifier_source.config @@ -0,0 +1,96 @@ +// Example configuration for the classifier bootstrap pipeline. +// Update the paths and classifiers to match your environment before running. + +params { + // Root directory where classifier artifacts will be written. + classifier_base_path = "/projects/kumar-lab/data/classifiers" + + // Directory containing source JABS project folders referenced below. + classifier_project_folders = "/projects/kumar-lab/projects" + + // Version label for this bootstrap build. + jabs_version = "v0.36.1" + + // Map of behaviors to their source projects and metadata. + single_mouse_classifiers = [ + "grooming": [ + "project_folder_name": "grooming", + "stitch_value": 10*30, + "filter_value": 3*30, + ], + "scratch": [ + "project_folder_name": "jabs-paper", + "stitch_value": 5, + "filter_value": 5, + ], + // "Leg_splaying": [ // Currently can't be exported successfully + // "project_folder_name": "ptz", + // "stitch_value": 5, + // "filter_value": 5, + // ], + "Side seizure": [ + "project_folder_name": "ptz", + "stitch_value": 5, + "filter_value": 5, + ], + "Tail_jerk": [ + "project_folder_name": "ptz", + "stitch_value": 5, + "filter_value": 5, + ], + "Wild_jumping": [ + "project_folder_name": "ptz", + "stitch_value": 5, + "filter_value": 5, + ], + "Escape": [ + "project_folder_name": "jabs-vivek", + "stitch_value": 5, + "filter_value": 5, + ], + "Rearing_supported": [ + "project_folder_name": "jabs-vivek", + "stitch_value": 5, + "filter_value": 5, + ], + "Rearing_unsupported": [ + "project_folder_name": "jabs-vivek", + "stitch_value": 5, + "filter_value": 5, + ], + "Turn_left": [ + "project_folder_name": "jabs-vivek", + "stitch_value": 5, + "filter_value": 5, + ], + "Turn_right": [ + "project_folder_name": "jabs-vivek", + "stitch_value": 5, + "filter_value": 5, + ], + ] +} + +apptainer { + enabled = true + autoMounts = true +} + +process { + withLabel: "jabs_classify" { + container = "/projects/kumar-lab/meta/images/JABS-behavior-classifier/headless/v0.36.1/latest.sif" + } + + withLabel: "cpu" { + queue = "compute" + resourceLimits = [ cpus: 1, memory: 24.GB, time: 24.h ] + } +} + +executor { + name = 'slurm' + // The number of tasks the executor will handle in a parallel manner + queueSize = 24 + submitRateLimit = '1 s' + // Determines the max rate of job submission per time unit, for example '10sec' eg. max 10 jobs per second or '1/2 s' i.e. 1 job submissions every 2 seconds. +} diff --git a/nextflow/bootstrap/main.nf b/nextflow/bootstrap/main.nf new file mode 100644 index 0000000..709384e --- /dev/null +++ b/nextflow/bootstrap/main.nf @@ -0,0 +1,233 @@ +nextflow.enable.dsl=2 + +/* + * This bootstrap workflow generates JABS classifiers and their associated metadata. + * It produces versioned, content-hashed artifacts and a Nextflow configuration + * file that can be used by the main analysis pipeline. + */ + +/** + * Creates a version directory and a JSON config file with metadata about the build. + */ +process CREATE_VERSION_CONFIG { + tag "config_${params.jabs_version}" + label 'cpu' + publishDir "${params.classifier_base_path}/${params.jabs_version}", mode: 'copy', overwrite: true + + output: + path "jabs.config.json" + + script: + """ + printf '{ +' > "jabs.config.json" + printf ' "jabs_version": "${params.jabs_version}", +' >> "jabs.config.json" + printf ' "creation_timestamp_utc": "%s" +' "\$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "jabs.config.json" + printf '} +' >> "jabs.config.json" + """ +} + + +process EXPORT_TRAINING_DATA { + tag "export_${behavior_path}" + label 'jabs_classify' + label 'cpu' + + publishDir path: "${params.classifier_base_path}/${params.jabs_version}/${behavior_path}", + mode: 'copy' + + input: + tuple val(behavior_name), val(behavior_path), val(project_folder_name) + + output: + tuple val(behavior_name), val(behavior_path), val(project_folder_name), path("training.h5"), env('HASH'), emit: h5_file_with_hash + path "${behavior_path}_*.training.h5" + path "${behavior_path}_*.training.h5.manifest.json" + + script: + def project_path = "${params.classifier_project_folders}/${project_folder_name}" + """ + # 1. Export to a local file + jabs-cli export-training --behavior "${behavior_name}" --outfile "training.h5" "${project_path}" + + # 2. Calculate hash and export it + export HASH=\$(sha256sum "training.h5" | awk '{ print \$1 }') + + # 3. Create the content-addressed files + cp training.h5 "${behavior_path}_\${HASH}.training.h5" + ln -s "${behavior_path}_\${HASH}.training.h5" "latest.training.h5" + + # 4. Create the manifest file + cat > "${behavior_path}_\${HASH}.training.h5.manifest.json" < "${behavior_path}_\${CLASSIFIER_HASH}.pickle.manifest.json" < + writer.writeLine("params {") + writer.writeLine(" single_mouse_classifiers = [") + behaviors.each { behavior_tuple -> + def behavior_name = behavior_tuple[0] + def behavior_path = behavior_tuple[1] + def details = params.single_mouse_classifiers[behavior_name] + def classifier_path = "${params.classifier_base_path}/${params.jabs_version}/\${behavior_path}/latest.pickle" + + writer.writeLine(" "\${behavior_name}": [") + writer.writeLine(" classifier_path: "\${classifier_path}",") + writer.writeLine(" stitch_value: \${details.stitch_value},") + writer.writeLine(" filter_value: \${details.filter_value}") + writer.writeLine(" ],") + } + writer.writeLine(" ]") + writer.writeLine("}") + } + """ +} + +/** + * Generates a Nextflow configuration file that maps behaviors to their + * generated classifier artifacts for the main analysis pipeline. + */ +process GENERATE_PIPELINE_CONFIG { + tag "generate_config" + label 'cpu' + publishDir "${params.classifier_base_path}/${params.jabs_version}", mode: 'copy', overwrite: true + + input: + val collected_behaviors // list of [behavior_name, behavior_path] + + output: + path "generated_classifiers.config" + + script: + // Convert collected_behaviors to a format Python can parse + def behaviors_json = groovy.json.JsonOutput.toJson(collected_behaviors) + def classifiers_json = groovy.json.JsonOutput.toJson(params.single_mouse_classifiers) + """ +#!/usr/bin/env python3 +import json + +behaviors = json.loads('${behaviors_json}') +classifiers = json.loads('${classifiers_json}') + +with open("generated_classifiers.config", "w") as f: + f.write("params {\\n") + f.write(" single_mouse_classifiers = [\\n") + + # Iterate through behaviors in pairs (behavior_name, behavior_path) + for i in range(0, len(behaviors), 2): + behavior_name = behaviors[i] + behavior_path = behaviors[i + 1] + details = classifiers[behavior_name] + classifier_path = "${params.classifier_base_path}/${params.jabs_version}/" + behavior_path + "/latest.pickle" + + f.write(f" '{behavior_name}': [\\n") + f.write(f" classifier_path: '{classifier_path}',\\n") + f.write(f" stitch_value: {details['stitch_value']},\\n") + f.write(f" filter_value: {details['filter_value']}\\n") + f.write(f" ],\\n") + + f.write(" ]\\n") + f.write("}\\n") +""" +} + + +/** + * Main workflow for generating classifiers and the pipeline config. + */ +workflow { + main: + CREATE_VERSION_CONFIG() + + classifier_ch = Channel.from(params.single_mouse_classifiers.collect { k, v -> [k, v] }) + behavior_projects_ch = classifier_ch.map { behavior_name, details -> + def behavior_path = behavior_name.replaceAll(' ', '_').replaceAll('[()]', '') + tuple(behavior_name, behavior_path, details.project_folder_name) + } + + exported_h5_ch = EXPORT_TRAINING_DATA(behavior_projects_ch) + + trained_classifiers_ch = TRAIN_CLASSIFIER(exported_h5_ch.h5_file_with_hash) + + // Collect only the behavior names that were successfully trained + trained_classifiers_ch.classifier_file + .map { behavior_name, behavior_path, classifier_file -> tuple(behavior_name, behavior_path) } + .collect() + .set{ collected_behaviors } + + GENERATE_PIPELINE_CONFIG(collected_behaviors) +} \ No newline at end of file From dc8c8fbbdc4201931efeb2ad4a9ae87431be00a3 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Tue, 30 Sep 2025 07:41:53 -0400 Subject: [PATCH 2/9] Classifier generation cleanup --- nextflow/bootstrap/main.nf | 54 +++++--------------------------------- 1 file changed, 7 insertions(+), 47 deletions(-) diff --git a/nextflow/bootstrap/main.nf b/nextflow/bootstrap/main.nf index 709384e..559c5d5 100644 --- a/nextflow/bootstrap/main.nf +++ b/nextflow/bootstrap/main.nf @@ -45,7 +45,8 @@ process EXPORT_TRAINING_DATA { output: tuple val(behavior_name), val(behavior_path), val(project_folder_name), path("training.h5"), env('HASH'), emit: h5_file_with_hash path "${behavior_path}_*.training.h5" - path "${behavior_path}_*.training.h5.manifest.json" + path "${behavior_path}_*.training.h5.manifest.json" + path "latest.training.h5" script: def project_path = "${params.classifier_project_folders}/${project_folder_name}" @@ -115,48 +116,6 @@ EOF """ } -/** - * Generates a Nextflow configuration file that maps behaviors to their - * generated classifier artifacts for the main analysis pipeline. - */ -process GENERATE_PIPELINE_CONFIG_ORIG { - tag "generate_config" - label 'cpu' - publishDir "${params.classifier_base_path}/${params.jabs_version}", mode: 'copy', overwrite: true - - input: - val collected_behaviors // list of [behavior_name, behavior_path] - - output: - path "generated_classifiers.config" - - script: - """ - #!/usr/bin/env groovy - def behaviors = ${collected_behaviors.inspect()} - def outFile = new File("generated_classifiers.config") - - outFile.withWriter { writer -> - writer.writeLine("params {") - writer.writeLine(" single_mouse_classifiers = [") - behaviors.each { behavior_tuple -> - def behavior_name = behavior_tuple[0] - def behavior_path = behavior_tuple[1] - def details = params.single_mouse_classifiers[behavior_name] - def classifier_path = "${params.classifier_base_path}/${params.jabs_version}/\${behavior_path}/latest.pickle" - - writer.writeLine(" "\${behavior_name}": [") - writer.writeLine(" classifier_path: "\${classifier_path}",") - writer.writeLine(" stitch_value: \${details.stitch_value},") - writer.writeLine(" filter_value: \${details.filter_value}") - writer.writeLine(" ],") - } - writer.writeLine(" ]") - writer.writeLine("}") - } - """ -} - /** * Generates a Nextflow configuration file that maps behaviors to their * generated classifier artifacts for the main analysis pipeline. @@ -186,20 +145,20 @@ classifiers = json.loads('${classifiers_json}') with open("generated_classifiers.config", "w") as f: f.write("params {\\n") f.write(" single_mouse_classifiers = [\\n") - + # Iterate through behaviors in pairs (behavior_name, behavior_path) for i in range(0, len(behaviors), 2): behavior_name = behaviors[i] behavior_path = behaviors[i + 1] details = classifiers[behavior_name] classifier_path = "${params.classifier_base_path}/${params.jabs_version}/" + behavior_path + "/latest.pickle" - + f.write(f" '{behavior_name}': [\\n") f.write(f" classifier_path: '{classifier_path}',\\n") f.write(f" stitch_value: {details['stitch_value']},\\n") f.write(f" filter_value: {details['filter_value']}\\n") f.write(f" ],\\n") - + f.write(" ]\\n") f.write("}\\n") """ @@ -230,4 +189,5 @@ workflow { .set{ collected_behaviors } GENERATE_PIPELINE_CONFIG(collected_behaviors) -} \ No newline at end of file +} + From 74d3e52994f12c4f28465826c360fd8fe445a695 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Tue, 30 Sep 2025 07:47:16 -0400 Subject: [PATCH 3/9] Add init-jabs project step --- nextflow/bootstrap/main.nf | 49 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/nextflow/bootstrap/main.nf b/nextflow/bootstrap/main.nf index 559c5d5..63bcebf 100644 --- a/nextflow/bootstrap/main.nf +++ b/nextflow/bootstrap/main.nf @@ -30,6 +30,24 @@ process CREATE_VERSION_CONFIG { """ } +process INIT_JABS_PROJECTS { + label 'jabs_classify' + label 'cpu' + cpus 8 + + input: + val(project_folder_name) + + output: + val(project_folder_name), emit: initialized_project + + script: + def project_path = "${params.classifier_project_folders}/${project_folder_name}" + def project_file = "${project_path}/jabs/project.json" + """ + jabs-init "${project_path}" \$(jq -r '[.behavior[] | .window_size] | unique | map("-w \\(.)") | join(" ")' ${project_file}) + """ +} process EXPORT_TRAINING_DATA { tag "export_${behavior_path}" @@ -172,13 +190,41 @@ workflow { main: CREATE_VERSION_CONFIG() + // Create channel of behaviors with their details classifier_ch = Channel.from(params.single_mouse_classifiers.collect { k, v -> [k, v] }) + + // Map to include behavior_path behavior_projects_ch = classifier_ch.map { behavior_name, details -> def behavior_path = behavior_name.replaceAll(' ', '_').replaceAll('[()]', '') tuple(behavior_name, behavior_path, details.project_folder_name) } - exported_h5_ch = EXPORT_TRAINING_DATA(behavior_projects_ch) + // Extract unique project folders + unique_projects_ch = classifier_ch + .map { behavior_name, details -> details.project_folder_name } + .unique() + + // Initialize each unique project + initialized_projects_ch = INIT_JABS_PROJECTS(unique_projects_ch) + + // Create a value channel from initialized projects for joining + initialized_projects_val = initialized_projects_ch.initialized_project + .collect() + .map { projects -> + projects.collectEntries { [it, true] } + } + + // Join behaviors with their initialized projects + ready_behaviors_ch = behavior_projects_ch + .combine(initialized_projects_val) + .map { behavior_name, behavior_path, project_folder_name, project_map -> + // Check if this behavior's project has been initialized + if (project_map[project_folder_name]) { + tuple(behavior_name, behavior_path, project_folder_name) + } + } + + exported_h5_ch = EXPORT_TRAINING_DATA(ready_behaviors_ch) trained_classifiers_ch = TRAIN_CLASSIFIER(exported_h5_ch.h5_file_with_hash) @@ -190,4 +236,3 @@ workflow { GENERATE_PIPELINE_CONFIG(collected_behaviors) } - From 7fa3d8e216b79e85462b20069b4465cbbe268a45 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Mon, 6 Oct 2025 16:02:42 -0400 Subject: [PATCH 4/9] Update classifier generation based on development usage --- nextflow/bootstrap/classifier_source.config | 43 +++++----- nextflow/bootstrap/main.nf | 89 ++++++++++++--------- 2 files changed, 77 insertions(+), 55 deletions(-) diff --git a/nextflow/bootstrap/classifier_source.config b/nextflow/bootstrap/classifier_source.config index 0e59308..d064d36 100644 --- a/nextflow/bootstrap/classifier_source.config +++ b/nextflow/bootstrap/classifier_source.config @@ -3,21 +3,21 @@ params { // Root directory where classifier artifacts will be written. - classifier_base_path = "/projects/kumar-lab/data/classifiers" + classifier_base_path = "/projects/kumar-lab/data/jabs/classifiers" // Directory containing source JABS project folders referenced below. - classifier_project_folders = "/projects/kumar-lab/projects" - + classifier_project_folders = "/projects/kumar-lab/data/jabs/projects" + // Version label for this bootstrap build. - jabs_version = "v0.36.1" + jabs_version = "v0.37.0" // Map of behaviors to their source projects and metadata. single_mouse_classifiers = [ - "grooming": [ - "project_folder_name": "grooming", - "stitch_value": 10*30, - "filter_value": 3*30, - ], + // "grooming": [ + // "project_folder_name": "grooming", + // "stitch_value": 10*30, + // "filter_value": 3*30, + // ], "scratch": [ "project_folder_name": "jabs-paper", "stitch_value": 5, @@ -33,12 +33,12 @@ params { "stitch_value": 5, "filter_value": 5, ], - "Tail_jerk": [ + "Tail jerk": [ "project_folder_name": "ptz", "stitch_value": 5, "filter_value": 5, ], - "Wild_jumping": [ + "Wild jumping": [ "project_folder_name": "ptz", "stitch_value": 5, "filter_value": 5, @@ -48,22 +48,22 @@ params { "stitch_value": 5, "filter_value": 5, ], - "Rearing_supported": [ + "Rearing (supported)": [ "project_folder_name": "jabs-vivek", "stitch_value": 5, "filter_value": 5, ], - "Rearing_unsupported": [ + "Rearing (unsupported)": [ "project_folder_name": "jabs-vivek", "stitch_value": 5, "filter_value": 5, ], - "Turn_left": [ + "Turn left": [ "project_folder_name": "jabs-vivek", "stitch_value": 5, "filter_value": 5, ], - "Turn_right": [ + "Turn right": [ "project_folder_name": "jabs-vivek", "stitch_value": 5, "filter_value": 5, @@ -78,12 +78,19 @@ apptainer { process { withLabel: "jabs_classify" { - container = "/projects/kumar-lab/meta/images/JABS-behavior-classifier/headless/v0.36.1/latest.sif" + container = "/projects/kumar-lab/meta/images/JABS-behavior-classifier/headless/v0.37.0/latest.sif" } - withLabel: "cpu" { queue = "compute" - resourceLimits = [ cpus: 1, memory: 24.GB, time: 24.h ] + cpus = 1 + memory = 24.GB + time = 24.h + } + withLabel: "highcpu" { + queue = "compute" + cpus = 16 + memory = 64.GB + time = 24.h } } diff --git a/nextflow/bootstrap/main.nf b/nextflow/bootstrap/main.nf index 63bcebf..cc0f248 100644 --- a/nextflow/bootstrap/main.nf +++ b/nextflow/bootstrap/main.nf @@ -30,22 +30,37 @@ process CREATE_VERSION_CONFIG { """ } +process CALCULTE_PROJECT_WINDOW_SIZES { + label 'cpu' + + input: + val(project_folder_name) + + output: + tuple val(project_folder_name), env('WINDOW_SIZES'), emit: project_with_windows + + script: + def project_path = "${params.classifier_project_folders}/${project_folder_name}" + def project_file = "${project_path}/jabs/project.json" + """ + export WINDOW_SIZES=\$(jq -r '[.behavior[] | .window_size] | unique | map("-w \\(.)") | join(" ")' ${project_file}) + """ +} + process INIT_JABS_PROJECTS { label 'jabs_classify' - label 'cpu' - cpus 8 + label 'highcpu' input: - val(project_folder_name) + tuple val(project_folder_name), val(window_sizes) output: val(project_folder_name), emit: initialized_project script: def project_path = "${params.classifier_project_folders}/${project_folder_name}" - def project_file = "${project_path}/jabs/project.json" """ - jabs-init "${project_path}" \$(jq -r '[.behavior[] | .window_size] | unique | map("-w \\(.)") | join(" ")' ${project_file}) + jabs-init "${project_path}" ${window_sizes} -p 16 """ } @@ -61,10 +76,8 @@ process EXPORT_TRAINING_DATA { tuple val(behavior_name), val(behavior_path), val(project_folder_name) output: - tuple val(behavior_name), val(behavior_path), val(project_folder_name), path("training.h5"), env('HASH'), emit: h5_file_with_hash - path "${behavior_path}_*.training.h5" - path "${behavior_path}_*.training.h5.manifest.json" - path "latest.training.h5" + tuple val(behavior_name), val(behavior_path), val(project_folder_name), path("*.h5"), env('HASH'), emit: h5_file_with_hash + path "*.h5.manifest.json" script: def project_path = "${params.classifier_project_folders}/${project_folder_name}" @@ -76,11 +89,10 @@ process EXPORT_TRAINING_DATA { export HASH=\$(sha256sum "training.h5" | awk '{ print \$1 }') # 3. Create the content-addressed files - cp training.h5 "${behavior_path}_\${HASH}.training.h5" - ln -s "${behavior_path}_\${HASH}.training.h5" "latest.training.h5" + mv training.h5 "\${HASH}.h5" # 4. Create the manifest file - cat > "${behavior_path}_\${HASH}.training.h5.manifest.json" < "\${HASH}.h5.manifest.json" < "${behavior_path}_\${CLASSIFIER_HASH}.pickle.manifest.json" < "\${HASH}.pickle.manifest.json" < details.project_folder_name } .unique() + // Calcualte project windows outside of container so we have access to jq + project_with_windows = CALCULTE_PROJECT_WINDOW_SIZES(unique_projects_ch) + // Initialize each unique project - initialized_projects_ch = INIT_JABS_PROJECTS(unique_projects_ch) + initialized_projects_ch = INIT_JABS_PROJECTS(project_with_windows) // Create a value channel from initialized projects for joining initialized_projects_val = initialized_projects_ch.initialized_project @@ -230,8 +245,8 @@ workflow { // Collect only the behavior names that were successfully trained trained_classifiers_ch.classifier_file - .map { behavior_name, behavior_path, classifier_file -> tuple(behavior_name, behavior_path) } - .collect() + .map { behavior_name, behavior_path, _classifier_file, hash -> tuple(behavior_name, behavior_path, hash) } + .toList() .set{ collected_behaviors } GENERATE_PIPELINE_CONFIG(collected_behaviors) From 1044c6320337214632248aeb3ed339d63e743a12 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Mon, 6 Oct 2025 17:01:09 -0400 Subject: [PATCH 5/9] Update main workflow with changes to single_mouse_classifier objects --- nextflow/configs/profiles/sumner2.config | 75 ++---------------------- nextflow/modules/jabs_classifiers.nf | 5 +- 2 files changed, 6 insertions(+), 74 deletions(-) diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config index da11354..0877c8d 100644 --- a/nextflow/configs/profiles/sumner2.config +++ b/nextflow/configs/profiles/sumner2.config @@ -2,6 +2,8 @@ workDir = "/flashscratch/${USER}/nextflow-work" +includeConfig { "/projects/kumar-lab/data/jabs/classifiers/v0.37.0/latest.config" } + params { /* * Additional Parameters for the sumner2 profile @@ -14,10 +16,6 @@ params { * - filter_processed: whether to filter processed files in the batch * - jabs_version: version of JABS used for the classifiers - * - classifier_project_folders: directory containing the classifier project folders - * - classifier_training_file_folder: directory containing the classifier training files - * - exported_classifier_folder: directory containing the exported classifiers - * - classifier_artifact_suffix: suffix for the classifier artifacts * - classifier_window_sizes: window sizes cached for use in classifiers * - single_mouse_classifiers: classifiers for single mouse behavior. Each classifier is described as behavior name (identical key used in JABS): [ @@ -37,75 +35,10 @@ params { heuristic_classifier_folder = "/JABS-postprocess/heuristic_classifiers/" filter_processed = false - jabs_version = "0.18.1" - classifier_project_folders = "/projects/kumar-lab/multimouse-pipeline/nextflow-artifacts/project_folders/" - classifier_training_file_folder = "/projects/kumar-lab/multimouse-pipeline/nextflow-artifacts/training_files/" - exported_classifier_folder = "/projects/kumar-lab/multimouse-pipeline/nextflow-artifacts/exported_classifiers/" - classifier_artifact_suffix = "_classifier_v${jabs_version}.pickle" + jabs_version = "v0.37.0" classifier_window_sizes = [2, 5, 10, 20, 30, 60] // Classifiers are described as behavior_name: project_folder - single_mouse_classifiers = [ - "grooming": [ - "project_folder_name": "grooming", - "stitch_value": 10*30, - "filter_value": 3*30, - ], - "scratch": [ - "project_folder_name": "jabs-paper", - "stitch_value": 5, - "filter_value": 5, - ], - // "Leg_splaying": [ // Currently can't be exported successfully - // "project_folder_name": "ptz", - // "stitch_value": 5, - // "filter_value": 5, - // ], - "Side_seizure": [ - "project_folder_name": "ptz", - "stitch_value": 5, - "filter_value": 5, - ], - "Tail_jerk": [ - "project_folder_name": "ptz", - "stitch_value": 5, - "filter_value": 5, - ], - "Wild_jumping": [ - "project_folder_name": "ptz", - "stitch_value": 5, - "filter_value": 5, - ], - "Escape": [ - "project_folder_name": "jabs-vivek", - "stitch_value": 5, - "filter_value": 5, - ], - "Rearing_supported": [ - "project_folder_name": "jabs-vivek", - "stitch_value": 5, - "filter_value": 5, - ], - "Rearing_unsupported": [ - "project_folder_name": "jabs-vivek", - "stitch_value": 5, - "filter_value": 5, - ], - "Turn_left": [ - "project_folder_name": "jabs-vivek", - "stitch_value": 5, - "filter_value": 5, - ], - "Turn_right": [ - "project_folder_name": "jabs-vivek", - "stitch_value": 5, - "filter_value": 5, - ], - // "Jerk": [ // Needs to be re-exported to remove static object features (unavailable for single mouse) - // "project_folder_name": "social-play", - // "stitch_value": 1, - // "filter_value": 6, - // ] - ] + heuristic_classifiers = ["corner", "corner_facing", "freeze", "locomotion", "periphery", "wall_facing"] // Number of 5-minute bins for transforming summary tables into bins diff --git a/nextflow/modules/jabs_classifiers.nf b/nextflow/modules/jabs_classifiers.nf index d890274..b70b032 100644 --- a/nextflow/modules/jabs_classifiers.nf +++ b/nextflow/modules/jabs_classifiers.nf @@ -67,10 +67,9 @@ process PREDICT_CLASSIFIERS { script: """ - for classifier in ${classifiers.keySet().collect { params.exported_classifier_folder + it + params.classifier_artifact_suffix }.join(' ')}; + for classifier_path in ${classifiers.collect { _behavior, details -> details.classifier_path }.join(' ')}; do - ln -s \${classifier} . - jabs-classify classify --classifier \$(basename \${classifier}) --input-pose ${in_pose} --out-dir . --feature-dir . + jabs-classify classify --classifier "\${classifier_path}" --input-pose ${in_pose} --out-dir . --feature-dir . done """ } From 02ac4fab340afe7fd6e8e83ff12d5a850285a9e2 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Mon, 6 Oct 2025 17:07:24 -0400 Subject: [PATCH 6/9] Removing includeConfig from sumner2 profile --- nextflow/configs/profiles/sumner2.config | 56 +++++++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config index 0877c8d..bc2e10f 100644 --- a/nextflow/configs/profiles/sumner2.config +++ b/nextflow/configs/profiles/sumner2.config @@ -2,8 +2,6 @@ workDir = "/flashscratch/${USER}/nextflow-work" -includeConfig { "/projects/kumar-lab/data/jabs/classifiers/v0.37.0/latest.config" } - params { /* * Additional Parameters for the sumner2 profile @@ -38,6 +36,60 @@ params { jabs_version = "v0.37.0" classifier_window_sizes = [2, 5, 10, 20, 30, 60] // Classifiers are described as behavior_name: project_folder + jabs_version = "v0.37.0" + classifier_window_sizes = [2, 3, 5, 10, 16, 20, 30, 60] + single_mouse_classifiers = [ + 'grooming': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/grooming/12199145d31213f79e23669d211cfa2fd33cedb9f11197b44f90a4b71297b6d2.pickle', + stitch_value: 300, + filter_value: 90 + ], + 'Side seizure': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Side_seizure/50e865cfb045ff5b0c6bbad1dabf6465de43ef52ab4e13900b915986a51e9a0c.pickle', + stitch_value: 5, + filter_value: 5 + ], + 'scratch': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/scratch/261f1c21f015d47d30154fbbec98cb8860a40fbe383d3d86d6778e3da0f88a18.pickle', + stitch_value: 5, + filter_value: 5 + ], + 'Tail jerk': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Tail_jerk/34313467d88e7fc64ee90e10549eb48cfb9782d5e7a246fe106188c8ae50cdff.pickle', + stitch_value: 5, + filter_value: 5 + ], + 'Wild jumping': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Wild_jumping/7f114ce1b23a978afe82b4d9b7ec94c20982a2d5e7d16aefe4c5e0383324096a.pickle', + stitch_value: 5, + filter_value: 5 + ], + 'Escape': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Escape/0fa6b28e58fcc873f0c4ef053f5118566589d62d8e445b34ceaf1f962b5f8915.pickle', + stitch_value: 5, + filter_value: 5 + ], + 'Rearing (unsupported)': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Rearing_unsupported/562453b77d1a681a75c49bd774b5451e79e9e5f8784b48978f0f8cabbd90721e.pickle', + stitch_value: 5, + filter_value: 5 + ], + 'Rearing (supported)': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Rearing_supported/09ce1ed47e12556866045306bd221328454a8ccb5bc7ea7778a97e4882b386b6.pickle', + stitch_value: 5, + filter_value: 5 + ], + 'Turn left': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Turn_left/4779947116201f5fab5f2ec04f0ad4ee97a1e83649b6089d64354a7b07c42e4b.pickle', + stitch_value: 5, + filter_value: 5 + ], + 'Turn right': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Turn_right/b6fe24481513b87b757a61ad9495f7a9af270fad3cbec5d6b25556ece5f9c50a.pickle', + stitch_value: 5, + filter_value: 5 + ], + ] heuristic_classifiers = ["corner", "corner_facing", "freeze", "locomotion", "periphery", "wall_facing"] From 7c5b4910b4d5ed7ffcc38cb4241dddc8ad631c50 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Mon, 6 Oct 2025 17:24:47 -0400 Subject: [PATCH 7/9] Adding additional docstrings to classifier generation workflow --- nextflow/bootstrap/main.nf | 50 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/nextflow/bootstrap/main.nf b/nextflow/bootstrap/main.nf index cc0f248..308e11d 100644 --- a/nextflow/bootstrap/main.nf +++ b/nextflow/bootstrap/main.nf @@ -8,6 +8,8 @@ nextflow.enable.dsl=2 /** * Creates a version directory and a JSON config file with metadata about the build. + * + * @return jabs.config.json A JSON configuration file containing the JABS version and creation timestamp */ process CREATE_VERSION_CONFIG { tag "config_${params.jabs_version}" @@ -30,6 +32,14 @@ process CREATE_VERSION_CONFIG { """ } +/** + * Calculates the unique window sizes required for a JABS project by parsing the project.json file. + * Window sizes are extracted from behavior definitions and formatted as command-line arguments. + * + * @param project_folder_name The name of the JABS project folder to analyze + * + * @return project_with_windows A tuple containing the project folder name and formatted window size arguments (e.g., "-w 5 -w 10") + */ process CALCULTE_PROJECT_WINDOW_SIZES { label 'cpu' @@ -47,6 +57,15 @@ process CALCULTE_PROJECT_WINDOW_SIZES { """ } +/** + * Initializes a JABS project by running the jabs-init command with the specified window sizes. + * This process prepares the project for training data export and classifier generation. + * + * @param project_folder_name The name of the JABS project folder to initialize + * @param window_sizes Formatted window size arguments (e.g., "-w 5 -w 10") to use for initialization + * + * @return initialized_project The project folder name after successful initialization + */ process INIT_JABS_PROJECTS { label 'jabs_classify' label 'highcpu' @@ -64,6 +83,17 @@ process INIT_JABS_PROJECTS { """ } +/** + * Exports training data for a specific behavior from a JABS project to an HDF5 file. + * Creates content-addressed files using SHA256 hashing and generates metadata manifests. + * + * @param behavior_name The name of the behavior to export training data for + * @param behavior_path The filesystem-safe path derived from the behavior name + * @param project_folder_name The name of the JABS project folder containing the training data + * + * @return h5_file_with_hash A tuple containing behavior info, the content-addressed HDF5 file, and its SHA256 hash + * @return *.h5.manifest.json A JSON manifest file containing metadata about the training data export + */ process EXPORT_TRAINING_DATA { tag "export_${behavior_path}" label 'jabs_classify' @@ -104,6 +134,20 @@ EOF """ } +/** + * Trains a JABS classifier from the exported training data HDF5 file. + * Creates content-addressed classifier files using SHA256 hashing and generates metadata manifests + * that link back to the source training data. + * + * @param behavior_name The name of the behavior being classified + * @param behavior_path The filesystem-safe path derived from the behavior name + * @param project_folder_name The name of the source JABS project folder + * @param training_h5 The HDF5 file containing training data + * @param training_hash The SHA256 hash of the training data file + * + * @return classifier_file A tuple containing behavior info, the content-addressed classifier pickle file, and its SHA256 hash + * @return *.pickle.manifest.json A JSON manifest file containing metadata about the classifier and its training provenance + */ process TRAIN_CLASSIFIER { tag "train_${behavior_path}" label 'jabs_classify' @@ -148,6 +192,12 @@ EOF /** * Generates a Nextflow configuration file that maps behaviors to their * generated classifier artifacts for the main analysis pipeline. + * The generated config includes classifier paths, stitch values, and filter values + * for each behavior, creating a complete single_mouse_classifiers parameter structure. + * + * @param collected_behaviors A list of tuples containing [behavior_name, behavior_path, classifier_hash] for all successfully trained classifiers + * + * @return generated_classifiers.config A Nextflow configuration file that can be included in the main pipeline to reference the generated classifiers */ process GENERATE_PIPELINE_CONFIG { tag "generate_config" From 737ea184bdb0c42f9a0ddc45b68f787c49d8bff2 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Tue, 7 Oct 2025 07:50:12 -0400 Subject: [PATCH 8/9] Fix jabs-postprocess incorrect arguments --- nextflow/modules/jabs_classifiers.nf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nextflow/modules/jabs_classifiers.nf b/nextflow/modules/jabs_classifiers.nf index b70b032..ccdb5d2 100644 --- a/nextflow/modules/jabs_classifiers.nf +++ b/nextflow/modules/jabs_classifiers.nf @@ -106,10 +106,10 @@ process GENERATE_BEHAVIOR_TABLES { """ behavior_command="--behavior ${classifiers.collect { entry -> "$entry.key --stitch_gap $entry.value.stitch_value --min_bout_length $entry.value.filter_value" }.join(' --behavior ')}" jabs-postprocess generate-tables \ - --project_folder . \ - --feature_folder . \ - --out_prefix ${in_pose.baseName} \ - --out_bin_size 5 \ + --project-folder . \ + --feature-folder . \ + --out-prefix ${in_pose.baseName} \ + --out-bin-size 5 \ \${behavior_command} """ } @@ -148,7 +148,7 @@ process PREDICT_HEURISTICS { --feature-folder . \ --behavior-config \${classifier} \ --out-prefix ${in_pose.baseName} \ - --out_bin_size 5 + --out-bin-size 5 done """ } From b5f3eb1bfaaea350a14504c7e608e7075eb52fb0 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Tue, 7 Oct 2025 14:09:41 -0400 Subject: [PATCH 9/9] Fix another jabs-postprocess command argument --- nextflow/modules/jabs_classifiers.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow/modules/jabs_classifiers.nf b/nextflow/modules/jabs_classifiers.nf index ccdb5d2..6f9efc8 100644 --- a/nextflow/modules/jabs_classifiers.nf +++ b/nextflow/modules/jabs_classifiers.nf @@ -104,7 +104,7 @@ process GENERATE_BEHAVIOR_TABLES { script: """ - behavior_command="--behavior ${classifiers.collect { entry -> "$entry.key --stitch_gap $entry.value.stitch_value --min_bout_length $entry.value.filter_value" }.join(' --behavior ')}" + behavior_command="--behavior ${classifiers.collect { entry -> "$entry.key --stitch-gap $entry.value.stitch_value --min-bout-length $entry.value.filter_value" }.join(' --behavior ')}" jabs-postprocess generate-tables \ --project-folder . \ --feature-folder . \