diff --git a/nextflow/bootstrap/classifier_source.config b/nextflow/bootstrap/classifier_source.config new file mode 100644 index 0000000..d064d36 --- /dev/null +++ b/nextflow/bootstrap/classifier_source.config @@ -0,0 +1,103 @@ +// Example configuration for the classifier bootstrap pipeline. +// Update the paths and classifiers to match your environment before running. + +params { + // Root directory where classifier artifacts will be written. + classifier_base_path = "/projects/kumar-lab/data/jabs/classifiers" + + // Directory containing source JABS project folders referenced below. + classifier_project_folders = "/projects/kumar-lab/data/jabs/projects" + + // Version label for this bootstrap build. + jabs_version = "v0.37.0" + + // Map of behaviors to their source projects and metadata. + single_mouse_classifiers = [ + // "grooming": [ + // "project_folder_name": "grooming", + // "stitch_value": 10*30, + // "filter_value": 3*30, + // ], + "scratch": [ + "project_folder_name": "jabs-paper", + "stitch_value": 5, + "filter_value": 5, + ], + // "Leg_splaying": [ // Currently can't be exported successfully + // "project_folder_name": "ptz", + // "stitch_value": 5, + // "filter_value": 5, + // ], + "Side seizure": [ + "project_folder_name": "ptz", + "stitch_value": 5, + "filter_value": 5, + ], + "Tail jerk": [ + "project_folder_name": "ptz", + "stitch_value": 5, + "filter_value": 5, + ], + "Wild jumping": [ + "project_folder_name": "ptz", + "stitch_value": 5, + "filter_value": 5, + ], + "Escape": [ + "project_folder_name": "jabs-vivek", + "stitch_value": 5, + "filter_value": 5, + ], + "Rearing (supported)": [ + "project_folder_name": "jabs-vivek", + "stitch_value": 5, + "filter_value": 5, + ], + "Rearing (unsupported)": [ + "project_folder_name": "jabs-vivek", + "stitch_value": 5, + "filter_value": 5, + ], + "Turn left": [ + "project_folder_name": "jabs-vivek", + "stitch_value": 5, + "filter_value": 5, + ], + "Turn right": [ + "project_folder_name": "jabs-vivek", + "stitch_value": 5, + "filter_value": 5, + ], + ] +} + +apptainer { + enabled = true + autoMounts = true +} + +process { + withLabel: "jabs_classify" { + container = "/projects/kumar-lab/meta/images/JABS-behavior-classifier/headless/v0.37.0/latest.sif" + } + withLabel: "cpu" { + queue = "compute" + cpus = 1 + memory = 24.GB + time = 24.h + } + withLabel: "highcpu" { + queue = "compute" + cpus = 16 + memory = 64.GB + time = 24.h + } +} + +executor { + name = 'slurm' + // The number of tasks the executor will handle in a parallel manner + queueSize = 24 + submitRateLimit = '1 s' + // Determines the max rate of job submission per time unit, for example '10sec' eg. max 10 jobs per second or '1/2 s' i.e. 1 job submissions every 2 seconds. +} diff --git a/nextflow/bootstrap/main.nf b/nextflow/bootstrap/main.nf new file mode 100644 index 0000000..308e11d --- /dev/null +++ b/nextflow/bootstrap/main.nf @@ -0,0 +1,303 @@ +nextflow.enable.dsl=2 + +/* + * This bootstrap workflow generates JABS classifiers and their associated metadata. + * It produces versioned, content-hashed artifacts and a Nextflow configuration + * file that can be used by the main analysis pipeline. + */ + +/** + * Creates a version directory and a JSON config file with metadata about the build. + * + * @return jabs.config.json A JSON configuration file containing the JABS version and creation timestamp + */ +process CREATE_VERSION_CONFIG { + tag "config_${params.jabs_version}" + label 'cpu' + publishDir "${params.classifier_base_path}/${params.jabs_version}", mode: 'copy', overwrite: true + + output: + path "jabs.config.json" + + script: + """ + printf '{ +' > "jabs.config.json" + printf ' "jabs_version": "${params.jabs_version}", +' >> "jabs.config.json" + printf ' "creation_timestamp_utc": "%s" +' "\$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "jabs.config.json" + printf '} +' >> "jabs.config.json" + """ +} + +/** + * Calculates the unique window sizes required for a JABS project by parsing the project.json file. + * Window sizes are extracted from behavior definitions and formatted as command-line arguments. + * + * @param project_folder_name The name of the JABS project folder to analyze + * + * @return project_with_windows A tuple containing the project folder name and formatted window size arguments (e.g., "-w 5 -w 10") + */ +process CALCULTE_PROJECT_WINDOW_SIZES { + label 'cpu' + + input: + val(project_folder_name) + + output: + tuple val(project_folder_name), env('WINDOW_SIZES'), emit: project_with_windows + + script: + def project_path = "${params.classifier_project_folders}/${project_folder_name}" + def project_file = "${project_path}/jabs/project.json" + """ + export WINDOW_SIZES=\$(jq -r '[.behavior[] | .window_size] | unique | map("-w \\(.)") | join(" ")' ${project_file}) + """ +} + +/** + * Initializes a JABS project by running the jabs-init command with the specified window sizes. + * This process prepares the project for training data export and classifier generation. + * + * @param project_folder_name The name of the JABS project folder to initialize + * @param window_sizes Formatted window size arguments (e.g., "-w 5 -w 10") to use for initialization + * + * @return initialized_project The project folder name after successful initialization + */ +process INIT_JABS_PROJECTS { + label 'jabs_classify' + label 'highcpu' + + input: + tuple val(project_folder_name), val(window_sizes) + + output: + val(project_folder_name), emit: initialized_project + + script: + def project_path = "${params.classifier_project_folders}/${project_folder_name}" + """ + jabs-init "${project_path}" ${window_sizes} -p 16 + """ +} + +/** + * Exports training data for a specific behavior from a JABS project to an HDF5 file. + * Creates content-addressed files using SHA256 hashing and generates metadata manifests. + * + * @param behavior_name The name of the behavior to export training data for + * @param behavior_path The filesystem-safe path derived from the behavior name + * @param project_folder_name The name of the JABS project folder containing the training data + * + * @return h5_file_with_hash A tuple containing behavior info, the content-addressed HDF5 file, and its SHA256 hash + * @return *.h5.manifest.json A JSON manifest file containing metadata about the training data export + */ +process EXPORT_TRAINING_DATA { + tag "export_${behavior_path}" + label 'jabs_classify' + label 'cpu' + + publishDir path: "${params.classifier_base_path}/${params.jabs_version}/${behavior_path}", + mode: 'copy' + + input: + tuple val(behavior_name), val(behavior_path), val(project_folder_name) + + output: + tuple val(behavior_name), val(behavior_path), val(project_folder_name), path("*.h5"), env('HASH'), emit: h5_file_with_hash + path "*.h5.manifest.json" + + script: + def project_path = "${params.classifier_project_folders}/${project_folder_name}" + """ + # 1. Export to a local file + jabs-cli export-training --behavior "${behavior_name}" --outfile "training.h5" "${project_path}" + + # 2. Calculate hash and export it + export HASH=\$(sha256sum "training.h5" | awk '{ print \$1 }') + + # 3. Create the content-addressed files + mv training.h5 "\${HASH}.h5" + + # 4. Create the manifest file + cat > "\${HASH}.h5.manifest.json" < "\${HASH}.pickle.manifest.json" < [k, v] }) + + // Map to include behavior_path + behavior_projects_ch = classifier_ch.map { behavior_name, details -> + def behavior_path = behavior_name.replaceAll(' ', '_').replaceAll('[()]', '') + tuple(behavior_name, behavior_path, details.project_folder_name) + } + + // Extract unique project folders + unique_projects_ch = classifier_ch + .map { behavior_name, details -> details.project_folder_name } + .unique() + + // Calcualte project windows outside of container so we have access to jq + project_with_windows = CALCULTE_PROJECT_WINDOW_SIZES(unique_projects_ch) + + // Initialize each unique project + initialized_projects_ch = INIT_JABS_PROJECTS(project_with_windows) + + // Create a value channel from initialized projects for joining + initialized_projects_val = initialized_projects_ch.initialized_project + .collect() + .map { projects -> + projects.collectEntries { [it, true] } + } + + // Join behaviors with their initialized projects + ready_behaviors_ch = behavior_projects_ch + .combine(initialized_projects_val) + .map { behavior_name, behavior_path, project_folder_name, project_map -> + // Check if this behavior's project has been initialized + if (project_map[project_folder_name]) { + tuple(behavior_name, behavior_path, project_folder_name) + } + } + + exported_h5_ch = EXPORT_TRAINING_DATA(ready_behaviors_ch) + + trained_classifiers_ch = TRAIN_CLASSIFIER(exported_h5_ch.h5_file_with_hash) + + // Collect only the behavior names that were successfully trained + trained_classifiers_ch.classifier_file + .map { behavior_name, behavior_path, _classifier_file, hash -> tuple(behavior_name, behavior_path, hash) } + .toList() + .set{ collected_behaviors } + + GENERATE_PIPELINE_CONFIG(collected_behaviors) +} diff --git a/nextflow/configs/profiles/sumner2.config b/nextflow/configs/profiles/sumner2.config index da11354..bc2e10f 100644 --- a/nextflow/configs/profiles/sumner2.config +++ b/nextflow/configs/profiles/sumner2.config @@ -14,10 +14,6 @@ params { * - filter_processed: whether to filter processed files in the batch * - jabs_version: version of JABS used for the classifiers - * - classifier_project_folders: directory containing the classifier project folders - * - classifier_training_file_folder: directory containing the classifier training files - * - exported_classifier_folder: directory containing the exported classifiers - * - classifier_artifact_suffix: suffix for the classifier artifacts * - classifier_window_sizes: window sizes cached for use in classifiers * - single_mouse_classifiers: classifiers for single mouse behavior. Each classifier is described as behavior name (identical key used in JABS): [ @@ -37,75 +33,64 @@ params { heuristic_classifier_folder = "/JABS-postprocess/heuristic_classifiers/" filter_processed = false - jabs_version = "0.18.1" - classifier_project_folders = "/projects/kumar-lab/multimouse-pipeline/nextflow-artifacts/project_folders/" - classifier_training_file_folder = "/projects/kumar-lab/multimouse-pipeline/nextflow-artifacts/training_files/" - exported_classifier_folder = "/projects/kumar-lab/multimouse-pipeline/nextflow-artifacts/exported_classifiers/" - classifier_artifact_suffix = "_classifier_v${jabs_version}.pickle" + jabs_version = "v0.37.0" classifier_window_sizes = [2, 5, 10, 20, 30, 60] // Classifiers are described as behavior_name: project_folder + jabs_version = "v0.37.0" + classifier_window_sizes = [2, 3, 5, 10, 16, 20, 30, 60] single_mouse_classifiers = [ - "grooming": [ - "project_folder_name": "grooming", - "stitch_value": 10*30, - "filter_value": 3*30, + 'grooming': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/grooming/12199145d31213f79e23669d211cfa2fd33cedb9f11197b44f90a4b71297b6d2.pickle', + stitch_value: 300, + filter_value: 90 ], - "scratch": [ - "project_folder_name": "jabs-paper", - "stitch_value": 5, - "filter_value": 5, + 'Side seizure': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Side_seizure/50e865cfb045ff5b0c6bbad1dabf6465de43ef52ab4e13900b915986a51e9a0c.pickle', + stitch_value: 5, + filter_value: 5 ], - // "Leg_splaying": [ // Currently can't be exported successfully - // "project_folder_name": "ptz", - // "stitch_value": 5, - // "filter_value": 5, - // ], - "Side_seizure": [ - "project_folder_name": "ptz", - "stitch_value": 5, - "filter_value": 5, + 'scratch': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/scratch/261f1c21f015d47d30154fbbec98cb8860a40fbe383d3d86d6778e3da0f88a18.pickle', + stitch_value: 5, + filter_value: 5 ], - "Tail_jerk": [ - "project_folder_name": "ptz", - "stitch_value": 5, - "filter_value": 5, + 'Tail jerk': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Tail_jerk/34313467d88e7fc64ee90e10549eb48cfb9782d5e7a246fe106188c8ae50cdff.pickle', + stitch_value: 5, + filter_value: 5 ], - "Wild_jumping": [ - "project_folder_name": "ptz", - "stitch_value": 5, - "filter_value": 5, + 'Wild jumping': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Wild_jumping/7f114ce1b23a978afe82b4d9b7ec94c20982a2d5e7d16aefe4c5e0383324096a.pickle', + stitch_value: 5, + filter_value: 5 ], - "Escape": [ - "project_folder_name": "jabs-vivek", - "stitch_value": 5, - "filter_value": 5, + 'Escape': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Escape/0fa6b28e58fcc873f0c4ef053f5118566589d62d8e445b34ceaf1f962b5f8915.pickle', + stitch_value: 5, + filter_value: 5 ], - "Rearing_supported": [ - "project_folder_name": "jabs-vivek", - "stitch_value": 5, - "filter_value": 5, + 'Rearing (unsupported)': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Rearing_unsupported/562453b77d1a681a75c49bd774b5451e79e9e5f8784b48978f0f8cabbd90721e.pickle', + stitch_value: 5, + filter_value: 5 ], - "Rearing_unsupported": [ - "project_folder_name": "jabs-vivek", - "stitch_value": 5, - "filter_value": 5, + 'Rearing (supported)': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Rearing_supported/09ce1ed47e12556866045306bd221328454a8ccb5bc7ea7778a97e4882b386b6.pickle', + stitch_value: 5, + filter_value: 5 ], - "Turn_left": [ - "project_folder_name": "jabs-vivek", - "stitch_value": 5, - "filter_value": 5, + 'Turn left': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Turn_left/4779947116201f5fab5f2ec04f0ad4ee97a1e83649b6089d64354a7b07c42e4b.pickle', + stitch_value: 5, + filter_value: 5 ], - "Turn_right": [ - "project_folder_name": "jabs-vivek", - "stitch_value": 5, - "filter_value": 5, + 'Turn right': [ + classifier_path: '/projects/kumar-lab/data/jabs/classifiers/v0.37.0/Turn_right/b6fe24481513b87b757a61ad9495f7a9af270fad3cbec5d6b25556ece5f9c50a.pickle', + stitch_value: 5, + filter_value: 5 ], - // "Jerk": [ // Needs to be re-exported to remove static object features (unavailable for single mouse) - // "project_folder_name": "social-play", - // "stitch_value": 1, - // "filter_value": 6, - // ] ] + heuristic_classifiers = ["corner", "corner_facing", "freeze", "locomotion", "periphery", "wall_facing"] // Number of 5-minute bins for transforming summary tables into bins diff --git a/nextflow/modules/jabs_classifiers.nf b/nextflow/modules/jabs_classifiers.nf index d890274..6f9efc8 100644 --- a/nextflow/modules/jabs_classifiers.nf +++ b/nextflow/modules/jabs_classifiers.nf @@ -67,10 +67,9 @@ process PREDICT_CLASSIFIERS { script: """ - for classifier in ${classifiers.keySet().collect { params.exported_classifier_folder + it + params.classifier_artifact_suffix }.join(' ')}; + for classifier_path in ${classifiers.collect { _behavior, details -> details.classifier_path }.join(' ')}; do - ln -s \${classifier} . - jabs-classify classify --classifier \$(basename \${classifier}) --input-pose ${in_pose} --out-dir . --feature-dir . + jabs-classify classify --classifier "\${classifier_path}" --input-pose ${in_pose} --out-dir . --feature-dir . done """ } @@ -105,12 +104,12 @@ process GENERATE_BEHAVIOR_TABLES { script: """ - behavior_command="--behavior ${classifiers.collect { entry -> "$entry.key --stitch_gap $entry.value.stitch_value --min_bout_length $entry.value.filter_value" }.join(' --behavior ')}" + behavior_command="--behavior ${classifiers.collect { entry -> "$entry.key --stitch-gap $entry.value.stitch_value --min-bout-length $entry.value.filter_value" }.join(' --behavior ')}" jabs-postprocess generate-tables \ - --project_folder . \ - --feature_folder . \ - --out_prefix ${in_pose.baseName} \ - --out_bin_size 5 \ + --project-folder . \ + --feature-folder . \ + --out-prefix ${in_pose.baseName} \ + --out-bin-size 5 \ \${behavior_command} """ } @@ -149,7 +148,7 @@ process PREDICT_HEURISTICS { --feature-folder . \ --behavior-config \${classifier} \ --out-prefix ${in_pose.baseName} \ - --out_bin_size 5 + --out-bin-size 5 done """ }